import tensorflow as tf

#精確率評價指標(biāo)
def metric_precision(y_true,y_pred): 
 TP=tf.reduce_sum(y_true*tf.round(y_pred))
 TN=tf.reduce_sum((1-y_true)*(1-tf.round(y_pred)))
 FP=tf.reduce_sum((1-y_true)*tf.round(y_pred))
 FN=tf.reduce_sum(y_true*(1-tf.round(y_pred)))
 precision=TP/(TP+FP)
 return precision

#召回率評價指標(biāo)
def metric_recall(y_true,y_pred): 
 TP=tf.reduce_sum(y_true*tf.round(y_pred))
 TN=tf.reduce_sum((1-y_true)*(1-tf.round(y_pred)))
 FP=tf.reduce_sum((1-y_true)*tf.round(y_pred))
 FN=tf.reduce_sum(y_true*(1-tf.round(y_pred)))
 recall=TP/(TP+FN)
 return recall

#F1-score評價指標(biāo)
def metric_F1score(y_true,y_pred): 
 TP=tf.reduce_sum(y_true*tf.round(y_pred))
 TN=tf.reduce_sum((1-y_true)*(1-tf.round(y_pred)))
 FP=tf.reduce_sum((1-y_true)*tf.round(y_pred))
 FN=tf.reduce_sum(y_true*(1-tf.round(y_pred)))
 precision=TP/(TP+FP)
 recall=TP/(TP+FN)
 F1score=2*precision*recall/(precision+recall)
 return F1score

#編譯階段引用自定義評價指標(biāo)示例
model.compile(optimizer='adam',
	 loss='binary_crossentropy',
	 metrics=['accuracy',
	 		metric_precision,
	 		metric_recall,
	 		metric_F1score])

from sklearn.model_selection import StratifiedKFold
import numpy

seed = 7 # 隨機種子
numpy.random.seed(seed) # 生成固定的隨機數(shù)
num_k = 5 # 多少折

# 整個數(shù)據(jù)集(自己定義)
X = 
Y = 

kfold = StratifiedKFold(n_splits=num_k, shuffle=True, random_state=seed) # 分層K折，保證類別比例一致

cvscores = []
for train, test in kfold.split(X, Y):

	# 可以用sequential或者function的方式建模（自己定義）
	model = 
 model.compile() # 自定義
 
	# 模型訓(xùn)練
 model.fit(X[train], Y[train], epochs=150, batch_size=10, verbose=0)
 
 # 模型測試
 scores = model.evaluate(X[test], Y[test], verbose=0)
 
 print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100)) # 打印出驗證集準(zhǔn)確率
 
 cvscores.append(scores[1] * 100)
 
print("%.2f%% (+/- %.2f%%)" % (numpy.mean(cvscores), numpy.std(cvscores))) # 輸出k-fold的模型平均和標(biāo)準(zhǔn)差結(jié)果

from keras.wrappers.scikit_learn import KerasClassifier # 使用keras下的sklearn API
from sklearn.cross_validation import StratifiedKFold, cross_val_score
import numpy as np

seed = 7 # 隨機種子
numpy.random.seed(seed) # 生成固定的隨機數(shù)
num_k = 5 # 多少折

# 整個數(shù)據(jù)集(自己定義)
X = 
Y = 

# 創(chuàng)建模型
def model():
 # 可以用sequential或者function的方式建模（自己定義）
	model = 
	return model 

model = KerasClassifier(build_fn=model, epochs=150, batch_size=10)
kfold = StratifiedKFold(Y, n_folds=num_k, shuffle=True, random_state=seed)
results = cross_val_score(model, X, Y, cv=kfold)
print(np.average(results)) # 輸出k-fold的模型平均結(jié)果

from sklearn import datasets
import numpy as np
from sklearn.preprocessing import label_binarize
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, precision_score, accuracy_score,recall_score, f1_score,roc_auc_score, precision_recall_fscore_support, roc_curve, classification_report
import matplotlib.pyplot as plt

iris = datasets.load_iris()
x, y = iris.data, iris.target
print("label:", y)
n_class = len(set(iris.target))
y_one_hot = label_binarize(y, np.arange(n_class))

# alpha = np.logspace(-2, 2, 20) #設(shè)置超參數(shù)范圍
# model = LogisticRegressionCV(Cs = alpha, cv = 3, penalty = 'l2') #使用L2正則化
model = LogisticRegression() # 內(nèi)置了最大迭代次數(shù)了，可修改
model.fit(x, y)
y_score = model.predict(x) # 輸出的是整數(shù)標(biāo)簽
mean_accuracy = model.score(x, y)
print("mean_accuracy: ", mean_accuracy)
print("predict label:", y_score)
print(y_score==y)
print(y_score.shape)
y_score_pro = model.predict_proba(x) # 輸出概率
print(y_score_pro)
print(y_score_pro.shape)
y_score_one_hot = label_binarize(y_score, np.arange(n_class)) # 這個函數(shù)的輸入必須是整數(shù)的標(biāo)簽哦
print(y_score_one_hot.shape)

obj1 = confusion_matrix(y, y_score) # 注意輸入必須是整數(shù)型的，shape=(n_samples, )
print('confusion_matrix\n', obj1)

print(y)
print('accuracy:{}'.format(accuracy_score(y, y_score))) # 不存在average
print('precision:{}'.format(precision_score(y, y_score,average='micro')))
print('recall:{}'.format(recall_score(y, y_score,average='micro')))
print('f1-score:{}'.format(f1_score(y, y_score,average='micro')))
print('f1-score-for-each-class:{}'.format(precision_recall_fscore_support(y, y_score))) # for macro
# print('AUC y_pred = one-hot:{}\n'.format(roc_auc_score(y_one_hot, y_score_one_hot,average='micro'))) # 對于multi-class輸入必須是proba，所以這種是錯誤的

# AUC值
auc = roc_auc_score(y_one_hot, y_score_pro,average='micro') # 使用micro，會計算n_classes個roc曲線，再取平均
print("AUC y_pred = proba:", auc)
# 畫ROC曲線
print("one-hot label ravelled shape:", y_one_hot.ravel().shape)
fpr, tpr, thresholds = roc_curve(y_one_hot.ravel(),y_score_pro.ravel()) # ravel()表示平鋪開來,因為輸入的shape必須是(n_samples,)
print("threshold： ", thresholds)
plt.plot(fpr, tpr, linewidth = 2,label='AUC=%.3f' % auc)
plt.plot([0,1],[0,1], 'k--') # 畫一條y=x的直線，線條的顏色和類型
plt.axis([0,1.0,0,1.0]) # 限制坐標(biāo)范圍
plt.xlabel('False Postivie Rate')
plt.ylabel('True Positive Rate')
plt.legend()
plt.show()

# p-r曲線針對的是二分類，這里就不描述了
ans = classification_report(y, y_score,digits=5) # 小數(shù)點后保留5位有效數(shù)字
print(ans)