机器学习分类算法之XGBoost（集成学习算法）

​

#导入所需要的包
from sklearn.metrics import precision_score
from sklearn.model_selection import train_test_split
import xgboost as xgb
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV #网格搜索
import matplotlib.pyplot as plt#可视化
import seaborn as sns#绘图包
# 忽略警告
import warnings
warnings.filterwarnings("ignore")

plt.figure(figsize=(20,20))
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus']=False
sns.heatmap(df.corr(),cmap="YlGnBu",annot=True)
plt.title("相关性分析图")

model=xgb.XGBClassifier()
# 训练模型
model.fit(X_train,y_train)
# 预测值
y_pred = model.predict(X_test)


'''
评估指标
'''
# 求出预测和真实一样的数目
true = np.sum(y_pred == y_test )
print('预测对的结果数目为：', true)
print('预测错的的结果数目为：', y_test.shape[0]-true)
# 评估指标
from sklearn.metrics import accuracy_score,precision_score,recall_score,f1_score,cohen_kappa_score
print('预测数据的准确率为： {:.4}%'.format(accuracy_score(y_test,y_pred)*100))
print('预测数据的精确率为：{:.4}%'.format(
      precision_score(y_test,y_pred)*100))
print('预测数据的召回率为：{:.4}%'.format(
      recall_score(y_test,y_pred)*100))
# print("训练数据的F1值为：", f1score_train)
print('预测数据的F1值为：',
      f1_score(y_test,y_pred))
print('预测数据的Cohen’s Kappa系数为：',
      cohen_kappa_score(y_test,y_pred))
# 打印分类报告
print('预测数据的分类报告为：','
',
      classification_report(y_test,y_pred))

scorel = []
for i in range(0,300,10):
    model = xgb.XGBClassifier(n_estimators=i+1,
                                 n_jobs=--4,
                                 random_state=90).fit(X_train,y_train)
    score = model.score(X_test,y_test)
    scorel.append(score)

print(max(scorel),(scorel.index(max(scorel))*10)+1)  #作图反映出准确度随着估计器数量的变化，51的附近最好
plt.figure(figsize=[20,5])
plt.plot(range(1,300,10),scorel)
plt.show()

scorel = []
for i in range(0,20,1):
    model = xgb.XGBClassifier(n_estimators=i+1,
                                 n_jobs=--4,
                                 random_state=90).fit(X_train,y_train)
    score = model.score(X_test,y_test)
    scorel.append(score)

print(max(scorel),(scorel.index(max(scorel))*1)+1)  #作图反映出准确度随着估计器数量的变化，51的附近最好
plt.figure(figsize=[20,5])
plt.plot(range(0,20,1),scorel)
plt.show()

scorel = []
for i in range(0,20,1):
    model = xgb.XGBClassifier(max_depth =i,
                                 n_estimators=8,
                                 n_jobs=--4,
                                 random_state=90).fit(X_train,y_train)
    score = model.score(X_test,y_test)
    scorel.append(score)

print(max(scorel),(scorel.index(max(scorel))*1)+1)  #作图反映出准确度随着估计器数量的变化，51的附近最好
plt.figure(figsize=[20,5])
plt.plot(range(0,20,1),scorel)
plt.show()

param_test1 = {
# 'n_estimators':list(range(3,15,1)),
 'max_depth':list(range(3,10,2)),
 'min_child_weight':list(range(1,6,2))
}
gsearch1 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=8, max_depth=8,
 min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,
 objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), 
 param_grid = param_test1, scoring='roc_auc',iid=False, cv=5)
gsearch1=gsearch1.fit(X,y)
gsearch1.best_params_, gsearch1.best_score_

param_test1 = {
 'max_depth':list(range(8,11,1)),
 'min_child_weight':list(range(1,4,1))
}
gsearch1 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=7, max_depth=3,
 min_child_weight=5, gamma=0, subsample=0.8, colsample_bytree=0.8,
 objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), 
 param_grid = param_test1, scoring='roc_auc',iid=False, cv=5)
gsearch1.fit(X,y)
gsearch1.best_params_, gsearch1.best_score_

param_test3 = {
 'gamma':[i/10.0 for i in range(0,5)]
}
gsearch3 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=8, max_depth=8,
 min_child_weight=3, gamma=0, subsample=0.8, colsample_bytree=0.8,
 objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), 
 param_grid = param_test3, scoring='roc_auc',iid=False, cv=5)
gsearch3.fit(X,y)
gsearch3.best_params_, gsearch3.best_score_

param_test5 = {
 'subsample':[i/100.0 for i in range(75,90,5)],
 'colsample_bytree':[i/100.0 for i in range(75,90,5)]
}
gsearch5 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=8, max_depth=8,
 min_child_weight=3, gamma=0.4, subsample=0.8, colsample_bytree=0.8,
 objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), 
 param_grid = param_test5, scoring='roc_auc',iid=False, cv=5)
gsearch5.fit(X_train,y_train)
gsearch5.best_params_, gsearch5.best_score_

param_test6 = {
 'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100]
}
gsearch6 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=8, max_depth=8,
 min_child_weight=3, gamma=0.2, subsample=0.85, colsample_bytree=0.85,
 objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), 
 param_grid = param_test6, scoring='roc_auc',iid=False, cv=5)
gsearch6.fit(X_train,y_train)
gsearch6.best_params_, gsearch6.best_score_

param_test6 = {
 'learning_rate':[0.01, 0.02, 0.1, 0.2]
}
gsearch6 = GridSearchCV(estimator = xgb.XGBClassifier( learning_rate =0.1, n_estimators=8, max_depth=8,
 min_child_weight=1, gamma=0.2, subsample=0.8, colsample_bytree=0.85,
 objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), 
 param_grid = param_test6, scoring='roc_auc',iid=False, cv=5)
gsearch6.fit(X_train,y_train)
gsearch6.best_params_, gsearch6.best_score_

# learning_rate =0.1, n_estimators=8, max_depth=8,
#  min_child_weight=1, gamma=0.2, subsample=0.8, colsample_bytree=0.85,
#  objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27
import numpy as np
from sklearn.model_selection import GridSearchCV

parameters = {'n_estimators':np.arange(6,10,1)
              ,'max_depth':np.arange(6,10,1)
              ,"colsample_bytree":np.arange(0.8,0.9,0.05)
              ,"subsample":np.arange(0.8,0.9,0.05)
              ,'gamma':np.arange(0.1,0.5,0.1)
              ,'min_child_weight':np.arange(1,3,1)
}

clf = xgb.XGBClassifier(random_state=25)
GS = GridSearchCV(clf, parameters, cv=5) # cv交叉验证
GS.fit(X_train,y_train)
print(GS.best_params_)
print(GS.best_score_)

model=xgb.XGBClassifier(eta =0.1, n_estimators=8, max_depth=8,
 min_child_weight=2, gamma=0.8, subsample=0.85, colsample_bytree=0.8)


# 训练模型
model.fit(X_train,y_train)
# 预测值
y_pred = model.predict(X_test)


'''
评估指标
'''
# 求出预测和真实一样的数目
true = np.sum(y_pred == y_test )
print('预测对的结果数目为：', true)
print('预测错的的结果数目为：', y_test.shape[0]-true)
# 评估指标
from sklearn.metrics import accuracy_score,precision_score,recall_score,f1_score,cohen_kappa_score
print('预测数据的准确率为： {:.4}%'.format(accuracy_score(y_test,y_pred)*100))
print('预测数据的精确率为：{:.4}%'.format(
      precision_score(y_test,y_pred)*100))
print('预测数据的召回率为：{:.4}%'.format(
      recall_score(y_test,y_pred)*100))
# print("训练数据的F1值为：", f1score_train)
print('预测数据的F1值为：',
      f1_score(y_test,y_pred))
print('预测数据的Cohen’s Kappa系数为：',
      cohen_kappa_score(y_test,y_pred))
# 打印分类报告
print('预测数据的分类报告为：','
',
      classification_report(y_test,y_pred))

from xgboost import plot_importance
# plt.figure(figsize=(15,15))
plt.rcParams["figure.figsize"] = (14, 8)
plot_importance(model)

xgboosts=xgb.to_graphviz(model)
xgboosts.format = 'png'
xgboosts.view('./xgboost')

# ROC曲线、AUC
from sklearn.metrics import precision_recall_curve
from sklearn import metrics
# 预测正例的概率
y_pred_prob=model.predict_proba(X_test)[:,1]
# y_pred_prob ,返回两列，第一列代表类别0,第二列代表类别1的概率
#https://blog.csdn.net/dream6104/article/details/89218239
fpr, tpr, thresholds = metrics.roc_curve(y_test,y_pred_prob, pos_label=2)
#pos_label，代表真阳性标签，就是说是分类里面的好的标签，这个要看你的特征目标标签是0,1，还是1,2
roc_auc = metrics.auc(fpr, tpr)  #auc为Roc曲线下的面积
# print(roc_auc)
plt.figure(figsize=(8,6))
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
plt.plot(fpr, tpr, 'r',label='AUC = %0.2f'% roc_auc)
plt.legend(loc='lower right')
# plt.plot([0, 1], [0, 1], 'r--')
plt.xlim([0, 1.1])
plt.ylim([0, 1.1])
plt.xlabel('False Positive Rate') #横坐标是fpr
plt.ylabel('True Positive Rate')  #纵坐标是tpr
plt.title('Receiver operating characteristic example')
plt.show()