竞赛实战--天池金融风控分类题目

背景

1、金融风控分类题目，作为机器学习竞赛是一个比力好的选择
2、怎样举行数据处理
代码

数据分析部分

1. #!/usr/bin/env python
2. # coding: utf-8
4. import os
5. import gc
6. import numpy as np
7. import pandas as pd
8. import warnings
9. import lightgbm as lgb
10. import catboost as cbt
11. import xgboost as xgb
12. from sklearn.metrics import roc_auc_score
13. from sklearn.model_selection import StratifiedKFold, KFold, train_test_split, GridSearchCV
14. from sklearn.preprocessing import LabelEncoder, StandardScaler
15. from tqdm import tqdm
16. import matplotlib.pyplot as plt
17. import seaborn as sns
18. from scipy.stats import kstest
20. warnings.filterwarnings("ignore")
21. pd.set_option('display.max_columns', None)
22. # plt.ion()

复制代码

2. # ## 导入数据
3. BASE_DIR = os.path.dirname(os.path.abspath(__file__))
4. DATA_PATH = os.path.join(BASE_DIR, 'data')
6. train_data_file = os.path.join(DATA_PATH, "train.csv")
7. train_data = pd.read_csv(train_data_file)
9. test_data_file = os.path.join(DATA_PATH, "testA.csv")
10. test_data = pd.read_csv(test_data_file)
12. target = train_data['isDefault']
13. train_data = train_data.drop(['isDefault'], axis=1)
15. data = pd.concat([train_data, test_data])
18. objectList = [i for i in train_data.columns if train_data[i].dtype == 'O']
19. classList = [i for i in train_data.select_dtypes(exclude=['object']).columns if len(train_data[i].unique()) <= 10]
20. numericalList = [i for i in train_data.select_dtypes(exclude=['object']).columns if i not in classList]

复制代码

对差别类型变量举行分类分组处理

1. # ## 变量分类和缺失值处理
2. info = pd.DataFrame(data.isnull().sum())
3. info = info[info[0] != 0]
4. miss_fea = info.index
6. miss_objectList = [i for i in miss_fea if i in objectList]
7. miss_classList = [i for i in miss_fea if i in classList]
8. miss_numericalList = [i for i in miss_fea if i in numericalList]
10. # 填补缺失值
11. data['employmentLength'] = data['employmentLength'].fillna(0)
12. data['n11'] = data['n11'].fillna(0)
13. data['n12'] = data['n12'].fillna(0)
14. data['employmentTitle'] = data['employmentTitle'].fillna(data['employmentTitle'].mode()[0])
15. data['postCode'] = data['postCode'].fillna(data['postCode'].mode()[0])
16. data['dti'] = data['dti'].fillna(data['postCode'].mean())
17. data['pubRecBankruptcies'] = data['pubRecBankruptcies'].fillna(data['pubRecBankruptcies'].mean())
18. data['revolUtil'] = data['revolUtil'].fillna(data['revolUtil'].mean())
19. data['title'] = data['title'].fillna(data['title'].mode()[0])
21. NoNameList = [i for i in miss_numericalList if i.startswith("n")]
22. for i in NoNameList:
23.     data[i] = data[i].fillna(data[i].mode()[0])
25. # ## object 变量处理
26. data['employmentLength'].replace({'10+ years': '10 years', '< 1 year': '0 years', '0': '0 years'}, inplace=True)
27. data['employmentLength'] = data['employmentLength'].apply(lambda s: int(str(s).split()[0]) if pd.notnull(s) else s)
29. data['earliesCreditLine'] = data['earliesCreditLine'].apply(lambda s: int(s[-4:]))
30. data = data.drop(['issueDate'], axis=1)
32. le = LabelEncoder()
33. data['grade'] = le.fit_transform(data['grade'])
34. data['subGrade'] = le.fit_transform(data['subGrade'])
36. # 删除不需要的列
37. dropList = ['id', 'ficoRangeHigh', 'applicationType', 'policyCode', 'n3', 'n11', 'n12', 'n13']
38. data.drop(dropList, axis=1, inplace=True)
41. train_data = data[:800000]
42. # 将target和train_data进行重新拼接
43. train_data['isDefault']=target
44. test_data = data[800000:]
45. print("Divide data.")

复制代码

1. # # ## 异常值处理
2. # percentile = pd.DataFrame()
3. # numList = [i for i in train_data.columns if i not in classList]
5. # # 正态分布检测
6. # for i in numList:
7. #     print(kstest(data[i], 'norm', (data[i].mean(), data[i].std())))
9. # # 异常值处理
10. # stdsc = StandardScaler()
11. # for i in numList:
12. #     new_i = "zheng_" + i
13. #     train_data[new_i] = stdsc.fit_transform(train_data[i].values.reshape(-1, 1))
14. #     data_std = np.std(train_data[new_i])
15. #     data_mean = np.mean(train_data[new_i])
16. #     outliers_cut_off = data_std * 3
17. #     lower_rule = data_mean - outliers_cut_off
18. #     upper_rule = data_mean + outliers_cut_off
19. #     train_data = train_data[(train_data[new_i] < upper_rule) & (train_data[new_i] > lower_rule)]
20. # train_data = train_data.iloc[:, :38]

复制代码

生存数据，在部分情况下由于数据体量过大，生存中间数据有助于后续处理。

1. FEATURE_PATH = os.path.join(BASE_DIR, 'feature')
2. feature_train_data = os.path.join(FEATURE_PATH, 'train_data.csv')
3. feature_test_data = os.path.join(FEATURE_PATH, 'test_data.csv')
4. train_data.to_csv(feature_train_data,index=0)
5. test_data.to_csv(feature_test_data,index=0)

复制代码

模子搭建部分

1. # 定义模型训练函数
2. def train_model(x_train, y_train, test_data, params, n_splits=5):
3.     skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2019)
4.     oof = np.zeros(len(x_train))
5.     predictions = np.zeros((len(test_data), n_splits))
7.     for fold_, (train_idx, valid_idx) in enumerate(skf.split(x_train, y_train)):
8.         print(f"\nFold {fold_ + 1}")
9.         x_tr, x_val = x_train.iloc[train_idx], x_train.iloc[valid_idx]
10.         y_tr, y_val = y_train.iloc[train_idx], y_train.iloc[valid_idx]
12.         train_set = lgb.Dataset(x_tr, label=y_tr)
13.         val_set = lgb.Dataset(x_val, label=y_val)
15.         clf = lgb.train(params, train_set, 5000, valid_sets=[val_set],
16.                         verbose_eval=250, early_stopping_rounds=50)
18.         oof[valid_idx] = clf.predict(x_val, num_iteration=clf.best_iteration)
19.         predictions[:, fold_] = clf.predict(test_data, num_iteration=clf.best_iteration)
20.    
21.     print("\n\nCV AUC: {:<0.4f}".format(roc_auc_score(y_train, oof)))
23.     return oof, predictions
25. # 训练模型并生成预测
26. oof, predictions = train_model(x_train_gbdt, y_train_gbdt, x_test_bgdt, default_params)

复制代码

参考资料


免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。