!pip install catboost

!pip install imblearn==0.0

!pip install xgboost==1.4.1

!pip install lightgbm==3.2.1

import pandas as pd import numpy as np import matplotlib as mpl import matplotlib.pyplot as plt import seaborn as sns from sklearn.impute import SimpleImputer from sklearn.preprocessing import LabelEncoder, StandardScaler from sklearn.model_selection import train_test_split, cross_validate, GridSearchCV, cross_val_score from imblearn.under_sampling import RandomUnderSampler from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, classification_report from sklearn.pipeline import make_pipeline from sklearn.model_selection import StratifiedKFold from xgboost import XGBClassifier from lightgbm import LGBMClassifier from catboost import CatBoostClassifier sns.set_style() import warnings warnings.filterwarnings('ignore') COLOR = '#ababab' mpl.rcParams['figure.titlesize'] = 16 mpl.rcParams['text.color'] = 'black' mpl.rcParams['axes.labelcolor'] = COLOR mpl.rcParams['xtick.color'] = COLOR mpl.rcParams['ytick.color'] = COLOR mpl.rcParams['grid.color'] = COLOR mpl.rcParams['grid.alpha'] = 0.1 df_credit = pd.read_csv('http://dl.dropboxusercontent.com/s/xn2a4kzf0zer0xu/acquisition_train.csv?dl=0') df_credit.head()

print('Number of rows: ', df_credit.shape[0]) print('Number of columns: ', df_credit.shape[1]) df_credit.info()

df_credit.dropna(subset=['target_default'], inplace=True)

print('Number of rows: ', df_credit.shape[0]) print('Number of columns: ', df_credit.shape[1]) df_credit.info()

df_credit.nunique().sort_values()

df_credit.drop(labels=['channel', 'external_data_provider_credit_checks_last_2_year'], axis=1, inplace=True)

df_credit.drop(labels=['email', 'reason', 'zip', 'job_name', 'external_data_provider_first_name', 'lat_lon', 'shipping_zip_code', 'user_agent', 'profile_tags', 'marketing_channel', 'profile_phone_number', 'application_time_applied', 'ids'], axis=1, inplace=True)

df_credit.describe()

np.isinf(df_credit['reported_income']).sum()

df_credit['reported_income'] = df_credit['reported_income'].replace(np.inf, np.nan) df_credit.loc[df_credit['external_data_provider_email_seen_before'] == -999, 'external_data_provider_email_seen_before'] = np.nan

df_credit_numerical = df_credit[['score_3', 'risk_rate', 'last_amount_borrowed', 'last_borrowed_in_months', 'credit_limit', 'income', 'ok_since', 'n_bankruptcies', 'n_defaulted_loans', 'n_accounts', 'n_issues', 'external_data_provider_email_seen_before']]

nrows = 3 ncols = 4 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(25, 16)) r = 0 c = 0 for i in df_credit_numerical: sns.distplot(df_credit_numerical[i], bins=15,kde=False, ax=ax[r][c]) if c == ncols - 1: r += 1 c = 0 else: c += 1 plt.show()

df_credit_num = df_credit.select_dtypes(exclude='object').columns df_credit_cat = df_credit.select_dtypes(include='object').columns # fill missing values for "last_amount_borrowed", "last_borrowed_in_months" and "n_issues" df_credit['last_amount_borrowed'].fillna(value=0, inplace=True) df_credit['last_borrowed_in_months'].fillna(value=0, inplace=True) df_credit['n_issues'].fillna(value=0, inplace=True) # fill missing values for numerical variables imputer = SimpleImputer(missing_values=np.nan, strategy='median') imputer = imputer.fit(df_credit.loc[:, df_credit_num]) df_credit.loc[:, df_credit_num] = imputer.transform(df_credit.loc[:, df_credit_num]) # fill missing values for categorical variables imputer = SimpleImputer(missing_values=np.nan, strategy='most_frequent') imputer = imputer.fit(df_credit.loc[:, df_credit_cat]) df_credit.loc[:, df_credit_cat] = imputer.transform(df_credit.loc[:, df_credit_cat])

nrows = 3 ncols = 4 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(25, 16)) r = 0 c = 0 for i in df_credit_numerical: sns.distplot(df_credit_numerical[i], bins=15,kde=False, ax=ax[r][c]) if c == ncols - 1: r += 1 c = 0 else: c += 1 plt.show()

bin_var = df_credit.nunique()[df_credit.nunique() == 2].keys().tolist() num_var = [col for col in df_credit.select_dtypes(['int', 'float']).columns.tolist() if col not in bin_var] cat_var = [col for col in df_credit.select_dtypes(['object']).columns.tolist() if col not in bin_var] df_credit_encoded = df_credit.copy() # label encoding for the binary variables le = LabelEncoder() for col in bin_var: df_credit_encoded[col] = le.fit_transform(df_credit_encoded[col]) # encoding with get_dummies for the categorical variables df_credit_encoded = pd.get_dummies(df_credit_encoded, columns=cat_var) df_credit_encoded.head()

# feature matrix X = df_credit_encoded.drop('target_default', axis=1) # target vector y = df_credit_encoded['target_default'] X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True, stratify=y)

# standardize numerical variables scaler = StandardScaler().fit(X_train) X_train = scaler.transform(X_train) # resample rus = RandomUnderSampler() X_train_rus, y_train_rus = rus.fit_resample(X_train, y_train)

def val_model(X, y, clf, show=True): """ Apply cross-validation on the training set. # Arguments X: DataFrame containing the independent variables. y: Series containing the target vector. clf: Scikit-learn estimator instance. # Returns float, mean value of the cross-validation scores. """ X = np.array(X) y = np.array(y) pipeline = make_pipeline(StandardScaler(), clf) scores = cross_val_score(pipeline, X, y, scoring='recall') if show == True: print(f'Recall: {scores.mean()}, {scores.std()}') return scores.mean() #evaluate the models xgb = XGBClassifier() lgb = LGBMClassifier() cb = CatBoostClassifier() model = [] recall = [] for clf in (xgb, lgb, cb): model.append(clf.__class__.__name__) recall.append(val_model(X_train_rus, y_train_rus, clf, show=False)) pd.DataFrame(data=recall, index=model, columns=['Recall'])

# final XGBoost model xgb = XGBClassifier(max_depth=3, learning_rate=0.0001, n_estimators=50, gamma=1, min_child_weight=6) xgb.fit(X_train_rus, y_train_rus) # prediction X_test_xgb = scaler.transform(X_test) y_pred_xgb = xgb.predict(X_test_xgb) # classification report print(classification_report(y_test, y_pred_xgb)) # confusion matrix fig, ax = plt.subplots() sns.heatmap(confusion_matrix(y_test, y_pred_xgb, normalize='true'), annot=True, ax=ax) ax.set_title('Confusion Matrix - XGBoost') ax.set_xlabel('Predicted Value') ax.set_ylabel('Real Value') plt.show()

# final LightGBM model lgb = LGBMClassifier(num_leaves=70, max_depth=5, learning_rate=0.01, min_data_in_leaf=400) lgb.fit(X_train_rus, y_train_rus) # prediction X_test_lgb = scaler.transform(X_test) y_pred_lgb = lgb.predict(X_test_lgb) # classification report print(classification_report(y_test, y_pred_lgb)) # confusion matrix fig, ax = plt.subplots() sns.heatmap(confusion_matrix(y_test, y_pred_lgb, normalize='true'), annot=True, ax=ax) ax.set_title('Confusion Matrix - LightGBM') ax.set_xlabel('Predicted Value') ax.set_ylabel('Real Value') plt.show()

# final CatBoost model cb = CatBoostClassifier(learning_rate=0.03, depth=6, l2_leaf_reg=5, logging_level='Silent') cb.fit(X_train_rus, y_train_rus) # prediction X_test_cb = scaler.transform(X_test) y_pred_cb = cb.predict(X_test_cb) # classification report print(classification_report(y_test, y_pred_cb)) # confusion matrix fig, ax = plt.subplots() sns.heatmap(confusion_matrix(y_test, y_pred_cb, normalize='true'), annot=True, ax=ax) ax.set_title('Confusion Matrix - CatBoost') ax.set_xlabel('Predicted Value') ax.set_ylabel('Real Value') plt.show()