import pandas as pd
import shap
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
from numpy import loadtxt
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score, roc_auc_score, jaccard_score
from sklearn.inspection import permutation_importance
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler
import warnings
df = pd.read_csv('credit_risk_dataset.csv')
df
df["loan_status"].value_counts()
print(f"{int(100*df[df['loan_status']==1].shape[0] / df.shape[0])}% of Rows are Defaults")
The dataset is imbalanced at almost 1:5 ratio, therefore the defaults are the least frequent class and the most important one.
Impute NaN and Drop Duplicates
df["person_emp_length"].fillna(df["person_emp_length"].median(), inplace=True)
df["loan_int_rate"].fillna(df["loan_int_rate"].median(), inplace=True)
df.drop_duplicates(inplace=True)
df[df.isnull().any(axis=1)]
Correlation Analysis
df.corr()
corr = df.corr()
corr = corr.round(2)
plt.figure(figsize=(12, 9))
sns.heatmap(corr,
xticklabels=corr.columns,
yticklabels=corr.columns,
annot=True)
The variable of interest `loan_status` shows a degree of correlation with the loan's interest rate `loan_int_rate` and with the percentage of the income that the rent represents.
One Hot Encoding
categorical_columns = df.columns[df.dtypes == 'object']
categorical_columns
for column in categorical_columns:
df = pd.concat([df.drop(columns=[column]), pd.get_dummies(df[column], prefix=column, drop_first=True)],axis=1)
df
Fit the XGBOOST Model in a Two-Folded Cross-Validated Grid Search
# Split data into X and y
X = df.drop(columns='loan_status')
Y = df['loan_status']
scaler = StandardScaler().fit(X)
X = scaler.transform(X)
# Split data into train and test sets
seed = 7
test_size = 0.33
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=test_size, random_state=seed)
param_grid = {
'min_child_weight': [1, 5, 10],
'gamma': [0.5, 1, 1.5, 2, 5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'max_depth': [3, 4, 5]
}
model = XGBClassifier(scale_pos_weight=1, n_estimators=100, eval_metric='logloss', objective='binary:logistic', use_label_encoder=False)
clf = GridSearchCV(model, param_grid, scoring='f1_macro', cv=2, verbose=3)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print('-'*50)
print('Best Estimator\n')
print(clf.best_estimator_)
print('-'*50)
Top 10 Performing Models
grid_search_cv = pd.DataFrame.from_dict(clf.cv_results_).sort_values(by=['rank_test_score']).head(100)
grid_search_cv.index = grid_search_cv.rank_test_score
grid_search_cv.filter(regex='param_')
More Depth seems to have a positive impact in the model as 56% of the 100 best performing ones have the maximum depth in the grid, while less `min child weight` seems to have produced the better results with 61% of the best performing models having 1.
Evaluate Predictions
predictions = [(value > 0.5).astype(int) for value in y_pred]
tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()
print(f"True Negative: {tn}\nFalse Positive: {fp}\nFalse Negative: {fn}\nTrue Positive: {tp}")
print("\n")
print(classification_report(y_test, y_pred, target_names = ['Non-Default', 'Default']))
The default class (most important) has a F1 Score of 82%, showing a decent result for a production model.
Feature Importance Through Permutation
df = df.drop(columns='loan_status')
perm_importance = permutation_importance(clf.best_estimator_, X_test, y_test)
sorted_idx = perm_importance.importances_mean.argsort()
plt.barh(df.columns[sorted_idx], perm_importance.importances_mean[sorted_idx], height=0.2)
plt.xlabel("Permutation Importance")
plt.tick_params(axis='y', which='major', labelsize=7)
plt.figure(figsize=(14, 10))
plt.show()