Risk Probability Prediction

from jupyterthemes.stylefx import set_nb_theme set_nb_theme('gruvboxd')

%run Libraries.ipynb

%run Utility_tools.ipynb

raw = pd.read_csv('Risk_Analytics.csv') df = deepcopy(raw)

df.drop('Loan ID', axis=1, inplace=True)

df.rename(columns=lambda col: col.replace('?', '').replace(' ', '_').lower().strip(), inplace=True)

reduce_memory_usage(df)

def plot_countplot(data): sns.set_theme(style="darkgrid") plt.figure(figsize=(9,7)) ax = sns.countplot(data=data, x='payment') for p in ax.patches: total = len(data) ax.annotate('{:.1f}%'.format(p.get_height()/total*100), (p.get_x()+0.25, p.get_height()+0.01)) plt.show();

plot_countplot(df)

df.isna().sum()

df[df['profession'].isna()].shape[0]/df.shape[0]

df[~df['suite_type'].notna()].shape[0]/df.shape[0]

df.drop('profession', axis=1, inplace=True)

df[df['number_of_family_members'].isna()]

#I reckon there has to be a linear relationship between number_of_children and number_of_family_members. df['number_of_children'].corr(df['number_of_family_members'])

#As seen from above, our missing values of family members have no children. df[df['number_of_children']==0]['number_of_family_members'].value_counts(normalize=True)*100 #So, we'll fill in the missing family members with 2.

df['number_of_family_members'].fillna(value=2, inplace=True)

#Checking the validity of the dataset df[df['number_of_children']>df['number_of_family_members']]

#Let's fill in our last missing values column. def plot_pie(data, column): plt.figure(figsize=(9,6)) data[column].value_counts(normalize=True).plot(kind='pie') plt.show();

plot_pie(df, 'suite_type')

df['suite_type'].value_counts()

df['suite_type'].fillna(value='Unaccompanied', inplace=True)

df.duplicated().sum()

#Remove duplicate records duplicate_check_remove(df)

df.shape

df_cont = df.select_dtypes(include='number') df_cat = df.select_dtypes(include='category')

df_cat['have_a_phone'] = df_cont['have_a_phone'] df_cat['have_a_mail'] = df_cont['have_a_mail'] df_cat['number_of_children'] = df_cont['number_of_children'] df_cat['payment'] = df_cont['payment'] df_cat['number_of_family_members'] = df_cont['number_of_family_members'] df_cont['income_credit_ratio'] = df_cont['total_income'] / df_cont['amount_of_credit'] #engineered df_cont.drop(['number_of_children', 'have_a_phone', 'have_a_mail', 'number_of_family_members'], axis=1, inplace=True)

df_cont.sample(4)

#Comparing means pd.pivot_table(data=df_cont, index='payment', aggfunc=[np.mean ,np.median])

#Checking equal variances assumption. Mostly violated, so we continue with Welch T-test pd.pivot_table(data=df_cont, index='payment', aggfunc=[np.var, 'count'])

for col in df_cont.columns[1:]: print(f'''P-value for {col}: {ttest(df_cont[df_cont['payment']==0][col], df_cont[df_cont['payment']==1][col], correction='auto').T.iloc[3,:][0]} ''')

from sklearn.feature_selection import VarianceThreshold print(df_cont.shape) var_filter = VarianceThreshold(threshold = 0.0) train = var_filter.fit_transform(df_cont) print(train.shape) #All of them remained

find_outliers_iqr(df_cont)

def coerce_outliers(value): if value > upperlimit: value = upperlimit elif value < lowerlimit: value = lowerlimit return value for feature in df_cont.columns[1:]: Q3 = df_cont[feature].quantile(q = 0.75) Q1 = df_cont[feature].quantile(q = 0.25) IQR = Q3 - Q1 outlier_range = IQR * 1.5 upperlimit = Q3 + outlier_range lowerlimit = Q1 - outlier_range df_cont[feature] = df_cont[feature].apply(coerce_outliers)

df_cont.drop(['total_income', 'amount_of_credit'], axis=1, inplace=True)

#We've got very few categories, but should we use one-hot encoding we'll end up with a lot of columns for col in df_cat: print(f'''Number of unique values for {col}: {df_cat[col].nunique()}''')

a = list(df_cat.columns)

del a[-2]

for col in a: expected, observed, stats = chi2_independence(data=df, x=col, y='payment') print(stats)

df_cat.drop(['loan_type', 'gender', 'have_a_car', 'have_a_phone', 'suite_type', 'education_type', 'family_status', 'income_type', 'organization', 'number_of_children', 'number_of_family_members'], axis=1, inplace=True)

df_cat = pd.concat([df_cat[df_cat['payment']==0].replace(0, 1), df_cat[df_cat['payment']==1].replace(1, 0)], axis=0)

df_cat['have_a_house'] = df_cat['have_a_house'].replace('Y', 1).replace('N', 0)

df = pd.concat([df_cont.iloc[:,1:], df_cat], axis=1)

df.reset_index(drop=True, inplace=True)

df.tail()

df.to_csv('Risk--Analytics.csv')