#0 EDA - My machine learning pipeline

Imports

import os import numpy as np import pandas as pd from easydict import EasyDict as edict from pandas_profiling import ProfileReport import matplotlib.pyplot as plt import seaborn as sns from tqdm import tqdm

Config

config = edict() config.main = edict() config.main.TRAIN_FILE = "/work/train.csv" config.main.TEST_FILE = "/work/test.csv" config.main.TARGET_VAR = "target"

Loading data

df_train = pd.read_csv(config.main.TRAIN_FILE) df_test = pd.read_csv(config.main.TEST_FILE)

EDA

df_train.head()

df_test.head()

df_train.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 300000 entries, 0 to 299999
Data columns (total 32 columns):
 #   Column  Non-Null Count   Dtype  
---  ------  --------------   -----  
 0   id      300000 non-null  int64  
 1   cat0    300000 non-null  object 
 2   cat1    300000 non-null  object 
 3   cat2    300000 non-null  object 
 4   cat3    300000 non-null  object 
 5   cat4    300000 non-null  object 
 6   cat5    300000 non-null  object 
 7   cat6    300000 non-null  object 
 8   cat7    300000 non-null  object 
 9   cat8    300000 non-null  object 
 10  cat9    300000 non-null  object 
 11  cat10   300000 non-null  object 
 12  cat11   300000 non-null  object 
 13  cat12   300000 non-null  object 
 14  cat13   300000 non-null  object 
 15  cat14   300000 non-null  object 
 16  cat15   300000 non-null  object 
 17  cat16   300000 non-null  object 
 18  cat17   300000 non-null  object 
 19  cat18   300000 non-null  object 
 20  cont0   300000 non-null  float64
 21  cont1   300000 non-null  float64
 22  cont2   300000 non-null  float64
 23  cont3   300000 non-null  float64
 24  cont4   300000 non-null  float64
 25  cont5   300000 non-null  float64
 26  cont6   300000 non-null  float64
 27  cont7   300000 non-null  float64
 28  cont8   300000 non-null  float64
 29  cont9   300000 non-null  float64
 30  cont10  300000 non-null  float64
 31  target  300000 non-null  int64  
dtypes: float64(11), int64(2), object(19)
memory usage: 73.2+ MB

df_test.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 200000 entries, 0 to 199999
Data columns (total 31 columns):
 #   Column  Non-Null Count   Dtype  
---  ------  --------------   -----  
 0   id      200000 non-null  int64  
 1   cat0    200000 non-null  object 
 2   cat1    200000 non-null  object 
 3   cat2    200000 non-null  object 
 4   cat3    200000 non-null  object 
 5   cat4    200000 non-null  object 
 6   cat5    200000 non-null  object 
 7   cat6    200000 non-null  object 
 8   cat7    200000 non-null  object 
 9   cat8    200000 non-null  object 
 10  cat9    200000 non-null  object 
 11  cat10   200000 non-null  object 
 12  cat11   200000 non-null  object 
 13  cat12   200000 non-null  object 
 14  cat13   200000 non-null  object 
 15  cat14   200000 non-null  object 
 16  cat15   200000 non-null  object 
 17  cat16   200000 non-null  object 
 18  cat17   200000 non-null  object 
 19  cat18   200000 non-null  object 
 20  cont0   200000 non-null  float64
 21  cont1   200000 non-null  float64
 22  cont2   200000 non-null  float64
 23  cont3   200000 non-null  float64
 24  cont4   200000 non-null  float64
 25  cont5   200000 non-null  float64
 26  cont6   200000 non-null  float64
 27  cont7   200000 non-null  float64
 28  cont8   200000 non-null  float64
 29  cont9   200000 non-null  float64
 30  cont10  200000 non-null  float64
dtypes: float64(11), int64(1), object(19)
memory usage: 47.3+ MB

df_train.describe()

df_test.describe()

TARGET VARIABLE

plt.figure(figsize = (10,5)) sns.barplot(data = df_train, x = 'target') plt.show()

df_train.target.value_counts()

MISSING VALUES

def missing_zero_values_table(df): #Function to access missing and zero values in your dataset zero_val = (df == 0.00).astype(int).sum(axis=0) #Number of zeros in the dataset mis_val = df.isnull().sum() #Number of missing values in the dataset mis_val_percent = 100 * df.isnull().sum() / len(df) #percent missing values mz_table = pd.concat([zero_val, mis_val, mis_val_percent], axis=1) mz_table = mz_table.rename( columns = {0 : 'Zero Values', 1 : 'Missing Values', 2 : '% of Total Values'}) #Create missing value table mz_table['Total Zero Missing Values'] = mz_table['Zero Values'] + mz_table['Missing Values'] mz_table['% Total Zero Missing Values'] = 100 * mz_table['Total Zero Missing Values'] / len(df) mz_table['Data Type'] = df.dtypes mz_table = mz_table[ mz_table.iloc[:,1] != 0].sort_values( '% of Total Values', ascending=False).round(1) print ("Your selected dataframe has " + str(df.shape[1]) + " columns and " + str(df.shape[0]) + " Rows.\n" "There are " + str(mz_table.shape[0]) + " columns that have missing values.") return mz_table

missing_zero_values_table(df_train)

Your selected dataframe has 32 columns and 300000 Rows.
There are 0 columns that have missing values.

missing_zero_values_table(df_test)

Your selected dataframe has 31 columns and 200000 Rows.
There are 0 columns that have missing values.

PANDAS PROFILING

profile_train = ProfileReport(df_train, title='Pandas Train Profiling Report', html={'style':{'full_width':True}})

profile_train

IPyWidgets are not supported

profile_test = ProfileReport(df_test, title='Pandas Train Profiling Report', html={'style':{'full_width':True}})

profile_test

IPyWidgets are not supported