!pip install xgboost==1.4.2

!pip install lightgbm==3.2.1

!pip install catboost==0.26.1

#linear algebra import numpy as np # data processing import pandas as pd # data visualization import seaborn as sns %matplotlib inline from matplotlib import pyplot as plt from matplotlib import style # Algorithms from sklearn import linear_model from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestClassifier from sklearn.linear_model import Perceptron from sklearn.linear_model import SGDClassifier from sklearn.tree import DecisionTreeClassifier from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC, LinearSVC from sklearn.naive_bayes import GaussianNB from xgboost import XGBClassifier from lightgbm import LGBMClassifier from catboost import Pool, CatBoostClassifier from sklearn.ensemble import GradientBoostingClassifier from sklearn.metrics import precision_score, recall_score, classification_report

test_df = pd.read_csv(r"/work/titanic/test.csv") train_df = pd.read_csv(r"/work/titanic/train.csv")

!pip install dataprep

from dataprep.eda import create_report report = create_report(train_df)

#Getting info about the training dataset train_df.info()

#Getting important statistics about the data train_df.describe()

train_df.head() # Show the first 5 columns of the dataset

train_df.corr()

total = train_df.isnull().sum().sort_values(ascending=False) percent_1 = train_df.isnull().sum()/train_df.isnull().count()*100 percent_2 = (round(percent_1, 1)).sort_values(ascending=False) missing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%']) missing_data.head(5)

train_df.columns.values

survived = 'survived' not_survived = 'not survived' fig, axes = plt.subplots(nrows=1, ncols=2,figsize=(10, 4)) women = train_df[train_df['Sex']=='female'] men = train_df[train_df['Sex']=='male'] ax = sns.distplot(women[women['Survived']==1].Age.dropna(), bins=18, label = survived, ax = axes[0], kde =False) ax = sns.distplot(women[women['Survived']==0].Age.dropna(), bins=40, label = not_survived, ax = axes[0], kde =False) ax.legend() ax.set_title('Female') ax = sns.distplot(men[men['Survived']==1].Age.dropna(), bins=18, label = survived, ax = axes[1], kde = False) ax = sns.distplot(men[men['Survived']==0].Age.dropna(), bins=40, label = not_survived, ax = axes[1], kde = False) ax.legend() _ = ax.set_title('Male')

FacetGrid = sns.FacetGrid(train_df, row='Embarked', size=4.5, aspect=1.6) FacetGrid.map(sns.pointplot, 'Pclass', 'Survived', 'Sex', palette=None, order=None, hue_order=None ) FacetGrid.add_legend()

sns.barplot(x="Pclass", y="Survived", data = train_df)

grid = sns.FacetGrid(train_df, col="Survived", row="Pclass",size=2.2, aspect=1.6) grid.map(plt.hist, 'Age', alpha=0.5, bins=20) grid.add_legend()

data = [train_df, test_df] for dataset in data: dataset['relatives'] = dataset['SibSp'] + dataset['Parch'] dataset.loc[dataset['relatives'] > 0, 'not_alone'] = 0 dataset.loc[dataset['relatives'] == 0, 'not_alone'] = 1 dataset['not_alone'] = dataset['not_alone'].astype(int) train_df['not_alone'].value_counts()

axes = sns.factorplot('relatives', 'Survived', data=train_df, aspect=20.5)

train_df = train_df.drop(['PassengerId'], axis=1)

import re deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8} data = [train_df, test_df] for dataset in data: dataset['Cabin'] = dataset['Cabin'].fillna("U0") dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)").search(x).group()) dataset['Deck'] = dataset['Deck'].map(deck) dataset['Deck'] = dataset['Deck'].fillna(0) dataset['Deck'] = dataset['Deck'].astype(int) # we can now drop the cabin feature train_df = train_df.drop(['Cabin'], axis=1) test_df = test_df.drop(['Cabin'], axis=1)

data = [train_df, test_df] for dataset in data: mean = train_df["Age"].mean() std = test_df["Age"].std() is_null = dataset["Age"].isnull().sum() # compute random numbers between the mean, std and is_null rand_age = np.random.randint(mean - std, mean + std, size = is_null) # fill NaN values in Age column with random values generated age_slice = dataset["Age"].copy() age_slice[np.isnan(age_slice)] = rand_age dataset["Age"] = age_slice dataset["Age"] = train_df["Age"].astype(int) train_df["Age"].isnull().sum()

train_df["Embarked"].describe()

common_value = 'S' data = [train_df, test_df] for dataset in data: dataset['Embarked'] = dataset['Embarked'].fillna(common_value)

train_df.info()

#converting fare to from float to int64 using astype() function data= [train_df, test_df] for dataset in data: dataset['Fare'] = dataset['Fare'].fillna(0) dataset['Fare'] = dataset['Fare'].astype(int)

# we will extract the titles from the name and build a new feature from out of that data = [train_df, test_df] titles = {'Mr': 1,'Mrs': 3,"Miss" : 2, "Master" : 4, "Rare" : 5} for dataset in data: #extract titles dataset['Title'] = dataset.Name.str.extract(' ([A-Za-z]+)\.', expand=False) # replace titles with more common title or as Rare dataset['Title'] = dataset['Title'].replace(["Lady", 'Countess', 'Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona', 'Rare']) dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss') dataset['Title'] = dataset['Title'].replace('Ms', 'Miss') dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs') # convert titles into numbers dataset['Title'] = dataset['Title'].map(titles) # filling NaN with 0, to get safe dataset['Title'] = dataset['Title'].fillna(0) train_df = train_df.drop(['Name'], axis=1) test_df = test_df.drop(['Name'], axis=1)

#converting sex into numeric genders = {"male" : 0, "female": 1} data = [train_df, test_df] for dataset in data: dataset['Sex'] = dataset['Sex'].map(genders)

train_df['Ticket'].describe()

# this feature will be dofficult to `convert to uswful categories so we will drop it train_df= train_df.drop(['Ticket'], axis=1) test_df= test_df.drop(['Ticket'], axis=1)

#convert embarked into numeric ports = {"S":0, "C":1, "Q" :2} data = [train_df, test_df] for dataset in data: dataset['Embarked'] = dataset["Embarked"].map(ports)

data = [train_df, test_df] for dataset in data: dataset['Age'] = dataset['Age'].astype(int) dataset.loc[ dataset['Age'] <= 11, 'Age'] = 0 dataset.loc[(dataset['Age'] > 11) & (dataset['Age'] <= 18), 'Age'] = 1 dataset.loc[(dataset['Age'] > 18) & (dataset['Age'] <= 22), 'Age'] = 2 dataset.loc[(dataset['Age'] > 22) & (dataset['Age'] <= 27), 'Age'] = 3 dataset.loc[(dataset['Age'] > 27) & (dataset['Age'] <= 33), 'Age'] = 4 dataset.loc[(dataset['Age'] > 33) & (dataset['Age'] <= 40), 'Age'] = 5 dataset.loc[(dataset['Age'] > 40) & (dataset['Age'] <= 66), 'Age'] = 6 dataset.loc[ dataset['Age'] > 66, 'Age'] = 6 # let's see how it's distributed train_df['Age'].value_counts()

data = [train_df, test_df] for dataset in data: dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0 dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1 dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare'] = 2 dataset.loc[(dataset['Fare'] > 31) & (dataset['Fare'] <= 99), 'Fare'] = 3 dataset.loc[(dataset['Fare'] > 99) & (dataset['Fare'] <= 250), 'Fare'] = 4 dataset.loc[ dataset['Fare'] > 250, 'Fare'] = 5 dataset['Fare'] = dataset['Fare'].astype(int)

#Age times class data = [train_df, test_df] for dataset in data: dataset['Age_class'] = dataset['Age'] * dataset['Pclass']

#Fare per person for dataset in data: dataset['Fare_Per_Person'] = dataset['Fare']/(dataset['relatives']+1) dataset['Fare_Per_Person'] = dataset['Fare_Per_Person'].astype(int) # Let's take a last look at the training set, before we start training the models. train_df.head(10)

X_train = train_df.drop("Survived", axis=1) Y_train = train_df["Survived"] X_test = test_df.drop("PassengerId", axis=1).copy()

train_df.corr()

sgd= linear_model.SGDClassifier(max_iter=5, tol=None) sgd.fit(X_train, Y_train) Y_pred = sgd.predict(X_test) sgd.score(X_train, Y_train) acc_sgd = round(sgd.score(X_train, Y_train) * 100, 2)

random_forest = RandomForestClassifier(n_estimators=100) random_forest.fit(X_train, Y_train) Y_prediction = random_forest.predict(X_test) random_forest.score(X_train, Y_train) acc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)

logreg = LogisticRegression() logreg.fit(X_train, Y_train) Y_pred = logreg.predict(X_test) acc_log = round(logreg.score(X_train, Y_train) * 100, 2)

#KNN knn = KNeighborsClassifier(n_neighbors = 3) knn.fit(X_train, Y_train) Y_pred = knn.predict(X_test) acc_knn = round(knn.score(X_train, Y_train) * 100, 2)

gaussian = GaussianNB() gaussian.fit(X_train, Y_train) Y_pred = gaussian.predict(X_test) acc_gaussian = round(gaussian.score(X_train, Y_train) * 100, 2)

perceptron = Perceptron(max_iter=5) perceptron.fit(X_train, Y_train) Y_pred = perceptron.predict(X_test) acc_perceptron = round(perceptron.score(X_train, Y_train) * 100, 2)

linear_svc = LinearSVC() linear_svc.fit(X_train, Y_train) Y_pred = linear_svc.predict(X_test) acc_linear_svc = round(linear_svc.score(X_train, Y_train) * 100, 2)

decision_tree = DecisionTreeClassifier() decision_tree.fit(X_train, Y_train) Y_pred = decision_tree.predict(X_test) acc_decision_tree = round(decision_tree.score(X_train, Y_train) * 100, 2)

results = pd.DataFrame({ 'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression', 'Random Forest', 'Naive Bayes', 'Perceptron', 'Stochastic Gradient Decent', 'Decision Tree'], 'Score': [acc_linear_svc, acc_knn, acc_log, acc_random_forest, acc_gaussian, acc_perceptron, acc_sgd, acc_decision_tree]}) result_df = results.sort_values(by='Score', ascending=False) result_df = result_df.set_index('Score') result_df.head(9)

from sklearn.model_selection import cross_val_score rf = RandomForestClassifier(n_estimators=100) scores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = "accuracy") print("Scores:", scores) print("Mean:", scores.mean()) print("Standard Deviation:", scores.std())

train_df = train_df.drop("not_alone", axis=1) test_df = test_df.drop("not_alone", axis=1) train_df = train_df.drop("Parch", axis=1) test_df = test_df.drop("Parch", axis=1)

random_forest = RandomForestClassifier(n_estimators=100, oob_score = True) random_forest.fit(X_train, Y_train) Y_prediction = random_forest.predict(X_test) random_forest.score(X_train, Y_train) acc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2) print(round(acc_random_forest,2,), "%")

xgb = XGBClassifier() xgb.fit(X_train, Y_train) y_pred = xgb.predict(X_test) acc_xgb = round(xgb.score(X_train, Y_train) * 100, 2) acc_xgb

lgbm = LGBMClassifier() lgbm.fit(X_train, Y_train) y_pred = lgbm.predict(X_test) acc_lgbm = round(lgbm.score(X_train, Y_train) * 100, 2) acc_lgbm

catboost = CatBoostClassifier() catboost.fit(X_train, Y_train) y_pred = catboost.predict(X_test) acc_catboost = round(catboost.score(X_train, Y_train) * 100, 2) acc_catboost

from sklearn.metrics import precision_recall_curve # getting the probabilities of our predictions y_scores = random_forest.predict_proba(X_train) y_scores = y_scores[:,1] precision, recall, threshold = precision_recall_curve(Y_train, y_scores) def plot_precision_and_recall(precision, recall, threshold): plt.plot(threshold, precision[:-1], "r-", label="precision", linewidth=3) plt.plot(threshold, recall[:-1], "b", label="recall", linewidth=2.5) plt.xlabel("threshold", fontsize=19) plt.legend(loc="upper right", fontsize=19) plt.ylim([0, 1]) plt.figure(figsize=(14, 7)) plot_precision_and_recall(precision, recall, threshold) plt.show()

# another way to plot precision and recall together def plot_precision_vs_recall(precision, recall): plt.plot(recall, precision, "g--", linewidth=2.5) plt.ylabel("recall", fontsize=19) plt.xlabel("precision", fontsize=19) plt.axis([0, 1.5, 0, 1.5]) plt.figure(figsize=(14, 7)) plot_precision_vs_recall(precision, recall) plt.show()

from sklearn.metrics import roc_curve # compute true positive rate and false positive rate false_positive_rate, true_positive_rate, thresholds = roc_curve(Y_train, y_scores) # plotting them against each other def plot_roc_curve(false_positive_rate, true_positive_rate, label=None): plt.plot(false_positive_rate, true_positive_rate, linewidth=2, label=label) plt.plot([0, 1], [0, 1], 'r', linewidth=4) plt.axis([0, 1, 0, 1]) plt.xlabel('False Positive Rate (FPR)', fontsize=16) plt.ylabel('True Positive Rate (TPR)', fontsize=16) plt.figure(figsize=(14, 7)) plot_roc_curve(false_positive_rate, true_positive_rate) plt.show()

from sklearn.metrics import roc_auc_score r_a_score = roc_auc_score(Y_train, y_scores) print("ROC-AUC-Score:", r_a_score)

submission = pd.DataFrame({'PassengerId':test_df['PassengerId'],'Survived':Y_prediction}) #Visualize the first 5 rows submission.head(15)