# Run this cell to start.
import numpy as np
import pandas as pd
# Safe settings for Pandas.
pd.set_option('mode.chained_assignment', 'raise')
%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
# Read the dataset as a data frame.
titanic = pd.read_csv("titanic_stlearn.csv")
# Boolean with True for passengers with not-NA sibsp values, False otherwise.
have_sibsp = titanic['sibsp'].notna()
# Select rows with value (not-NA) sibsp values.
with_sibsp = titanic[have_sibsp]
with_sibsp.head()
with_sibsp.describe()
with_sibsp.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 1307 entries, 0 to 1318
Data columns (total 11 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 name 1307 non-null object
1 gender 1307 non-null object
2 age 1305 non-null float64
3 class 1307 non-null object
4 embarked 1307 non-null object
5 country 1231 non-null object
6 ticketno 1306 non-null float64
7 fare 1291 non-null float64
8 sibsp 1307 non-null float64
9 parch 1307 non-null float64
10 survived 1307 non-null object
dtypes: float64(5), object(6)
memory usage: 122.5+ KB
def find_missing_data(data):
Total=data.isnull().sum().sort_values(ascending= False)
Percentage= (data.isnull().sum()/data.isnull().count()).sort_values(ascending = False)
return pd.concat([Total, Percentage], axis=1, keys= ['Total', 'Percent'])
find_missing_data(with_sibsp)
with_sibsp={}
#Preperation of this data frame
#Change strings to integers for further analysis
with_sibsp['gender'].replace({'male':0, 'female':2}, inplace=True)
with_sibsp.loc[:, 'survived'].replace('yes', 1 ,inplace=True)
with_sibsp.loc[:, 'survived'].replace('no', 0 ,inplace=True)
with_sibsp.head()
#age and gender
import seaborn as sns
survived= 'survived'
not_survived= 'not survived'
fig, axes= plt.subplots(nrows=1, ncols=2, figsize=(10, 4))
female=with_sibsp[with_sibsp['gender']==2]
male= with_sibsp[with_sibsp['gender']==0]
ax= sns.histplot(female[female['survived']==1].age.dropna(), bins=18, label=survived, ax=axes[0], kde=False)
ax= sns.histplot(female[female['survived']==0].age.dropna(), bins=40, label=not_survived, ax=axes[0], kde=False)
ax.legend()
ax.set_title('female')
ax=sns.histplot(male[male['survived']==1].age.dropna(), bins=18, label=survived, ax=axes[1], kde=False)
ax=sns.histplot(male[male['survived']==0].age.dropna(), bins=40, label=not_survived, ax=axes[1], kde=False)
ax.legend()
ax.set_title('male')
input_1
This graph conveys that men between the ages of eighteen and thirty have a high probability of surviving, whereas female survival chances are higher between 14 and 40.
Males between the ages of 5 and 18 have the lowest survival chance.
But, how is the gender and age distribution in each class?
#Age and Gender by class
class_rd= '3rd'
class_nd= '2nd'
class_st= '1st'
fig, axes=plt.subplots(nrows=1, ncols=2, figsize=(10,4))
female= with_sibsp[with_sibsp['gender']==2]
male= with_sibsp[with_sibsp['gender']==0]
ax= sns.histplot(female[female['class']=="3rd"].age.dropna(), bins=18, label= "3rd", ax= axes[0], kde=False)
ax= sns.histplot(female[female['class']=="2nd"].age.dropna(), bins=18, label= "2nd",ax= axes[0], kde=False)
ax= sns.histplot(female[female['class']=="1st"].age.dropna(), bins=18, label= "1st",ax= axes[0], kde=False)
ax.legend()
ax.set_title('female')
ax=sns.histplot(male[male['class']=="3rd"].age.dropna(), bins=18, label= "3rd", ax=axes[1], kde=False)
ax=sns.histplot(male[male['class']=="2nd"].age.dropna(), bins=18, label= "2nd", ax=axes[1], kde=False)
ax=sns.histplot(male[male['class']=="1st"].age.dropna(), bins=18, label= "1st", ax=axes[1], kde=False)
ax.legend()
ax.set_title('male')
This distribution by class shows us that the passengers in 3rd class were younger than those in class 1 and 2.
It can also be seen that more children are found in the 3rd class when compared to 1st and 2nd class. But the is not enough evidence to test the hypotheses.
#0 is for male and 2 is for female. This graph conveys the distribution of those that survived, by class and gender. This shows that ultimately, the chances to survive are higher in the first and second class for females and men had the lowest chance to survive in the second and third class.
sns.barplot(x='class', y='survived', hue='gender', data=with_sibsp)
In order to determine whether people travelled alone or accompanied, I created a bar plot showing that most males and females ravelled alone, with no spouses (or relatives).
sns.countplot(x='sibsp', hue='gender', data=with_sibsp)
sns.countplot(x='parch', hue='gender', data=with_sibsp)
#Family size descriptor from sibsp and parch
with_sibsp["Famsize"]= with_sibsp["sibsp"] + with_sibsp["parch"]+1
#Visualisation of probability to survive of Famsize in 3rd class.
facet=sns.FacetGrid(with_sibsp[with_sibsp['class']=="3rd"], hue="survived", aspect=4)
facet.map(sns.kdeplot, 'Famsize', shade=True)
facet.set(xlim=(0, with_sibsp['Famsize'].max()))
facet.add_legend()
plt.xlim(0);
The probability survive is higher than the probability to die when you're alone, but families with 4 are more likely to survive than a couple for instance.
#combining column for "sibsp" and "parch" to creare a "relatives column for further calculation"
sum=with_sibsp.loc[:, 'sibsp']+ with_sibsp.loc[:, 'parch']
with_sibsp.loc[:, 'relatives']= with_sibsp.loc[:, 'sibsp']+ with_sibsp.loc[:, 'parch']
with_sibsp.head()
sns.factorplot('relatives', 'survived',
data=with_sibsp, hue='class', aspect= 2.5,)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:3717: UserWarning: The `factorplot` function has been renamed to `catplot`. The original name will be removed in a future release. Please update your code. Note that the default `kind` in `factorplot` (`'point'`) has changed `'strip'` in `catplot`.
warnings.warn(msg)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
FutureWarning
After creating the relatives column, it is evident that the chances to survive increase with 1-3 relatives.
#Lets look at al passengers and see if we can gather additional information relative to family size and survival probability
axes=sns.factorplot('relatives', 'survived',
data=with_sibsp, hue='gender', aspect=2.5,)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:3717: UserWarning: The `factorplot` function has been renamed to `catplot`. The original name will be removed in a future release. Please update your code. Note that the default `kind` in `factorplot` (`'point'`) has changed `'strip'` in `catplot`.
warnings.warn(msg)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
FutureWarning
Here it is evident that survival probability for males and females increases the more relatives they have when between 1-3, whilst chances to survive decrease when family size exceeds this threshold. However, there is an exception for families with 5-6 members.
Lets do this for each class
#1st class
axes= sns.factorplot('relatives', 'survived',
data=with_sibsp[with_sibsp['class']=="1st"], hue='gender', aspect=2.5,)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:3717: UserWarning: The `factorplot` function has been renamed to `catplot`. The original name will be removed in a future release. Please update your code. Note that the default `kind` in `factorplot` (`'point'`) has changed `'strip'` in `catplot`.
warnings.warn(msg)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
FutureWarning
#2nd class
axes=sns.factorplot('relatives', 'survived',
data=with_sibsp[with_sibsp['class']=="2nd"], hue='gender', aspect=2.5,)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:3717: UserWarning: The `factorplot` function has been renamed to `catplot`. The original name will be removed in a future release. Please update your code. Note that the default `kind` in `factorplot` (`'point'`) has changed `'strip'` in `catplot`.
warnings.warn(msg)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
FutureWarning
#3rd class
axes=sns.factorplot('relatives', 'survived',
data=with_sibsp[with_sibsp['class']=="3rd"], hue='gender', aspect=2.5,)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
FutureWarning
The above graphs demonstrate that most of the variance and lowest survival p are in the 3rd class (most passengers, youngest population), therefore survival chances were lowest here.
It is also evident that chances of survival for single females in 3rd class is similar to having 2 relatives, whereas men are more likely to die regardless of relatives in 3rd class.
Females with 4-10 relatives had an almost 0% chance at survival, apart from females with 6 relatives.
We can see evidence that more females in 3rd class died because they didn't want to leave behind their families, but we must compute logistic regression to be sure of this. The results are significant if the p value is <0.05
import pandas as pd
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
class_3=with_sibsp.loc[with_sibsp.loc[:,'class']=="3rd"]
class_3['gender']=pd.factorize(class_3['gender'])[0]
class_3['survived']=pd.factorize(class_3['survived'])[0]
x=class_3[["gender", "sibsp", "parch"]]
y= class_3["survived"]
lm= LogisticRegression()
lm.fit (x,y)
lm.coef_
import statsmodels.api as sm
logit_model=sm.Logit(y,x)
result=logit_model.fit()
print(result.summary2())
Optimization terminated successfully.
Current function value: 0.641859
Iterations 6
Results: Logit
===============================================================
Model: Logit Pseudo R-squared: -0.128
Dependent Variable: survived AIC: 913.5892
Date: 2022-01-17 10:46 BIC: 927.2723
No. Observations: 707 Log-Likelihood: -453.79
Df Model: 2 LL-Null: -402.17
Df Residuals: 704 LLR p-value: 1.0000
Converged: 1.0000 Scale: 1.0000
No. Iterations: 6.0000
----------------------------------------------------------------
Coef. Std.Err. z P>|z| [0.025 0.975]
----------------------------------------------------------------
gender 0.5090 0.1620 3.1408 0.0017 0.1914 0.8266
sibsp -0.5999 0.1169 -5.1329 0.0000 -0.8289 -0.3708
parch -0.2312 0.1065 -2.1716 0.0299 -0.4400 -0.0225
===============================================================
coefficients of logistic regression show a positive correlation between survival by being female and negative by having siblings or spouses.
the results passed the significance test and convey that chances to survive ultimately decrease when a passenger traveled with "sibsp" or parch.
#LOG model for males
import pandas as pd
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt
only_male= class_3[class_3['gender']==0]
x=[["sibsp", "parch"]]
y=only_male["survived"]
lm=LogisticRegression()
lm.fit(x,y)
lm.coef
logit_model=sm.Logit(Y,X)
result=logit_model.fit()
print(result.summary2())
Execution error
ValueError: could not convert string to float: 'sibsp'
the coefficient of LR for male passengers show a negative correlation by having siblings/spouses. Only "sibsp" passed the significance test, showing that probability so survive decrease when a male passenger travels with a sibling or spouse.
#Females
import pandas as pd
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt
only_female= class_3[class_3['gender']==2]
x= only_female[["sibsp", "parch"]]
y= only_female["survived"]
lm= LogisticRegression()
lm.fit (x,y)
lm.coef_
import statsmodels.api as sm
logit_model=sm.Logit(y,x)
result=logit_model.fit()
print(result.summary2())
Execution error
NameError: name 'stock_value' is not defined
Coefficients of Logistical regression for female passengers ultimately shows a negative correlation by having sibling or spouses. Only the significance test for "sibsp" was passed, showing us that chances to survive decreases when a female passenger is travelling with "sibsp"
#To see if the probability to survive increases or decreases by having sibsp or parch in the 3rd class, I also wanted to have a look relative to class 1.
class_1=with_sibsp[with_sibsp.loc[:,'class']=="1st"]
class_1['gender']=pd.factorize(class_1['gender'][0])
class_1['survived']=pd.factorize(class_1['survived'][0])
x=class_1[["gender", "sibsp", "parch"]]
y= class_1["survived"]
lm- LogisticRegression()
lm.fit(x,y)
lm.coef_
logit=sm.Logit(y,x)
result=logit_model.fit()
print(result.summary2())
Execution error
KeyError: 0
Coefficient of logistical regression conveys a positive correlation between survival by being female and negative by having spouse/siblings accompanying them.
the results passed the significance test, showing us that chances of survival ultimately decrease when a passenger travels with a "sibsp" or "parch"