import pandas as pd
import numpy as np
train = pd.read_csv('Letters_train.csv')
train.head()
test = pd.read_csv('Letters_test.csv')
1 a) i)
#make a function that tells if the letter is B or not
def is_b(text):
if text == 'B':
return 1
else:
return 0
#apply is_b function to the training and test dataset
train['isB'] = train['letter'].apply(is_b)
test['isB'] = test['letter'].apply(is_b)
test.head()
#see how many No's and Yes's in the dataset
train.groupby('isB').count()
1 a) ii)
!pip install statsmodels==0.13.0
Collecting statsmodels==0.13.0
Downloading statsmodels-0.13.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.8 MB)
|████████████████████████████████| 9.8 MB 13.6 MB/s
Collecting patsy>=0.5.2
Downloading patsy-0.5.2-py2.py3-none-any.whl (233 kB)
|████████████████████████████████| 233 kB 44.7 MB/s
Requirement already satisfied: pandas>=0.25 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from statsmodels==0.13.0) (1.2.5)
Requirement already satisfied: scipy>=1.3 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from statsmodels==0.13.0) (1.7.1)
Requirement already satisfied: numpy>=1.17 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from statsmodels==0.13.0) (1.19.5)
Requirement already satisfied: pytz>=2017.3 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from pandas>=0.25->statsmodels==0.13.0) (2021.3)
Requirement already satisfied: python-dateutil>=2.7.3 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from pandas>=0.25->statsmodels==0.13.0) (2.8.2)
Requirement already satisfied: six in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from patsy>=0.5.2->statsmodels==0.13.0) (1.16.0)
Installing collected packages: patsy, statsmodels
Successfully installed patsy-0.5.2 statsmodels-0.13.0
WARNING: You are using pip version 21.2.4; however, version 21.3.1 is available.
You should consider upgrading via the '/root/venv/bin/python -m pip install --upgrade pip' command.
import os
import statsmodels.formula.api as smf
#build a logistic regression model
logreg = smf.logit(formula = 'isB ~ xbox + ybox + width + height + onpix + xbar + ybar + x2bar + y2bar + xybar + x2ybar + xy2bar + xedge + xedgeycor + yedge + yedgexcor',
data = train).fit()
print(logreg.summary())
Optimization terminated successfully.
Current function value: 0.155134
Iterations 10
Logit Regression Results
==============================================================================
Dep. Variable: isB No. Observations: 2181
Model: Logit Df Residuals: 2164
Method: MLE Df Model: 16
Date: Mon, 08 Nov 2021 Pseudo R-squ.: 0.7200
Time: 23:39:28 Log-Likelihood: -338.35
converged: True LL-Null: -1208.3
Covariance Type: nonrobust LLR p-value: 0.000
==============================================================================
coef std err z P>|z| [0.025 0.975]
------------------------------------------------------------------------------
Intercept -13.5595 2.303 -5.888 0.000 -18.073 -9.046
xbox 0.0835 0.119 0.702 0.482 -0.149 0.316
ybox 0.0469 0.082 0.573 0.567 -0.113 0.207
width -1.3216 0.153 -8.621 0.000 -1.622 -1.021
height -0.6980 0.128 -5.433 0.000 -0.950 -0.446
onpix 0.9792 0.130 7.515 0.000 0.724 1.235
xbar 0.6668 0.119 5.601 0.000 0.433 0.900
ybar -0.5639 0.109 -5.158 0.000 -0.778 -0.350
x2bar -0.4077 0.092 -4.425 0.000 -0.588 -0.227
y2bar 1.3047 0.120 10.910 0.000 1.070 1.539
xybar 0.1775 0.087 2.049 0.040 0.008 0.347
x2ybar 0.4156 0.113 3.684 0.000 0.195 0.637
xy2bar -0.5295 0.108 -4.924 0.000 -0.740 -0.319
xedge -0.2515 0.089 -2.815 0.005 -0.427 -0.076
xedgeycor 0.1020 0.100 1.020 0.308 -0.094 0.298
yedge 1.7501 0.129 13.529 0.000 1.497 2.004
yedgexcor 0.3695 0.070 5.242 0.000 0.231 0.508
==============================================================================
Possibly complete quasi-separation: A fraction 0.22 of observations can be
perfectly predicted. This might indicate that there is complete
quasi-separation. In this case some parameters will not be identified.
y_prob = logreg.predict(test)
y_pred = pd.Series([1 if X > 1/2 else 0 for X in y_prob], index=y_prob.index)
from sklearn.metrics import confusion_matrix
y_test = test['isB']
tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()
(tn, fp, fn, tp)
#calculate accuracy of the model using test set
def accuracy(tn, fp, fn, tp):
return (tn + tp)/(tn+fp+fn+tp)
accuracy(tn, fp, fn, tp)
1 a) iii)
#separate x and y of train and test set
y_train = train['isB']
x_train = train.drop(['isB', 'letter'], axis=1)
y_test = test['isB']
x_test = test.drop(['isB','letter'], axis=1)
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc
fpr, tpr, _ = roc_curve(y_test, y_prob)
roc_auc = auc(fpr, tpr)
plt.figure(figsize=(8, 6))
plt.title('ROC Curve', fontsize=18)
plt.xlabel('FPR', fontsize=16)
plt.ylabel('TPR', fontsize=16)
plt.xlim([-0.01, 1.00])
plt.ylim([-0.01, 1.01])
plt.plot(fpr, tpr, lw=3, label='Logistic Regression (area = {:0.2f})'.format(roc_auc))
plt.plot([0,1],[0,1], color='navy', lw=3, linestyle='--', label='Naive Baseline (area = 0.50)')
plt.legend(loc='lower right', fontsize=14)
plt.show()
1 a) iv)
#cross-validation to get the optimal ccp alpha
from sklearn.metrics import make_scorer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import GridSearchCV
grid_values = {'ccp_alpha': np.linspace(0, 0.10, 201),
'min_samples_leaf': [5],
'min_samples_split': [20],
'max_depth': [30],
'random_state': [88]}
dtc = DecisionTreeClassifier()
dtc_cv_acc = GridSearchCV(dtc, param_grid = grid_values, cv=10, verbose=1,
scoring = 'accuracy')
dtc_cv_acc.fit(x_train, y_train)
Fitting 10 folds for each of 201 candidates, totalling 2010 fits
#derive the top 10 ccp_alpha values
acc = dtc_cv_acc.cv_results_['mean_test_score'] # what sklearn calls mean_test_score is the holdout set, i.e. the validation set.
ccp = dtc_cv_acc.cv_results_['param_ccp_alpha'].data
pd.DataFrame({'ccp alpha' : ccp, 'Validation Accuracy': acc}).head(10)
#run the model on test set
dtc_test = DecisionTreeClassifier(min_samples_leaf=5,
ccp_alpha=0.001,
#class_weight = {0: 1, 1: 20},
random_state = 88)
dtc_test = dtc_test.fit(x_train, y_train)
y_pred_dtc_test = dtc_test.predict(x_test)
cm_clatree = confusion_matrix(y_test, y_pred_dtc_test)
accuracy(cm_clatree.item((0,0)), cm_clatree.item((0,1)), cm_clatree.item((1, 0)), cm_clatree.item((1, 1)))
1 a) v)
#build random forest model with the default setting
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier()
rf_model = rf.fit(x_train, y_train)
y_pred_rf_test = rf_model.predict(x_test)
cm_rf = confusion_matrix(y_test, y_pred_rf_test)
accuracy(cm_rf.item((0,0)), cm_rf.item((0,1)), cm_rf.item((1, 0)), cm_rf.item((1, 1)))
1 b) i)
#baseline model -- see what letter is the most frequent letter
train.groupby('letter').count()
#accuracy of the baseline model with multi-class
578/(562+529+578+512)
1 b) ii)
x_train.head()
y_train_letter = train['letter']
y_test_letter = test['letter']
#no need to one-hot encode as all of the features are int64
#build LDA model
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.metrics import accuracy_score
lda = LinearDiscriminantAnalysis()
lda.fit(x_train, y_train_letter)
y_pred_lda = lda.predict(x_test)
len(y_pred_lda)
#calculate accuracy of the LDA model
#join the prediction and y_test to see the result
lda_result = {'y_pred_lda':y_pred_lda, 'y_test':y_test_letter}
lda_result = pd.DataFrame(data=lda_result)
lda_result['correct'] = lda_result['y_pred_lda'] == lda_result['y_test']
lda_result.groupby('correct').count()
#accuracy of the LDA model
842/(93+842)
1 b) iii)
#cross-validation to get the optimal ccp alpha
from sklearn.metrics import make_scorer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import GridSearchCV
grid_values = {'ccp_alpha': np.linspace(0, 0.10, 201),
'min_samples_leaf': [5],
'min_samples_split': [20],
'max_depth': [30],
'random_state': [88]}
dtc_b = DecisionTreeClassifier()
dtc_cv_acc_b = GridSearchCV(dtc_b, param_grid = grid_values, cv=10, verbose=1,
scoring = 'accuracy')
dtc_cv_acc_b.fit(x_train, y_train_letter)
Fitting 10 folds for each of 201 candidates, totalling 2010 fits
#derive the top 10 ccp_alpha values
acc_b = dtc_cv_acc_b.cv_results_['mean_test_score'] # what sklearn calls mean_test_score is the holdout set, i.e. the validation set.
ccp_b = dtc_cv_acc_b.cv_results_['param_ccp_alpha'].data
pd.DataFrame({'ccp alpha' : ccp_b, 'Validation Accuracy': acc_b}).head(10)
#run the model on test set
dtc_test_b = DecisionTreeClassifier(min_samples_leaf=5,
ccp_alpha=0.0005,
#class_weight = {0: 1, 1: 20},
random_state = 88)
dtc_test_b = dtc_test_b.fit(x_train, y_train_letter)
y_pred_dtc_test_b = dtc_test_b.predict(x_test)
cm_clatree_b = confusion_matrix(y_test_letter, y_pred_dtc_test_b)
accuracy(cm_clatree_b.item((0,0)), cm_clatree_b.item((0,1)), cm_clatree_b.item((1, 0)), cm_clatree_b.item((1, 1)))
1 b) iv)
x_train.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2181 entries, 0 to 2180
Data columns (total 16 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 xbox 2181 non-null int64
1 ybox 2181 non-null int64
2 width 2181 non-null int64
3 height 2181 non-null int64
4 onpix 2181 non-null int64
5 xbar 2181 non-null int64
6 ybar 2181 non-null int64
7 x2bar 2181 non-null int64
8 y2bar 2181 non-null int64
9 xybar 2181 non-null int64
10 x2ybar 2181 non-null int64
11 xy2bar 2181 non-null int64
12 xedge 2181 non-null int64
13 xedgeycor 2181 non-null int64
14 yedge 2181 non-null int64
15 yedgexcor 2181 non-null int64
dtypes: int64(16)
memory usage: 272.8 KB
#fit random forest model with max_features = 16
rf_b = RandomForestClassifier(max_features = 16)
rf_model_b = rf_b.fit(x_train, y_train_letter)
#accuracy of the "vanilla" bagging model with Random Forest
y_pred_rf_test_b = rf_model_b.predict(x_test)
cm_rf_b = confusion_matrix(y_test_letter, y_pred_rf_test_b)
accuracy(cm_rf_b.item((0,0)), cm_rf_b.item((0,1)), cm_rf_b.item((1, 0)), cm_rf_b.item((1, 1)))
1 b) v)
#Random Forest with CV
import time
from sklearn.model_selection import KFold
grid_values = {'max_features': np.linspace(1,16,16, dtype='int32'),
'min_samples_leaf': [5],
'n_estimators': [500],
'random_state': [88]}
tic = time.time()
rf_b2 = RandomForestClassifier()
# Note: here we set verbose=2 to keep track of the progress (the running time) of the cross validation.
cv = KFold(n_splits=5,random_state=333,shuffle=True)
rf_cv_b2 = GridSearchCV(rf_b2, param_grid=grid_values, scoring='accuracy', cv=cv,verbose=2)
rf_cv_b2.fit(x_train, y_train_letter)
toc = time.time()
print('time:', round(toc-tic, 2),'s')
Fitting 5 folds for each of 16 candidates, totalling 80 fits
[CV] END max_features=1, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.4s
[CV] END max_features=1, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.5s
[CV] END max_features=1, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.4s
[CV] END max_features=1, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.3s
[CV] END max_features=1, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.4s
[CV] END max_features=2, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.4s
[CV] END max_features=2, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.5s
[CV] END max_features=2, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.5s
[CV] END max_features=2, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=2, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=3, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=3, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=3, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=3, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=3, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.7s
[CV] END max_features=4, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.8s
[CV] END max_features=4, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.7s
[CV] END max_features=4, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.6s
[CV] END max_features=4, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.7s
[CV] END max_features=4, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.7s
[CV] END max_features=5, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.9s
[CV] END max_features=5, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.8s
[CV] END max_features=5, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.8s
[CV] END max_features=5, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.8s
[CV] END max_features=5, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.8s
[CV] END max_features=6, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.0s
[CV] END max_features=6, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.0s
[CV] END max_features=6, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.9s
[CV] END max_features=6, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.9s
[CV] END max_features=6, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 1.9s
[CV] END max_features=7, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.1s
[CV] END max_features=7, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.1s
[CV] END max_features=7, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.0s
[CV] END max_features=7, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.0s
[CV] END max_features=7, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.2s
[CV] END max_features=8, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.2s
[CV] END max_features=8, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.1s
[CV] END max_features=8, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.1s
[CV] END max_features=8, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.1s
[CV] END max_features=8, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.1s
[CV] END max_features=9, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.3s
[CV] END max_features=9, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.2s
[CV] END max_features=9, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.2s
[CV] END max_features=9, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.2s
[CV] END max_features=9, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.3s
[CV] END max_features=10, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.4s
[CV] END max_features=10, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.3s
[CV] END max_features=10, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.3s
[CV] END max_features=10, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.5s
[CV] END max_features=10, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.4s
[CV] END max_features=11, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.5s
[CV] END max_features=11, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.5s
[CV] END max_features=11, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.6s
[CV] END max_features=11, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.5s
[CV] END max_features=11, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.5s
[CV] END max_features=12, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.6s
[CV] END max_features=12, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.8s
[CV] END max_features=12, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.6s
[CV] END max_features=12, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=12, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.6s
[CV] END max_features=13, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.8s
[CV] END max_features=13, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=13, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=13, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=13, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=14, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=14, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=14, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.8s
[CV] END max_features=14, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.9s
[CV] END max_features=14, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.7s
[CV] END max_features=15, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.9s
[CV] END max_features=15, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.9s
[CV] END max_features=15, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.8s
[CV] END max_features=15, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.9s
[CV] END max_features=15, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 2.9s
[CV] END max_features=16, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 3.0s
[CV] END max_features=16, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 3.0s
[CV] END max_features=16, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 3.1s
[CV] END max_features=16, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 3.2s
[CV] END max_features=16, min_samples_leaf=5, n_estimators=500, random_state=88; total time= 3.0s
time: 178.98 s
#select the best hyperparameter
max_features = rf_cv_b2.cv_results_['param_max_features'].data
accuracy_scores = rf_cv_b2.cv_results_['mean_test_score']
plt.figure(figsize=(8, 6))
plt.xlabel('max features', fontsize=16)
plt.ylabel('CV Accuracy', fontsize=16)
plt.scatter(max_features, accuracy_scores, s=30)
plt.plot(max_features, accuracy_scores, linewidth=3)
plt.grid(True, which='both')
plt.xlim([1, 17])
plt.ylim([0.9, 1.0])
print(rf_cv_b2.best_params_)
{'max_features': 2, 'min_samples_leaf': 5, 'n_estimators': 500, 'random_state': 88}
#evaluate the CV-d Random Forest on the test dataset using the best parameters
rf_optimal = RandomForestClassifier(max_features=2, min_samples_leaf=5,
n_estimators=500, random_state=88)
rf_optimal_model = rf_optimal.fit(x_train, y_train_letter)
y_pred_rf_optimal = rf_optimal_model.predict(x_test)
cm_rf_optimal = confusion_matrix(y_test_letter, y_pred_rf_optimal)
accuracy(cm_rf_optimal.item((0,0)), cm_rf_optimal.item((0,1)), cm_rf_optimal.item((1, 0)), cm_rf_optimal.item((1, 1)))
1 b) vi)
from sklearn.ensemble import GradientBoostingClassifier
gbr = GradientBoostingClassifier(n_estimators=3300, max_leaf_nodes=10)
gbr.fit(x_train, y_train_letter)
#accuracy of Gradient Boosting Tree
from sklearn.metrics import confusion_matrix
def accuracy(tn, fp, fn, tp):
return (tn + tp)/(tn+fp+fn+tp)
y_pred_gbr = gbr.predict(x_test)
cm_gbr = confusion_matrix(y_test_letter, y_pred_gbr)
accuracy(cm_gbr.item((0,0)), cm_gbr.item((0,1)), cm_gbr.item((1, 0)), cm_gbr.item((1, 1)))
1 c)
#code from Lab 8b for bootstrapping
import time
def bootstrap_validation(test_data, test_label, train_label, model, metrics_list, sample=500, random_state=66):
tic = time.time()
n_sample = sample
n_metrics = len(metrics_list)
output_array=np.zeros([n_sample, n_metrics])
output_array[:]=np.nan
print(output_array.shape)
for bs_iter in range(n_sample):
bs_index = np.random.choice(test_data.index, len(test_data.index), replace=True)
bs_data = test_data.loc[bs_index]
bs_label = test_label.loc[bs_index]
bs_predicted = model.predict(bs_data)
for metrics_iter in range(n_metrics):
metrics = metrics_list[metrics_iter]
output_array[bs_iter, metrics_iter]=metrics(bs_predicted,bs_label)
# if bs_iter % 100 == 0:
# print(bs_iter, time.time()-tic)
output_df = pd.DataFrame(output_array)
return output_df
Bootstrap for LDA
#bootstrap for LDA
bs_output_lda = bootstrap_validation(x_test,y_test_letter,y_train_letter,lda,
metrics_list=[accuracy_score],
sample = 5000)
(5000, 1)
lda_accuracy = accuracy_score(y_test_letter, y_pred_lda)
fig, axs = plt.subplots(ncols=2, figsize=(12,5))
axs[1].set_xlabel('Boot Accuracy - Test Set Accuracy', fontsize=16)
axs[0].set_ylabel('Count', fontsize=16)
axs[0].hist(bs_output_lda.iloc[:,0], bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[0].set_xlim([0.4,0.7])
axs[1].hist(bs_output_lda.iloc[:,0]-lda_accuracy, bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[1].set_xlim([-0.15,0.15])
Bootstrap for CART
#bootstrap for CART
bs_output_cart = bootstrap_validation(x_test,y_test_letter,y_train_letter,dtc_test_b,
metrics_list=[accuracy_score],
sample = 5000)
(5000, 1)
cart_accuracy = accuracy_score(y_test_letter, y_pred_dtc_test_b)
fig, axs = plt.subplots(ncols=2, figsize=(12,5))
axs[0].set_xlabel('Bootstrap Accuracy Estimate', fontsize=16)
axs[1].set_xlabel('Boot Accuracy - Test Set Accuracy', fontsize=16)
axs[0].set_ylabel('Count', fontsize=16)
axs[0].hist(bs_output_cart.iloc[:,0], bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[0].set_xlim([0.4,0.7])
axs[1].hist(bs_output_cart.iloc[:,0]-cart_accuracy, bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[1].set_xlim([-0.15,0.15])
Bootstrap for Bagging
#bootstrap for Bagging
bs_output_bagging = bootstrap_validation(x_test,y_test_letter,y_train_letter,rf_model_b,
metrics_list=[accuracy_score],
sample = 5000)
(5000, 1)
bagging_accuracy = accuracy_score(y_test_letter, y_pred_rf_test_b)
fig, axs = plt.subplots(ncols=2, figsize=(12,5))
axs[0].set_xlabel('Bootstrap Accuracy Estimate', fontsize=16)
axs[1].set_xlabel('Boot Accuracy - Test Set Accuracy', fontsize=16)
axs[0].set_ylabel('Count', fontsize=16)
axs[0].hist(bs_output_bagging.iloc[:,0], bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[0].set_xlim([0.4,0.7])
axs[1].hist(bs_output_bagging.iloc[:,0]-bagging_accuracy, bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[1].set_xlim([-0.15,0.15])
Bootstrap for Random Forest
#Bootstrap for Random Forest
bs_output_rf = bootstrap_validation(x_test,y_test_letter,y_train_letter,rf_optimal_model,
metrics_list=[accuracy_score],
sample = 5000)
(5000, 1)
rf_accuracy = accuracy_score(y_test_letter, y_pred_rf_optimal)
fig, axs = plt.subplots(ncols=2, figsize=(12,5))
axs[0].set_xlabel('Bootstrap Accuracy Estimate', fontsize=16)
axs[1].set_xlabel('Boot Accuracy - Test Set Accuracy', fontsize=16)
axs[0].set_ylabel('Count', fontsize=16)
axs[0].hist(bs_output_rf.iloc[:,0], bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[0].set_xlim([0.4,0.7])
axs[1].hist(bs_output_rf.iloc[:,0]-rf_accuracy, bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[1].set_xlim([-0.15,0.15])
Bootstrap for Boosting
#Bootstrap for boosting
bs_output_boost = bootstrap_validation(x_test,y_test_letter,y_train_letter,gbr,
metrics_list=[accuracy_score],
sample = 5000)
(5000, 1)
boosting_accuracy = accuracy_score(y_test_letter, y_pred_gbr)
fig, axs = plt.subplots(ncols=2, figsize=(12,5))
axs[0].set_xlabel('Bootstrap Accuracy Estimate', fontsize=16)
axs[1].set_xlabel('Boot Accuracy - Test Set Accuracy', fontsize=16)
axs[0].set_ylabel('Count', fontsize=16)
axs[0].hist(bs_output_boost.iloc[:,0], bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[0].set_xlim([0.4,0.7])
axs[1].hist(bs_output_boost.iloc[:,0]-boosting_accuracy, bins=20,edgecolor='green', linewidth=2,color = "grey")
axs[1].set_xlim([-0.15,0.15])
# The 95% confidence interval of LDA
CI_0_lda = np.quantile(bs_output_lda.iloc[:,0]-lda_accuracy,np.array([0.025,0.975]))
CI_lda = [lda_accuracy - CI_0_lda[1],lda_accuracy - CI_0_lda[0]]
print("The 95-percent confidence interval of LDA accuracy is %s" % CI_lda) #0.5,0.64
The 95-percent confidence interval of LDA accuracy is [0.8823529411764706, 0.9197860962566845]
# The 95% confidence interval of CART
CI_0_cart = np.quantile(bs_output_cart.iloc[:,0]-cart_accuracy,np.array([0.025,0.975]))
CI_cart = [cart_accuracy - CI_0_cart[1],cart_accuracy - CI_0_cart[0]]
print("The 95-percent confidence interval of CART accuracy is %s" % CI_cart) #0.5,0.64
The 95-percent confidence interval of CART accuracy is [0.8994385026737972, 0.9347593582887701]
# The 95% confidence interval of Bagging
CI_0_bagging = np.quantile(bs_output_bagging.iloc[:,0]-bagging_accuracy,np.array([0.025,0.975]))
CI_bagging = [bagging_accuracy - CI_0_bagging[1],bagging_accuracy - CI_0_bagging[0]]
print("The 95-percent confidence interval of Bagging accuracy is %s" % CI_bagging) #0.5,0.64
The 95-percent confidence interval of Bagging accuracy is [0.9465240641711229, 0.9721925133689839]
# The 95% confidence interval of RF
CI_0_rf = np.quantile(bs_output_rf.iloc[:,0]-rf_accuracy,np.array([0.025,0.975]))
CI_rf = [rf_accuracy - CI_0_rf[1],rf_accuracy - CI_0_rf[0]]
print("The 95-percent confidence interval of RF accuracy is %s" % CI_rf) #0.5,0.64
The 95-percent confidence interval of RF accuracy is [0.9497326203208557, 0.9743315508021392]
# The 95% confidence interval of Boosting
CI_0_boosting = np.quantile(bs_output_boost.iloc[:,0]-boosting_accuracy,np.array([0.025,0.975]))
CI_boosting = [boosting_accuracy - CI_0_boosting[1],boosting_accuracy - CI_0_boosting[0]]
print("The 95-percent confidence interval of Boosting accuracy is %s" % CI_boosting) #0.5,0.64
The 95-percent confidence interval of Boosting accuracy is [0.9657754010695186, 0.9860962566844919]