# Install required packages
!pip install -q -r requirements.txt
# Add any additional libraries or submodules below
# Display plots inline
%matplotlib inline
# Data libraries
import pandas as pd
import numpy as np
# Plotting libraries
import matplotlib.pyplot as plt
import seaborn as sns
# Plotting defaults
plt.rcParams['figure.figsize'] = (8,6)
plt.rcParams['figure.dpi'] = 80
# sklearn modules
import sklearn
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Lasso,RidgeCV,LinearRegression
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import KFold,GridSearchCV,train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import PolynomialFeatures
from sklearn.compose import ColumnTransformer, make_column_transformer
from sklearn.compose import make_column_selector
# Load data
d = pd.read_csv("the_office.csv")
d.head(5)
sns.pairplot(d.drop(["season","episode"],axis=1))
d.loc[d.director=="Ken Wittingham","director"]="Ken Whittingham"
d.loc[d.director=="Lee Eisenberg;Gene Stupnitsky","director"]="Gene Stupnitsky;Lee Eisenberg"
d.loc[d.director=="Greg Daneils","director"]="Greg Daniels"
d.loc[d.director=="Charles McDougal","director"]="Charles McDougall"
d.loc[d.director=="Claire Scanlong","director"]="Claire Scanlon"
d.loc[d.director=="Paul Lieerstein","director"]="Paul Lieberstein"
df_new = d
df_new["director"] = df_new["director"].str.split(";")
df_new = df_new.explode("director")
df_new["writer"] = df_new["writer"].str.split(";")
df_new = df_new.explode("writer")
df_new["main_chars"] = df_new["main_chars"].str.split(";")
df_new = df_new.explode("main_chars")
df_new = df_new.drop(['air_date','season', 'episode','episode_name'], axis=1)
plt.figure(figsize=(8,9))
sns.boxplot(x=df_new.imdb_rating,y=df_new.director)
sns.boxplot(x=df_new.imdb_rating,y=df_new.writer)
sns.boxplot(x=df_new.imdb_rating,y=df_new.main_chars)
#####date
from datetime import datetime
d["weekday"] = pd.to_datetime(d.air_date).map(lambda x: datetime.weekday(x)+1)
sns.boxplot(x=d.weekday,y=d.imdb_rating)
d.groupby(["weekday"]).count().episode
df_dummy = pd.get_dummies(df_new, drop_first=False)
df_dummy=df_dummy.reset_index(drop =True)
df_dummy.head(5)
## some important function we need to use afterward
def get_coefs(m):
"""Returns the model coefficients from a Scikit-learn model object as an array,
includes the intercept if available.
"""
# If pipeline, use the last step as the model
if (isinstance(m, sklearn.pipeline.Pipeline)):
m = m.steps[-1][1]
if m.intercept_ is None:
return m.coef_
return np.concatenate([[m.intercept_], m.coef_])
def model_fit(m, X, y, plot = False):
"""Returns the root mean squared error of a fitted model based on provided X and y values.
Args:
m: sklearn model object
X: model matrix to use for prediction
y: outcome vector to use to calculating rmse and residuals
plot: boolean value, should fit plots be shown
"""
y_hat = m.predict(X)
rmse = mean_squared_error(y, y_hat, squared=False)
res = pd.DataFrame(
data = {'y': y, 'y_hat': y_hat, 'resid': y - y_hat}
)
if plot:
plt.figure(figsize=(12, 6))
plt.subplot(121)
sns.lineplot(x='y', y='y_hat', color="grey", data = pd.DataFrame(data={'y': [min(y),max(y)], 'y_hat': [min(y),max(y)]}))
sns.scatterplot(x='y', y='y_hat', data=res).set_title("Fit plot")
plt.subplot(122)
sns.scatterplot(x='y', y='resid', data=res).set_title("Residual plot")
plt.subplots_adjust(left=0.0)
plt.suptitle("Model rmse = " + str(round(rmse, 4)), fontsize=16)
plt.show()
return rmse
# standardise the numerical variables
standardised_X = pd.DataFrame(make_column_transformer(
(StandardScaler(),make_column_selector(dtype_include=int)),
remainder='passthrough'
).fit_transform(df_dummy.drop("imdb_rating",axis=1)))
standardised_X.columns = df_dummy.drop("imdb_rating",axis=1).columns
#feature selection
y = df_dummy.imdb_rating
X = np.c_[np.ones(len(y)),standardised_X]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,random_state=0)
alphas = np.linspace(0.001, 0.1, num=100)
l_gs = GridSearchCV(
Lasso(),
param_grid={'alpha': alphas},
cv=KFold(5, shuffle=True, random_state=1234),
scoring="neg_root_mean_squared_error"
).fit(X_train, y_train)
l_cv_res = pd.DataFrame().assign(
alpha = alphas,
rmse = -1 * l_gs.cv_results_['mean_test_score'], # mean of the rmse over the folds
rmse_se = l_gs.cv_results_['std_test_score'] / np.sqrt(l_gs.n_splits_), #standard error of the rmse
)
#plot about rmse vs. alpha
plt.figure(figsize=(14, 6))
plt.subplot(121)
sns.lineplot(x='alpha', y='rmse', data=l_cv_res)
plt.subplot(122)
sns.lineplot(x='alpha', y='rmse', data=l_cv_res).set_xscale('log')
#feature selection with fixed alpha
y = df_dummy.imdb_rating
X = np.c_[np.ones(len(y)),standardised_X]
la_coef = Lasso(alpha=0.005).fit(X, y).coef_
variables = pd.DataFrame({"variables":df_dummy.columns,"values":la_coef})
variables.loc[0,"variables"]="intercept"
print(variables[variables["values"]!=0].variables.tolist())
# split data with selected features into train set and test set
new_variable = variables[variables["values"]!=0].variables
y = df_dummy.imdb_rating
X = standardised_X[new_variable]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,random_state=0)
# build a polynomial model
from sklearn.preprocessing import PolynomialFeatures
from sklearn.compose import ColumnTransformer, make_column_transformer
# Using 21 selected features
m2 = make_pipeline(
make_column_transformer(
(PolynomialFeatures(degree=3, include_bias=False), ['total_votes']),
(PolynomialFeatures(degree=3, include_bias=False), ['n_lines']),
(PolynomialFeatures(degree=3, include_bias=False), ['n_directions']),
(PolynomialFeatures(degree=3, include_bias=False), ['n_words']),
(PolynomialFeatures(degree=3, include_bias=False), ['n_speak_char']),
remainder='passthrough'
),
LinearRegression(fit_intercept=True)
)
parameters = {
'columntransformer__polynomialfeatures-1__degree': np.arange(1,4,1),
'columntransformer__polynomialfeatures-2__degree': np.arange(1,4,1),
'columntransformer__polynomialfeatures-3__degree': np.arange(1,4,1),
'columntransformer__polynomialfeatures-4__degree': np.arange(1,4,1),
'columntransformer__polynomialfeatures-5__degree': np.arange(1,4,1),
}
polin = GridSearchCV(
m2,
parameters,
cv=KFold(5, shuffle=True, random_state=1234),
scoring="neg_root_mean_squared_error"
).fit(X_train, y_train)
print( "best alpha:", polin.best_params_)
print( "best rmse :", polin.best_score_ * -1)
print( "validation rmse:", model_fit(polin.best_estimator_, X_test, y_test) )
print( "estimates of coefficients:", get_coefs(polin.best_estimator_) )
model_fit(polin.best_estimator_, X_test, y_test, plot=True)
plt.subplot(231)
x = np.arange(min(standardised_X["total_votes"]), max(standardised_X["total_votes"]), 0.1)
y = 0.555*x-2.4883*x**2+0.0362*x**3
plt.plot(x,y)
plt.title("Total votes")
plt.subplot(232)
x = np.arange(min(standardised_X["n_lines"]), max(standardised_X["n_lines"]), 0.1)
y = 0.232*x+0.0057*x**2-0.0187*x**3
plt.plot(x,y)
plt.title("n_lines")
plt.subplot(233)
x = np.arange(min(standardised_X["n_directions"]), max(standardised_X["n_directions"]), 0.1)
y = 0.0851*x-0.0169*x**2
plt.plot(x,y)
plt.title("n_directions")
plt.subplot(234)
x = np.arange(min(standardised_X["n_words"]), max(standardised_X["n_words"]), 0.1)
y = -0.122*x-0.0634*x**2+0.0357*x**3
plt.plot(x,y)
plt.title("n_words")
plt.subplot(235)
x = np.arange(min(standardised_X["n_speak_char"]), max(standardised_X["n_speak_char"]), 0.1)
y = 0.0888*x-0.0358*x**2-0.012*x**3
plt.plot(x,y)
plt.title("n_speak_char")
mu=np.mean(df_dummy["n_lines"])
sd=np.std(df_dummy["n_lines"])
x=2
x*sd+mu
mu=np.mean(df_dummy["n_directions"])
sd=np.std(df_dummy["n_directions"])
x=2.3
x*sd+mu
mu=np.mean(df_dummy["n_words"])
sd=np.std(df_dummy["n_words"])
x=-0.5
x*sd+mu
mu=np.mean(df_dummy["n_speak_char"])
sd=np.std(df_dummy["n_speak_char"])
x=1.3
x*sd+mu
# Run the following to render to PDF
!jupyter nbconvert --to markdown project1.ipynb
