# Install required packages !pip install -q -r requirements.txt

!pip install schrutepy==0.1.2

# Add any additional libraries or submodules below # If you do not have schrutepy you can pip install it from schrutepy import schrutepy # Display plots inline %matplotlib inline # Data libraries import pandas as pd import numpy as np # Plotting libraries import matplotlib.pyplot as plt import seaborn as sns # Plotting defaults plt.rcParams['figure.figsize'] = (8,5) plt.rcParams['figure.dpi'] = 80 # sklearn modules import sklearn from sklearn.metrics import mean_squared_error from sklearn.pipeline import make_pipeline from sklearn.model_selection import GridSearchCV, KFold

# Load data d = pd.read_csv("/work/the_office (1).csv") d.head(100)

print(f'The dataframe has missing values or NaN\'s : {d.isnull().values.any()}\n\n') sns.pairplot(d, y_vars = 'imdb_rating');

print('Column datatypes') print(d.dtypes)

directors = list(d['director'].unique()) print(d['director'].value_counts()) plt.hist(d['director'].value_counts(), bins = list(range(1,16)), ec = 'black'); plt.title('Count of Episodes Directed');

#Index directors based on who has directed more than 5 episodes prominentdirectors = d['director'].value_counts()[d['director'].value_counts() > 5] #Create a new indicator column for each director in the above list for director in prominentdirectors.index.tolist() : d[f'director_{director}'] = (d['director'] == f'{director}').astype(int)

#Writers requires a bit more engineering than directors, as they are stored as strings which can contain multiple names #The below code creates a list of every in every row, where writers are seperated by a semicolon all_writers = [i.split(',') for i in d['writer'].tolist() for i in i.split(';')] #Cast the list back to an array, so we can reuse our previous functions writers_series = pd.DataFrame(np.array(all_writers)) print(writers_series.value_counts().head(10)) plt.hist(writers_series.value_counts(), bins = list(range(1,22)), label = 'Number of Episodes Directed', ec = 'black'); plt.xticks(list(range(1,22,2))); plt.title('Count of Episodes Written');

#Filter for 10 or more episdoes written writers_series = writers_series.value_counts()[writers_series.value_counts() >= 10] #Pull them from the series into a list top_10_writers = [x[0] for x in writers_series.keys()] #Finally create the indicator columns for writers in top_10_writers : d[f'writer_{writers}'] = (d['writer'].str.contains(f'{writers}')).astype(int)

#load data df2 = schrutepy.load_schrute() #set every character name to lowercase, remove white space and quotation marks. This picks up a lot of the spelling errors df2['character'] = df2['character'].apply(lambda char : char.lower().replace(' ', '').strip('"')) #count lines per character and sort by size, take top 25 characters top_25_characters = list(df2.groupby(['character']).size().sort_values(ascending=False).keys()[0:25]) #Now group df by season and episode, then count lines per character. Then turn that back into a df and drop the index df2 = df2.groupby(['season', 'episode'])['character'].value_counts().to_frame(name = 'character_lines').reset_index() #Now pivot the table, so characters are now the columns, seasons and episodes are a multilevel index, and number of lines are the row values #Finally pull out only columns that are one of our top 25 characters by # of lines #Note 0's have gone to nan's df2 = df2.pivot_table(index = ['season', 'episode'], columns = 'character', values = 'character_lines')[top_25_characters] #set NaNs to 0 and convert floats to integers df2 = df2.fillna(value = 0).astype(int) #Finally join this to our original df d = d.join(df2, ['season', 'episode'])

the_office = d.drop(['episode_name', 'director', 'writer', 'air_date', 'main_chars', 'total_votes', 'episode', 'season'], axis = 1) the_office

def get_coefs(m): """Returns the model coefficients from a Scikit-learn model object as an array, includes the intercept if available. """ # If pipeline, use the last step as the model if (isinstance(m, sklearn.pipeline.Pipeline)): m = m.steps[-1][1] if m.intercept_ is None: return m.coef_ return np.concatenate([[m.intercept_], m.coef_]) def model_fit(m, X, y, plot = False): """Returns the root mean squared error of a fitted model based on provided X and y values. Args: m: sklearn model object X: model matrix to use for prediction y: outcome vector to use to calculating rmse and residuals plot: boolean value, should fit plots be shown """ y_hat = m.predict(X) rmse = mean_squared_error(y, y_hat, squared=False) res = pd.DataFrame( data = {'y': y, 'y_hat': y_hat, 'resid': y - y_hat} ) if plot: plt.figure(figsize=(12, 6)) plt.subplot(121) sns.lineplot(x='y', y='y_hat', color="grey", data = pd.DataFrame(data={'y': [min(y),max(y)], 'y_hat': [min(y),max(y)]})) sns.scatterplot(x='y', y='y_hat', data=res).set_title("Fit plot") plt.subplot(122) sns.scatterplot(x='y', y='resid', data=res).set_title("Residual plot") plt.subplots_adjust(left=0.0) plt.suptitle("Model rmse = " + str(round(rmse, 4)), fontsize=16) plt.show() return rmse

from sklearn.model_selection import train_test_split # Our model matrix X and outcome vector y X = the_office.drop(['imdb_rating'], axis=1) y = the_office.imdb_rating # Obtain a test and train split for both X and y X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

print("orig sizes :", X.shape, y.shape) print("train sizes:", X_train.shape, y_train.shape) print("test sizes :", X_test.shape, y_test.shape)

from sklearn.linear_model import LinearRegression lm = LinearRegression().fit(X_train, y_train)

model_fit(lm, X_test, y_test, plot=True)

from sklearn.preprocessing import StandardScaler S = StandardScaler().fit(X) lm_scaled = make_pipeline( StandardScaler(), LinearRegression() ).fit(X_train, y_train) model_fit(lm_scaled, X_test, y_test, plot=True)

from sklearn.linear_model import Lasso

l = make_pipeline( StandardScaler(), Lasso(alpha=0.15) ).fit(X_train, y_train)

print("lasso rmse:", model_fit(l, X_test, y_test, plot=True)) print("lasso coefs:", get_coefs(l))

get_coefs(l)

def find_order_features(A): """Returns two lists, one containing all the features in the model matrix X which have nonzero coefficients from the model, the second containing all the corresponding coefficients, and decreasingly order both lists in terms of the highest coefficients in the model and corresponding features. """ features = [] coefficients = [] indices = [] # Convert array to list list_all_coef = list(A) length = len(list_all_coef) for k in range(1, length): if list_all_coef[k] != 0: features.append(X.columns[k-1]) coefficients.append(list_all_coef[k]) indices.append(k) abs_coefficients = [] for i in range(len(coefficients)): abs_coefficients.append(abs(coefficients[i])) feat_coefs = pd.DataFrame(features) feat_coefs['coefficients'] = coefficients feat_coefs['abs_coefficients'] = abs_coefficients #ordered_coefficients = sorted(coefficients, reverse = True) return (feat_coefs)

find_order_features(get_coefs(l))

alphas = np.linspace(0.01, 1, num=100) betas = [] # Store coefficients rmses = [] # Store validation rmses for a in alphas: m = make_pipeline( StandardScaler(), Lasso(alpha=a) ).fit(X_train, y_train) # Again ignore the intercept since it isn't included in the penalty betas.append(get_coefs(m)[1:]) rmses.append(model_fit(m, X_test, y_test)) res = pd.DataFrame( data = betas, # Coefficients columns = X.columns # Coefficient names ).assign( alpha = alphas, # Add alpahs rmse = rmses # Add validation rmses ).melt( id_vars = ('alpha', 'rmse') # Move columns into the rows )

#SORT LEGEND sns.lineplot(x='alpha', y='value', hue='variable', data=res, legend=False).set_title("Coefficients") plt.show()

alphas = np.linspace(0.01, 1, num=100) l_gs = GridSearchCV( make_pipeline( StandardScaler(), Lasso() ), param_grid={'lasso__alpha': alphas}, cv=KFold(10, shuffle=True, random_state=1234), scoring="neg_root_mean_squared_error" ).fit(X_train, y_train)

print( "best alpha:", l_gs.best_params_['lasso__alpha']) print( "best rmse :", l_gs.best_score_ * -1) print( "validation rmse:", model_fit(l_gs.best_estimator_, X_test, y_test))

l_cv_res = pd.DataFrame().assign( alpha = alphas, rmse = -1 * l_gs.cv_results_['mean_test_score'], # mean of the rmse over the folds rmse_se = l_gs.cv_results_['std_test_score'] / np.sqrt(l_gs.n_splits_), # standard error of the rmse )

plt.figure(figsize=(14, 6)) plt.subplot(121) sns.lineplot(x='alpha', y='rmse', data=l_cv_res) plt.subplot(122) sns.lineplot(x='alpha', y='rmse', data=l_cv_res).set_xscale('log')

best = l_gs.best_params_['lasso__alpha'] l2 = make_pipeline( StandardScaler(), Lasso(alpha=best) ).fit(X_train, y_train)

print("lasso rmse:", model_fit(l2, X_test, y_test, plot=True)) print("lasso coefs:", get_coefs(l2))

feats_df = find_order_features(get_coefs(l2)) feats_df.sort_values('abs_coefficients', ascending = False)

# Run the following to render to PDF !jupyter nbconvert --to pdf project1.ipynb