!pip install category_encoders

# Imports import numpy as np import pandas as pd import warnings warnings.filterwarnings('ignore') # Feature Engineering from category_encoders import OneHotEncoder, OrdinalEncoder from sklearn.experimental import enable_iterative_imputer from sklearn.impute import IterativeImputer, SimpleImputer from sklearn.base import TransformerMixin from sklearn.base import BaseEstimator from sklearn.preprocessing import QuantileTransformer, StandardScaler from sklearn.compose import ColumnTransformer # Cross Validation and PipeLine from sklearn.model_selection import RandomizedSearchCV, cross_val_score from sklearn.pipeline import Pipeline # Models from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, StackingClassifier, \ GradientBoostingClassifier, BaggingClassifier, VotingClassifier from sklearn.neighbors import KNeighborsClassifier from sklearn.linear_model import LogisticRegression, SGDClassifier from sklearn.svm import LinearSVC # Model Evaluation from sklearn.metrics import balanced_accuracy_score, f1_score, plot_confusion_matrix, precision_score, recall_score,accuracy_score from sklearn.inspection import permutation_importance

df = pd.read_csv('HR_Analytics/aug_train.csv') y = df['target'] # Keep target in X for preprocessing X = df.drop('target', axis=1) X.shape, y.shape

class OrdinalHelper(BaseEstimator, TransformerMixin): """Changes the name of the values in the ordinal columns so that OrdinalEncoder can be used in our pipeline. (Which defaults to alphabetical and there isn't an easy way to add a mapping per column)""" def __init__(self): pass def fit(self, X, y=None): ''' Given our data, return a dictionary where the keys are the column names and the values are the mapping dictionary for that column. The mapping dictionary will first organize each columns unique values by the percent of them that result in target==1. Then it will assign a new name for each unique value relative to its rank. This is because using OrdinalEncoder in a Pipeline defaults to sorting the unique values alphabetically. ''' # Use the training data (always) temp_X = pd.read_csv('HR_Analytics/aug_train.csv') # Get only the columns we care about global ord_cols col_needed = ord_cols + ['target'] temp_X = temp_X[col_needed] # Need to impute it here (There are some redundancies in my code, but this was the # simplest way to make sure that I wasn't doing any data leakage in my final model) sc = SimpleImputer(missing_values=np.nan, fill_value='missing', strategy="constant") temp_X = sc.fit_transform(temp_X) temp_X = pd.DataFrame(temp_X)# # Create mapping scheme ordinal_col_mapping = [] name_mapping = {} for col in temp_X.columns[:-1]: temp_dic = {} temp_dic['col'] = col val_count = [] # Set of unique value, count in X[col] for val in temp_X[col].unique(): # Get the P(target=1|X[col]==val) target_true = np.sum(temp_X.loc[np.where(temp_X[col] == val)][len(temp_X.columns)-1])\ /len(temp_X.loc[np.where(temp_X[col] == val)][len(temp_X.columns)-1]) val_count.append((val, target_true)) # Sort val_count (ascending): val_count.sort(key=lambda x: x[1]) mapping = {} for index, tup in enumerate(val_count): mapping[tup[0]] = chr(97+index) + tup[0] name_mapping[col] = mapping self.mapping = name_mapping return self def transform(self, X, y=None): ''' Apply our mapping to all of the ordinal columns in X ''' for row_index, row in enumerate(X): for col_index, val in enumerate(row): X[row_index, col_index] = self.mapping[col_index][val] return X

# Our categorical column we want to OneHotEncoder cat_cols = ['city', 'gender'] ord_cols = ['enrolled_university', 'major_discipline', 'experience', 'last_new_job', 'company_type', 'company_size', 'education_level', 'relevent_experience'] # Our numerical columns num_cols = ['city_development_index', 'training_hours'] # Ordinal Pipeline ord_pipe = Pipeline([("impute", SimpleImputer(missing_values=np.nan, fill_value='missing', strategy="constant")), ("encode", OrdinalHelper()), ('ordinal', OrdinalEncoder(handle_unknown='ignore'))]) # Categorical Pipeline cat_pipe = Pipeline([("impute", SimpleImputer(missing_values=np.nan, fill_value='missing', strategy="constant")), ("encode", OneHotEncoder())]) # Numerical Pipeline num_pipe = Pipeline([("impute", IterativeImputer(missing_values=np.nan, max_iter=10, initial_strategy="median")), ("scaler", StandardScaler()), ("transformer", QuantileTransformer(output_distribution='normal'))]) # Group them together with ColumnTransformer preprocessing = ColumnTransformer([('categorical', cat_pipe, cat_cols), ('ordinal', ord_pipe, ord_cols), ('numerical', num_pipe, num_cols)])

# Helper class class DummyEstimator(BaseEstimator): "Pass through class, methods are present but do nothing." def fit(self): pass def score(self): pass

np.random.seed(1) # Use a random seed to have consistent results pipe = Pipeline([("preprocessing", preprocessing), ("clf", DummyEstimator())]) search_space = [ {'clf': [ExtraTreesClassifier(n_jobs=-1)], 'clf__min_samples_leaf': np.linspace(1, 30, 4, dtype=int), 'clf__bootstrap': [True, False], 'clf__max_samples': [5, 13, 20, 500], 'clf__class_weight': [None, 'balanced', 'balanced_subsample'], 'clf__n_estimators': np.linspace(50, 500, 4, dtype=int)}, {'clf': [RandomForestClassifier(n_jobs=-1)], 'clf__min_samples_leaf': np.linspace(1, 10, 4, dtype=int), 'clf__bootstrap': [True, False], 'clf__max_samples': [5, 13, 20] , 'clf__class_weight': [None, 'balanced', 'balanced_subsample'], 'clf__n_estimators': np.linspace(50, 300, 4, dtype=int)}, {'clf': [KNeighborsClassifier(n_jobs=-1)], 'clf__n_neighbors': np.linspace(3, 13, 3, dtype=int), 'clf__weights': ['uniform', 'distance'], 'clf__p': [1,2]}, {'clf': [LinearSVC()], 'clf__class_weight': ['balanced', None], 'clf__C': np.linspace(0.001, 10, 10)}, {'clf': [LogisticRegression(n_jobs=-1)], 'clf__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'], 'clf__class_weight': ['balanced', None], 'clf__penalty': ['l1', 'l2', 'elasticnet', 'none']}, {'clf': [SGDClassifier(n_jobs=-1)], 'clf__loss': ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron'], 'clf__class_weight': ['balanced', None], 'clf__penalty': ['l1', 'l2', 'elasticnet']}] # Create grid search gs = RandomizedSearchCV(pipe, search_space, scoring='f1_weighted', n_iter=25, cv=5, n_jobs=-1, error_score='raise') gs.fit(X, y);

# Helper Function for displaying results of cross_val_score def print_val_score(res): mean = np.mean(res) max_ = np.max(res) min_ = np.min(res) std = np.std(res) median = np.median(res) print(f"Mean: {mean:.3f} | Min: {min_:.3f} | Median: {median:.3f} | Max: {max_:.3f} | Std: {std:.3f}") # Calculate the general stats for our first model to have a way to compare other methods rfc_clf = RandomForestClassifier(bootstrap=False, class_weight='balanced', min_samples_leaf=4, n_estimators=216, n_jobs=-1) pipe = Pipeline([('preprocessing', preprocessing), ('model', rfc_clf)]) res = cross_val_score(pipe, X, y, cv=5, scoring='f1_weighted') print_val_score(res)

# Different Models that I will be using: rfc_clf = RandomForestClassifier(bootstrap=False, class_weight='balanced', min_samples_leaf=4, n_estimators=216, n_jobs=-1) lr_clf = LogisticRegression(class_weight='balanced', n_jobs=-1, solver='sag') etc_clf = ExtraTreesClassifier(class_weight='balanced', max_samples=20, min_samples_leaf=10, n_estimators=50, n_jobs=-1) # Different pipelines pipe1 = Pipeline([('preprocessing', preprocessing), ('model', rfc_clf)]) pipe2 = Pipeline([('preprocessing', preprocessing), ('model', lr_clf)]) pipe3= Pipeline([('preprocessing', preprocessing), ('model', etc_clf)]) estimators = [('rfc', pipe1), ('lr', pipe2), ('etc', pipe3)] # Initialize my VotingClassifier voting_clf = VotingClassifier(estimators = estimators, voting = 'soft',) # Use Cross_val_score to see how well my model performs: res = cross_val_score(voting_clf, X, y, cv=5, scoring='f1_weighted') print_val_score(res)

rfc_clf = RandomForestClassifier(bootstrap=False, class_weight='balanced', min_samples_leaf=4, n_estimators=216, n_jobs=-1) bag = BaggingClassifier(rfc_clf, n_estimators=11, max_samples=0.8, max_features=0.9, oob_score=True) bagging_pipe = Pipeline([('preprocessing', preprocessing), ('model', bag)]) res = cross_val_score(bagging_pipe, X, y, cv=5, scoring='f1_weighted') print_val_score(res)

final_estimator = GradientBoostingClassifier() estimators = [('rfc', pipe1), ('lr', pipe2), ('etc', pipe3)] reg = StackingClassifier(estimators=estimators, final_estimator=final_estimator, n_jobs=-1) res = cross_val_score(reg, X, y, cv=5, scoring='f1_weighted') print_val_score(res)

X_test = pd.read_csv('HR_Analytics/aug_test.csv') y_test = np.load('HR_Analytics/aug_test_y.npy')

np.random.seed(1) # Consistent results # Different Models that I will be using: rfc_clf = RandomForestClassifier(bootstrap=False, class_weight='balanced', min_samples_leaf=4, n_estimators=216, n_jobs=-1) lr_clf = LogisticRegression(class_weight='balanced', n_jobs=-1, solver='sag') etc_clf = ExtraTreesClassifier(class_weight='balanced', max_samples=20, min_samples_leaf=10, n_estimators=50, n_jobs=-1) # Different pipelines pipe1 = Pipeline([('preprocessing', preprocessing), ('model', rfc_clf)]) pipe2 = Pipeline([('preprocessing', preprocessing), ('model', lr_clf)]) pipe3= Pipeline([('preprocessing', preprocessing), ('model', etc_clf)]) estimators = [('rfc', pipe1), ('lr', pipe2), ('etc', pipe3)] reg = StackingClassifier(estimators=estimators, final_estimator=GradientBoostingClassifier(), n_jobs=-1) reg.fit(X,y);

y_pred = reg.predict(X_test) print(f"Weighted f1_Score = {f1_score(y_test, y_pred, average='weighted'):.3f}")

plot_confusion_matrix(reg, X_test, y_test, display_labels=['Not Looking', 'Looking for Work']); rec = recall_score(y_test, y_pred) prec = precision_score(y_test, y_pred) print(f"Recall Score: {rec:.2f} | Precision Score: {prec:.2f}",\ f" | f1 Weighted Score: {f1_score(y_test, y_pred, average='weighted'):.2f}");

np.random.seed(1) r = permutation_importance(reg, X_test, y_test, n_repeats=100) print(f"{'Feature':^25}{'Mean':^8}{'std':^13}") print('-'*(25+8+13)) for i in r.importances_mean.argsort()[::-1]: if r.importances_mean[i] - 2 * r.importances_std[i] > 0: print(f"{X.columns[i]:<25}",\ f"{r.importances_mean[i]:.2f}",\ f" +/- {r.importances_std[i]:.2f}")