Behaviour Analytics in Python

# Converting the event_time column df['event_time'] = pd.to_datetime(df['event_time'], format='%Y-%m-%d %H:%M:%S UTC') # Split time column into month day year hour df['event_day'] = df['event_time'].dt.day df['event_month'] = df['event_time'].dt.month df['event_year'] = df['event_time'].dt.year df['event_hour'] = df['event_time'].dt.hour # df = df.drop(['event_time'], axis=1) # Remove the column after it is processed.

brands = df.brand.unique() # Get list of unique brands. brands = [b for b in brands if str(b) != 'nan'] for brand in brands: brand_group = df[df["brand"] == brand] # Filter dataset by brand. common_category = brand_group.mode()['category_code'][0] # Get modal category_code per brand. same_brand = df["brand"] == brand null_category = df["category_code"].isna() df.loc[(same_brand) & (null_category),'category_code'] = common_category

category_codes = df.category_code.unique() # Get list of unique brands. category_codes = [c for c in category_codes if str(c) != 'nan'] for category_code in category_codes: code_group = df[df["category_code"] == category_code] # Filter dataset by category_code. common_brand = code_group.mode()['brand'][0] # Get modal brand per category_code. same_category = df["category_code"] == brand null_brand = df["brand"].isna() df.loc[(same_category) & (null_brand),'brand'] = common_brand

# Splitting the category into category, sub_category and item_type. df[["category", "sub_category", "item"]] = df['category_code'].str.split('.', expand=True)

# What are we good at? What are we not so good at? views = df[df["event_type"] == "view"] purchases = df[df["event_type"] == "purchase"] removals = df[df["event_type"] == "remove_from_cart"]

total_made = float("{:.2f}".format(purchases['price'].sum())) total_lost = float("{:.2f}".format(removals['price'].sum())) print(f"Total Made: ${total_made:,}") print(f"Total Lost: ${total_lost:,}")

from IPython.display import display, Markdown, Latex var = 10 display(Markdown(f"# Getting to a {var}% increase")) # If you particularly want to display maths, this is more direct:

unique_customers = df.user_id.nunique() # number of customers # 368,232 print("Number of customers: ",unique_customers) # Spend per customer user_spending = purchases.groupby(['user_id'])['price'].sum().reset_index() spend_per_customer = user_spending['price'].mean() print("Spend per customer: ", spend_per_customer) # $48.56 # Loss per customer user_removals = removals.groupby(['user_id'])['price'].sum().reset_index() loss_per_customer = user_removals['price'].mean() print("Loss per customer: ", loss_per_customer) # $86.29 # Loss per session? user_removals = removals.groupby(['user_session'])['price'].sum().to_frame().reset_index() loss_per_session = user_removals['price'].mean() print("Loss per session: ", loss_per_session) # $42.08 # Number of successful recoveries. spending_target = total_lost / loss_per_customer print("Number of recovering needed: ", spending_target) # 54627.99 recoveries_per_customer = unique_customers/spending_target print("Recoveries per customer: ", recoveries_per_customer) # 6.74 # Number of sessions per month. per user session_counts = df.groupby(['user_id'])['user_session'].size().to_frame().reset_index() user_sessions = session_counts['user_session'].mean() print("user_sessions per month: ", user_sessions) # 12.58 # Number of recoveries per month. # purchases_n_removals = df[df["event_type"].isin(["remove_from_cart", "purchase"])] # user_activity = purchases_n_removals.groupby(["user_id", "event_type", "product_id", "user_session"]).size().to_frame().reset_index() # user_activity.drop_duplicates(inplace=True) # user_activity.drop(["event_type"], axis=1, inplace=True) # user_activity.groupby(user_activity.columns.tolist(),as_index=False).size() # print("Recoveries this month", len(recoveries[recoveries[0] > 1])) # How much money is recovered? # # df.groupby(df.columns.tolist(),as_index=False).size() # go back to removals and purchases # add a purchase date and a removal date purchases["purchase_date"] = purchases['event_time'] removals["removal_date"] = removals['event_time'] criteria = ["user_id", "product_id", "user_session", "price"] recovery_data = pd.merge(left=purchases, right=removals, left_on=criteria, right_on=criteria) # join on user_id and product_id # look for where purchase date is greater than removal date recoveries = recovery_data.query('purchase_date > removal_date') print("Recoveries: ", len(recoveries)) # 26746 print("Total recovered: ", recoveries["price"].sum()) # $148635.94 updated_goal = total_lost - 148635.94 print("New target: ", updated_goal) # 4565571.1

# Bubble Chart showing the money lost by category. category_data = removals.groupby(['category'])["price"].sum().reset_index() fig = px.scatter(category_data, x="price", y="price", size="price", color="category", hover_name="category", log_x=True, size_max=60) fig.show()

# Most popular Sub Categories. sub_category_data = removals.groupby(['sub_category'])["price"].sum().reset_index() fig = px.pie(sub_category_data, values='price', names='sub_category', title='Revenue Lost by Sub Category') fig.show()

majority_lost = removals[removals.category.isin(['furniture', 'appliances'])]['price'].sum() majority_lost = float("{:.2f}".format(majority_lost)) print(f"Money lost from furniture and appliances: ${majority_lost:,}")

# Creating a Tree-Map Diagram for drill downs. # import plotly.express as px # df = px.data.tips() # fig = px.treemap(df, path=['day', 'time', 'sex'], values='total_bill') # fig.show()

# Number of users per hour. # Should the data be restricted by event type? mau_hour_df = df.groupby(['event_hour'])['user_id'].size().reset_index() fig = px.bar(mau_hour_df, x='event_hour', y='user_id', title="Users by Time of Day", labels={ "event_hour": "Hour", "user_id": "Users", }) fig.show()

# event_type object # this can certainly be encoded. This is your target though. # There will only be two labels: purchase, remove from cart. # obj_df["body_style_cat"] = obj_df["body_style"].cat.codes # data = df[df.event_type.isin(['remove_from_cart', 'purchase'])] data = pd.concat([purchases,removals]) print(data.columns) drop_columns = ["category_code", "category_id", "user_session"] one_hot_encode_columns = ["category", "sub_category", "item"] target_encode_columns = ["brand"] data.drop(drop_columns, axis=1, inplace=True) # pd.get_dummies(obj_df, columns=["drive_wheels"]).head() for column in one_hot_encode_columns: pd.get_dummies(data, columns=[column]).head() # data.drop(column) # encoding the target column itself, Event Type # oe_event = OneHotEncoder() # oe_results = oe_event.fit_transform(obj_df[["event_type"]]) # pd.DataFrame(oe_results.toarray(), columns=oe_event.categories_).head() ord_enc = OrdinalEncoder() # data["make_code"] = ord_enc.fit_transform(data[["event_type"]]) data["event_type"] = ord_enc.fit_transform(data[["event_type"]]) # Target Encoding encodings = data.groupby('brand')['event_type'].mean().reset_index() encodings exit() # data = data.merge(encodings, how='left', on='brand') # data.drop('brand', axis=1, inplace=True) # # https://towardsdatascience.com/handling-categorical-data-the-right-way-9d1279956fc6

# Load our Machine Learning Model. # model = TabularPredictor.load("AutogluonModels/")

train_data, test_data = train_test_split(data, test_size=0.2, random_state=0) label = 'event_type' y_train = train_data[label] y_test = test_data[label] X_train = pd.DataFrame(train_data.drop(columns=[label])) X_test = pd.DataFrame(test_data.drop(columns=[label]))

df.user_id.nunique() # number of customers # 368,232 # Spend per customer user_spending = purchases.groupby(['user_id'])['price'].sum().reset_index() spend_per_customer = user_spending['price'].mean() print(spend_per_customer) # $48.56 # Loss per customer user_removals = removals.groupby(['user_id'])['price'].sum().reset_index() loss_per_customer = user_spending['price'].mean() print(loss_per_customer) # $86.29 # Number of successful recoveries. print(471420.70/ 86.29) # Number of sessions per month. # Number of recoveries per month.

# who contributed the most to the loss? highest_loss = user_removals['price'].max() biggest_loser = user_removals.loc[user_removals['price'] == highest_loss, 'user_id'].iloc[0] print(biggest_loser) # 500179293

# Let's work with that one user. # user_session # Find the user session with the most loss? # Find the session where products of a particular category were removed. category_ = X_train["category"] == "appliances" sub_category_ = X_train["sub_category"] == "environment" item_ = X_train["item"] == 'vacuum' datapoint = X_train[category_ & sub_category_ & item_].sample(n=1, random_state=0) datapoint

class AutogluonWrapper: def __init__(self, predictor, feature_names): self.ag_model = predictor self.feature_names = feature_names def predict_proba(self, X): if isinstance(X, pd.Series): X = X.values.reshape(1,-1) if not isinstance(X, pd.DataFrame): X = pd.DataFrame(X, columns=self.feature_names) return self.ag_model.predict_proba(X)

baseline = X_train.sample(100) # X_train.mode() could also be reasonable baseline for both numerical/categorical features rather than an entire dataset. # We can now create a KernelExplainer which will return Kernel SHAP values to explain particular AutoGluon predictions. ag_wrapper = AutogluonWrapper(model, X_train.columns) explainer = shap.KernelExplainer(ag_wrapper.predict_proba, baseline) print("Baseline prediction: ", np.mean(ag_wrapper.predict_proba(baseline))) # this is the same as explainer.expected_value NSHAP_SAMPLES = 100 # how many samples to use to approximate each Shapely value, larger values will be slower

shap.summary_plot(shap_values, X_test)

shap.dependence_plot("education-num", shap_values, X_test)

# df.loc[df['First Season'] > 1990, 'First Season'] = 1 # df['First Season'] = (df['First Season'] > 1990).astype(int) # selecting and changing