Análisis de la Personalidad del Consumidor

Fuente y diccionario de datos: https://www.kaggle.com/datasets/imakash3011/customer-personality-analysis

Autor: Urdanegui Bisalaya, Sebastian Marat | Data Scientist | Data Analyst 📊

Librerías a importar

import pandas as pd import numpy as np import matplotlib.pyplot as plt from matplotlib import colors from matplotlib.colors import ListedColormap import seaborn as sns

Cargar el dataset

data = pd.read_csv("marketing_campaign.csv", sep = "\t") data.head()

print(f'La información del dataset se encuentra dividida en {data.shape[0]} filas y {data.shape[1]} columnas.')

data.columns

Descripción estadística del dataset

data.describe()

Limpieza de los datos

data.info()

Análisis de valores nulos en el dataframe

print("Valores nulos en el dataframe:") print(data.isnull().sum().sort_values(ascending = False))

ratio_data = ((data.isnull().sum()/len(data))*100).sort_values(ascending = False) print("Porcentaje de valores nulos en el dataframe: (%)") print(ratio_data)

Solución a los valores nulos del dataframe

Análisis de distribución del feature que presenta valores nulos

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) cmap = colors.ListedColormap(["#682F2F", "#9E726F", "#D6B2B1", "#B9C0C9", "#9F8A78", "#F3AB60"]) fig, ax = plt.subplots(figsize = (6,4)) sns.boxplot(data = data, y = data["Income"], color = "orange") sns.stripplot(data = data, y = data["Income"], size = 2, linewidth=0) plt.title("Boxplot - Income") for i in ['bottom', 'left']: ax.spines[i].set_color('black') ax.spines[i].set_linewidth(1.5) right_side = ax.spines["right"] right_side.set_visible(False) top_side = ax.spines["top"] top_side.set_visible(False) ax.set_axisbelow(True) ax.grid(color='gray', linewidth=1, axis='y', alpha=0.4) plt.show()

data.loc[(data["Income"].isnull()==True), "Income"]=data["Income"].mean()

Al analizar el boxplot de la variable "Income" tengo que la presencia de outliers es mínima y gran parte de los datos (aproximadamente, 95%) se encuentra dentro del rango intercuartílico, por ello, tomaré la decisión de introducir la media de los valores de la variable "Income" en los valores nulos del mismo.

ratio_data = ((data.isnull().sum()/len(data))*100).sort_values(ascending = False) print("Porcentaje de valores nulos en el dataframe: (%)") print(ratio_data)

# Regla IQR para identificar los valores atípicos q1 = np.quantile(data["Income"], 0.25) q2 = np.median(data["Income"]) q3 = np.quantile(data["Income"], 0.75) iqr = q3 - q1 upper_bound = q3 + (1.5*iqr) lower_bound = q1 - (1.5*iqr) print(iqr, upper_bound, lower_bound) outliers = data["Income"][(data["Income"] <= lower_bound) | (data["Income"] >= upper_bound)] print(f'Los siguientes valores son outliers: \n{outliers}')

Transformación de variable object a datetime

Observé que la variable "Dt_Customer" es del tipo "object", sin embargo, la variable hace referencia a la fecha en el que el cliente fue dado de alta por la empresa, entonces, el tipo debería ser "datetime".

data["Dt_Customer"] = pd.to_datetime(data["Dt_Customer"])

data.info()

Análisis de variables categóricas

print(f'La variable "Education" es categórica y sus clases son {np.unique(data["Education"])}. Tiene {len(np.unique(data["Education"]))} clases.') print(f'La variable "Marital_Status" es categórica y sus clases son {np.unique(data["Marital_Status"])}. Tiene {len(np.unique(data["Marital_Status"]))} clases.')

# Conteo del número de consumidores que pertenecen a cada clase de la variable "Education". count_education = pd.DataFrame(data["Education"].value_counts()).reset_index() count_education = count_education.rename(columns = {"index":"Type_Education"}) count_education

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) fig, ax = plt.subplots(figsize=(6,6), dpi = 80) x = np.array(count_education["Type_Education"]) y = count_education["Education"] plt.title("Consumer Education Level") ax = sns.barplot(data = count_education, x = "Type_Education", y = "Education", alpha = 0.8) ax.bar_label(ax.containers[0]) plt.show()

data["Education"] = data["Education"].astype("category")

# Conteo del número de consumidores que pertenecen a cada clase de la variable "Marital_Status". count_marital_status = pd.DataFrame(data["Marital_Status"].value_counts()).reset_index() count_marital_status = count_marital_status.rename(columns = {"index":"Status"}) count_marital_status

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) fig, ax = plt.subplots(figsize=(6,6), dpi = 80) x = np.array(count_marital_status["Status"]) y = count_marital_status["Marital_Status"] plt.title("Consumer Marital Status") ax = sns.barplot(data = count_marital_status, x = "Status", y = "Marital_Status", palette = "coolwarm") ax.bar_label(ax.containers[0]) plt.xticks(rotation=45) plt.show()

data["Marital_Status"] = data["Marital_Status"].astype("category")

data.info()

Tratamiento de variables categóricas

from sklearn.preprocessing import OrdinalEncoder encoder_education = OrdinalEncoder(categories = [['2n Cycle', 'Basic', 'Graduation', 'Master', 'PhD']]) encoder_education.fit(data[["Education"]]) data["Education_Encoder"] = encoder_education.transform(data[["Education"]])

data[["Education", "Education_Encoder"]].head()

data["Marital_Status_Processed"] = data["Marital_Status"].replace({ "Single":"Alone", "Absurd":"Alone", "Alone":"Alone", "YOLO":"Alone", "Together":"Together", "Married":"Married", "Widow":"Widow", "Divorced":"Divorced" })

from sklearn.preprocessing import OrdinalEncoder encoder_education = OrdinalEncoder(categories = [["Alone", "Together", "Married", "Widow", "Divorced"]]) encoder_education.fit(data[["Marital_Status_Processed"]]) data["Marital_Status_Encoder"] = encoder_education.transform(data[["Marital_Status_Processed"]])

data.head()

Análisis Exploratorio de Datos (EDA)

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) year = 2022 - data["Year_Birth"] fig, ax = plt.subplots(1, 2, figsize=(18,8), dpi = 80) plt.suptitle("Analysis of variable 'Year'", size = 20) ax[0].hist(x = data["Year_Birth"]) ax[0].set_title("Histogram of the year the customer was born", size = 18) ax[0].set(xlabel="Year the customer was born", ylabel="Count") ax[1].hist(x = year, color = "orange", alpha = 0.8) ax[1].set_title("Histogram of customer years", size = 18) ax[1].set(xlabel="Customer years", ylabel="Count") plt.show()

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) year = 2022 - data["Year_Birth"] fig, ax = plt.subplots(figsize = (8,6), dpi = 70) ax.scatter(x = year, y = data["Income"], color = "brown", alpha = 0.8) plt.title("Scatterplot between income and consumer age") plt.xlabel("Consumer age") plt.ylabel("Income") plt.axhline(y = 600000, color = "orange") plt.axvline(x = 100, color = "orange") ax.annotate(r'$Outliers_{Age}$', xy = (120, 100000), xytext = (120,300000), arrowprops = dict(facecolor = "orange", shrink = 0.05)) ax.annotate(r'$Outlier_{Income}$', xy = (50,666000), xytext = (70, 658000), arrowprops = dict(facecolor = "orange", shrink = 0.05)) plt.show()

y_dummy = np.array([data["AcceptedCmp1"].value_counts()[1], data["AcceptedCmp2"].value_counts()[1], data["AcceptedCmp3"].value_counts()[1], data["AcceptedCmp4"].value_counts()[1], data["AcceptedCmp5"].value_counts()[1], data["Response"].value_counts()[1]]) y_dummy x_dummy = ["AcceptedCmp1", "AcceptedCmp2", "AcceptedCmp3", "AcceptedCmp4", "AcceptedCmp5", "Response"] print(x_dummy) print(y_dummy)

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) fig, ax = plt.subplots(figsize = (10,8), dpi = 60) ax = sns.barplot(x = x_dummy, y = y_dummy) ax.bar_label(ax.containers[0]) plt.title("Number of consumers who accepted the offer based\non the number of the campaign", size = 18) plt.xlabel("Promotions") plt.ylabel("Count") plt.show()

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) spend_total = data["MntWines"]+data["MntFruits"]+data["MntMeatProducts"]+data["MntFishProducts"]+data["MntSweetProducts"]+data["MntGoldProds"] income = data["Income"] fig, ax = plt.subplots(figsize = (8,6), dpi = 70) ax = sns.scatterplot(x = income, y = spend_total, color = "orange") plt.ylabel("Total consumer spending") plt.xlabel("Consumer income") plt.title("Scatterplot between consumer income\nand total consumer spending", size = 18) plt.show()

sns.set_theme(style="whitegrid") sns.catplot(data = data, x = "Education", y = "Income", hue = "Marital_Status_Processed", kind = "bar", palette = "dark", alpha = 0.7) plt.title("Impact of education leven on income as\na function of marital status") plt.show()

complain_labels = ["No complain", "Complain"] complain_values = [data["Complain"].value_counts()[0], data["Complain"].value_counts()[1]] fig, ax = plt.subplots(figsize = (8,6), dpi = 70) sns.barplot(x = complain_labels, y = complain_values) ax.bar_label(ax.containers[0]) plt.title("Number of consumers who complained\nin the last 2 years", size = 18) plt.xlabel("Complain") plt.ylabel("Count") plt.show()

y_values_type_spend = [ data["MntWines"].sum(), data["MntFruits"].sum(), data["MntMeatProducts"].sum(), data["MntFishProducts"].sum(), data["MntSweetProducts"].sum(), data["MntGoldProds"].sum() ] x_values_type_spend = [ "MntWines", "MntFruits", "MntMeatProducts", "MntFishProducts", "MntSweetProducts", "MntGoldProds" ]

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) fig, ax = plt.subplots(figsize = (8,8), dpi = 70) sns.barplot(x = y_values_type_spend, y = x_values_type_spend) plt.title("What is the product that generated the most income\nin the last 2 years?", size = 18) plt.xlabel("Consumer spending") plt.ylabel("Type of products") ax.bar_label(ax.containers[0]) plt.show()

sns.set_theme(style = "whitegrid") spend_total = data["MntWines"]+data["MntFruits"]+data["MntMeatProducts"]+data["MntFishProducts"]+data["MntSweetProducts"]+data["MntGoldProds"] deal_purchases = data["NumDealsPurchases"] fig, ax = plt.subplots(figsize = (8,6), dpi = 70) ax = sns.scatterplot(x = spend_total, y = deal_purchases) plt.title("Scatterplot between the total spending and\nnumber of deals purchases", size = 18) plt.xlabel("Total spending") plt.show()

y_label_place = ["NumWebPurchases", "NumCatalogPurchases", "NumStorePurchases", "NumWebVisitsMonth"] x_label_place = [ data["NumWebPurchases"].sum(), data["NumCatalogPurchases"].sum(), data["NumStorePurchases"].sum(), data["NumWebVisitsMonth"].sum() ] sum_place = np.array(x_label_place).sum() x_label_place_per = [ data["NumWebPurchases"].sum()/sum_place*100, data["NumCatalogPurchases"].sum()/sum_place*100, data["NumStorePurchases"].sum()/sum_place*100, data["NumWebVisitsMonth"].sum()/sum_place*100, ] fig, ax = plt.subplots(2, 1, figsize = (12,10), dpi = 65) sns.barplot(x = y_label_place, y = x_label_place, ax = ax[0]) ax[0].bar_label(ax[0].containers[0]) ax[0].set_title("Where do consumers buy your products?\n", size = 18) ax[0].set(xlabel = "Places where consumers shop", ylabel = "Number of times consumers bought") sns.barplot(x = y_label_place, y = x_label_place_per, ax = ax[1]) ax[1].bar_label(ax[1].containers[0]) ax[1].set(xlabel = "Places where consumers shop", ylabel = "Percentage of times consumers bought (%)") fig.subplots_adjust(hspace = 0.2) plt.show()

Procesamiento del dataset

data_processed = data.copy() data_processed

data_processed.drop(["ID", "Education", "Marital_Status", "Marital_Status_Processed", "Dt_Customer", "AcceptedCmp3", "AcceptedCmp4", "AcceptedCmp5", "AcceptedCmp1", "AcceptedCmp2", "Complain", "Response"], axis = 1, inplace = True) data_processed

# Sumar las compras realizadas en cada categoría de los productos data_processed["Purchases"] = data["MntWines"]+data["MntFruits"]+data["MntMeatProducts"]+data["MntFishProducts"]+data["MntSweetProducts"]+data["MntGoldProds"] data_processed.drop(["MntWines", "MntFruits", "MntMeatProducts", "MntFishProducts", "MntSweetProducts", "MntGoldProds"], axis = 1, inplace = True) data_processed.head()

# Eliminación de dos columnas adicionales que no serán necesarios para el análisis. data_processed.drop(["Z_CostContact", "Z_Revenue"], axis = 1, inplace = True) # Creación de una columna que especifique la edad del consumidor al año 2022. data_processed["Years"] = 2022 - data_processed["Year_Birth"] data_processed.drop(["Year_Birth"], axis = 1, inplace = True) data_processed.head()

data_processed["Children"] = data_processed["Teenhome"] + data_processed["Kidhome"] data_processed.drop(["Teenhome", "Kidhome"], axis = 1, inplace = True) data_processed.head()

Eliminar valores outliers de la variable "Years"

data_processed = data_processed.drop(data_processed[data_processed["Years"]>100].index) data_processed.head()

data_processed = data_processed.drop(data_processed[data_processed["Income"]>600000].index) data_processed.head()

data_processed.describe()

Análisis de correlación

fig = plt.figure(figsize = (15,8)) sns.heatmap(data_processed.corr(), annot = True, cmap = "coolwarm") plt.title("Heatmap of dataset variables\n", size = 18) plt.show()

Escalamiento de los datos

from sklearn.preprocessing import StandardScaler scaler = StandardScaler() scaler.fit(data_processed) data_processed_scaled = pd.DataFrame(scaler.transform(data_processed), columns = data_processed.columns) data_processed_scaled

Reducción de dimensionalidad (PCA)

from sklearn.decomposition import PCA pca = PCA() pca.fit_transform(data_processed_scaled) data_processed_scaled_pca = pca.transform(data_processed_scaled) data_processed_scaled_pca

var = pca.explained_variance_ratio_ print(var)

sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"}) cum_var = np.cumsum(np.round(var, decimals=4)*100) fig, ax = plt.subplots(figsize=(8,6), dpi = 80) plt.title("Cumulative Variation Graph") plt.plot(cum_var, "b-*") plt.ylabel("Cumulative Variation") plt.xlabel("Dimensions PCA") plt.axvline(x=7, color = "r") for i in ['bottom', 'left']: ax.spines[i].set_color('black') ax.spines[i].set_linewidth(1.5) right_side = ax.spines["right"] right_side.set_visible(False) top_side = ax.spines["top"] top_side.set_visible(False) ax.set_axisbelow(True) ax.grid(color='gray', linewidth=1, axis='y', alpha=0.4) plt.show()

data_processed_scaled_pca_dim = pd.DataFrame(data_processed_scaled_pca) data_processed_scaled_pca_dim.drop([8,9,10,11], axis = 1, inplace = True) data_processed_scaled_pca_dim

data_processed_scaled_pca_dim.rename(columns = {0:"Income", 1:"Recency", 2:"NumDealsPurchases", 3:"NumWebPurchases", 4:"NumCatalogPurchases", 5:"NumStorePurchases", 6:"NumWebVisitsMonth", 7:"Education_Encoder"})

data_processed_scaled.columns

Importar librerías esenciales para clustering

from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN from sklearn.metrics import silhouette_score from scipy.cluster.hierarchy import dendrogram, linkage

Algoritmo de Clustering K-means

from sklearn.metrics.cluster import silhouette_score sum_of_squared_distances = [] silhouette_scores = [] K = range(2,10) for k in K: km = KMeans(n_clusters = k) y_predict_km = km.fit_predict(data_processed_scaled_pca_dim) sum_of_squared_distances.append(km.inertia_) silhouette_scores.append(silhouette_score(data_processed_scaled_pca_dim, y_predict_km))

fig, ax = plt.subplots(figsize = (8,6), dpi = 60) plt.plot(K, sum_of_squared_distances, "ro-") plt.ylabel("Inertia") plt.xlabel("k") plt.show()

fig, ax = plt.subplots(figsize = (8,6), dpi = 60) plt.plot(K, silhouette_scores, "bo-") plt.ylabel("Silhouette scores") plt.xlabel("k") plt.show()

km = KMeans(n_clusters = 2) y_predict_km = km.fit_predict(data_processed_scaled_pca_dim) print(silhouette_score(data_processed_scaled_pca_dim, y_predict_km)) data_processed["K-Means-PCA"] = y_predict_km

Algoritmo de DBSCAN

from sklearn.neighbors import NearestNeighbors neighbors = NearestNeighbors(n_neighbors=2) neighbors_fit = neighbors.fit(data_processed_scaled_pca_dim) distances, indexs = neighbors_fit.kneighbors(data_processed_scaled_pca_dim)

distances = np.sort(distances, axis = 0) distances = distances[:,1]

fig = plt.figure(figsize = (8,6), dpi = 60) plt.plot(distances) plt.show()

eps_values = np.arange(0.1, 2.5, 0.10) min_samples = np.arange(2,10)

from itertools import product dbscan_paramns = list(product(eps_values, min_samples)) n_of_clusters = [] sil_score = [] for p in dbscan_paramns: y_predict_dbscan = DBSCAN(eps = p[0], min_samples = p[1]).fit_predict(data_processed_scaled_pca_dim) try: sil_score.append(silhouette_score(data_processed_scaled_pca_dim, y_predict_dbscan)) except: sil_score.append(0) n_of_clusters.append(len(np.unique(y_predict_dbscan)))

df_param_tunning = pd.DataFrame.from_records(dbscan_paramns, columns = ["Eps", "Min_samples"]) df_param_tunning["sil_score"] = sil_score df_param_tunning["n_cluster"] = n_of_clusters

pivot_1 = pd.pivot_table(df_param_tunning, values = "sil_score", columns = "Eps", index = "Min_samples") pivot_2 = pd.pivot_table(df_param_tunning, values = "n_cluster", columns = "Eps", index = "Min_samples")

fig, ax=plt.subplots(figsize=(18,6)) sns.heatmap(pivot_1, annot=True, annot_kws={"size":10}, cmap="coolwarm", ax=ax) plt.show()

fig, ax=plt.subplots(figsize=(18,6)) sns.heatmap(pivot_2, annot=True, annot_kws={"size":10}, cmap="coolwarm", ax=ax) plt.show()

dbscan_train = DBSCAN(eps=2.4,min_samples=2) y_dbscan_1 = dbscan_train.fit_predict(data_processed_scaled_pca_dim) print(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_1)) data_processed["dbscan_pca_1"] = y_dbscan_1

dbscan_train = DBSCAN(eps=2.4,min_samples=3) y_dbscan_2 = dbscan_train.fit_predict(data_processed_scaled_pca_dim) print(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_2)) data_processed["dbscan_pca_2"] = y_dbscan_2

data_processed

Visualización de los clústeres, mediante el algoritmo de DBSCAN

sns.set_theme(style = "whitegrid") fig, axs = plt.subplots(2, 2, figsize = (15,10), dpi = 65) plt.suptitle(f'Cluster analysis through the K-means algorithm\nSilhouette Score K-means: {round(silhouette_score(data_processed_scaled_pca_dim, y_predict_km),4)} and K = 2', size = 18) sns.scatterplot(data = data_processed, x = "Income", y = "Purchases", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[0,0]) sns.scatterplot(data = data_processed, x = "Years", y = "Income", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[0,1]) sns.scatterplot(data = data_processed, x = "Education_Encoder", y = "Income", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[1,0]) sns.scatterplot(data = data_processed, x = "Marital_Status_Encoder", y = "Income", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[1,1]) fig.subplots_adjust(wspace = 0.3, hspace = 0.3) plt.show()

sns.set_theme(style = "whitegrid") fig, axs = plt.subplots(2, 2, figsize = (15,10), dpi = 65) plt.suptitle(f'Cluster analysis through the DBSCAN algorithm\nSilhouette Score DBSCAN: {round(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_1),4)} and K = 4 (-1 is noise)', size = 18) sns.scatterplot(data = data_processed, x = "Income", y = "Purchases", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[0,0]) sns.scatterplot(data = data_processed, x = "Years", y = "Income", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[0,1]) sns.scatterplot(data = data_processed, x = "Education_Encoder", y = "Income", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[1,0]) sns.scatterplot(data = data_processed, x = "Marital_Status_Encoder", y = "Income", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[1,1]) fig.subplots_adjust(wspace = 0.3, hspace = 0.3) plt.show()

sns.set_theme(style = "whitegrid") fig, axs = plt.subplots(2, 2, figsize = (15,10), dpi = 65) plt.suptitle(f'Cluster analysis through the DBSCAN algorithm\nSilhouette Score DBSCAN: {round(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_2),4)} and K = 2 (-1 is noise)', size = 18) sns.scatterplot(data = data_processed, x = "Income", y = "Purchases", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[0,0]) sns.scatterplot(data = data_processed, x = "Years", y = "Income", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[0,1]) sns.scatterplot(data = data_processed, x = "Education_Encoder", y = "Income", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[1,0]) sns.scatterplot(data = data_processed, x = "Marital_Status_Encoder", y = "Income", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[1,1]) fig.subplots_adjust(wspace = 0.3, hspace = 0.3) plt.show()