Análisis de la Personalidad del Consumidor
Fuente y diccionario de datos: https://www.kaggle.com/datasets/imakash3011/customer-personality-analysis
Autor: Urdanegui Bisalaya, Sebastian Marat | Data Scientist | Data Analyst 📊
Librerías a importar
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib.colors import ListedColormap
import seaborn as sns
Cargar el dataset
data = pd.read_csv("marketing_campaign.csv", sep = "\t")
data.head()
print(f'La información del dataset se encuentra dividida en {data.shape[0]} filas y {data.shape[1]} columnas.')
data.columns
Descripción estadística del dataset
data.describe()
Limpieza de los datos
data.info()
Análisis de valores nulos en el dataframe
print("Valores nulos en el dataframe:")
print(data.isnull().sum().sort_values(ascending = False))
ratio_data = ((data.isnull().sum()/len(data))*100).sort_values(ascending = False)
print("Porcentaje de valores nulos en el dataframe: (%)")
print(ratio_data)
Solución a los valores nulos del dataframe
Análisis de distribución del feature que presenta valores nulos
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
cmap = colors.ListedColormap(["#682F2F", "#9E726F", "#D6B2B1", "#B9C0C9", "#9F8A78", "#F3AB60"])
fig, ax = plt.subplots(figsize = (6,4))
sns.boxplot(data = data, y = data["Income"], color = "orange")
sns.stripplot(data = data, y = data["Income"], size = 2, linewidth=0)
plt.title("Boxplot - Income")
for i in ['bottom', 'left']:
ax.spines[i].set_color('black')
ax.spines[i].set_linewidth(1.5)
right_side = ax.spines["right"]
right_side.set_visible(False)
top_side = ax.spines["top"]
top_side.set_visible(False)
ax.set_axisbelow(True)
ax.grid(color='gray', linewidth=1, axis='y', alpha=0.4)
plt.show()
data.loc[(data["Income"].isnull()==True), "Income"]=data["Income"].mean()
Al analizar el boxplot de la variable "Income" tengo que la presencia de outliers es mínima y gran parte de los datos (aproximadamente, 95%) se encuentra dentro del rango intercuartílico, por ello, tomaré la decisión de introducir la media de los valores de la variable "Income" en los valores nulos del mismo.
ratio_data = ((data.isnull().sum()/len(data))*100).sort_values(ascending = False)
print("Porcentaje de valores nulos en el dataframe: (%)")
print(ratio_data)
# Regla IQR para identificar los valores atípicos
q1 = np.quantile(data["Income"], 0.25)
q2 = np.median(data["Income"])
q3 = np.quantile(data["Income"], 0.75)
iqr = q3 - q1
upper_bound = q3 + (1.5*iqr)
lower_bound = q1 - (1.5*iqr)
print(iqr, upper_bound, lower_bound)
outliers = data["Income"][(data["Income"] <= lower_bound) | (data["Income"] >= upper_bound)]
print(f'Los siguientes valores son outliers: \n{outliers}')
Transformación de variable object a datetime
Observé que la variable "Dt_Customer" es del tipo "object", sin embargo, la variable hace referencia a la fecha en el que el cliente fue dado de alta por la empresa, entonces, el tipo debería ser "datetime".
data["Dt_Customer"] = pd.to_datetime(data["Dt_Customer"])
data.info()
Análisis de variables categóricas
print(f'La variable "Education" es categórica y sus clases son {np.unique(data["Education"])}. Tiene {len(np.unique(data["Education"]))} clases.')
print(f'La variable "Marital_Status" es categórica y sus clases son {np.unique(data["Marital_Status"])}. Tiene {len(np.unique(data["Marital_Status"]))} clases.')
# Conteo del número de consumidores que pertenecen a cada clase de la variable "Education".
count_education = pd.DataFrame(data["Education"].value_counts()).reset_index()
count_education = count_education.rename(columns = {"index":"Type_Education"})
count_education
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
fig, ax = plt.subplots(figsize=(6,6), dpi = 80)
x = np.array(count_education["Type_Education"])
y = count_education["Education"]
plt.title("Consumer Education Level")
ax = sns.barplot(data = count_education, x = "Type_Education", y = "Education", alpha = 0.8)
ax.bar_label(ax.containers[0])
plt.show()
data["Education"] = data["Education"].astype("category")
# Conteo del número de consumidores que pertenecen a cada clase de la variable "Marital_Status".
count_marital_status = pd.DataFrame(data["Marital_Status"].value_counts()).reset_index()
count_marital_status = count_marital_status.rename(columns = {"index":"Status"})
count_marital_status
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
fig, ax = plt.subplots(figsize=(6,6), dpi = 80)
x = np.array(count_marital_status["Status"])
y = count_marital_status["Marital_Status"]
plt.title("Consumer Marital Status")
ax = sns.barplot(data = count_marital_status, x = "Status", y = "Marital_Status", palette = "coolwarm")
ax.bar_label(ax.containers[0])
plt.xticks(rotation=45)
plt.show()
data["Marital_Status"] = data["Marital_Status"].astype("category")
data.info()
Tratamiento de variables categóricas
from sklearn.preprocessing import OrdinalEncoder
encoder_education = OrdinalEncoder(categories = [['2n Cycle', 'Basic', 'Graduation', 'Master', 'PhD']])
encoder_education.fit(data[["Education"]])
data["Education_Encoder"] = encoder_education.transform(data[["Education"]])
data[["Education", "Education_Encoder"]].head()
data["Marital_Status_Processed"] = data["Marital_Status"].replace({
"Single":"Alone",
"Absurd":"Alone",
"Alone":"Alone",
"YOLO":"Alone",
"Together":"Together",
"Married":"Married",
"Widow":"Widow",
"Divorced":"Divorced"
})
from sklearn.preprocessing import OrdinalEncoder
encoder_education = OrdinalEncoder(categories = [["Alone", "Together", "Married", "Widow", "Divorced"]])
encoder_education.fit(data[["Marital_Status_Processed"]])
data["Marital_Status_Encoder"] = encoder_education.transform(data[["Marital_Status_Processed"]])
data.head()
Análisis Exploratorio de Datos (EDA)
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
year = 2022 - data["Year_Birth"]
fig, ax = plt.subplots(1, 2, figsize=(18,8), dpi = 80)
plt.suptitle("Analysis of variable 'Year'", size = 20)
ax[0].hist(x = data["Year_Birth"])
ax[0].set_title("Histogram of the year the customer was born", size = 18)
ax[0].set(xlabel="Year the customer was born", ylabel="Count")
ax[1].hist(x = year, color = "orange", alpha = 0.8)
ax[1].set_title("Histogram of customer years", size = 18)
ax[1].set(xlabel="Customer years", ylabel="Count")
plt.show()
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
year = 2022 - data["Year_Birth"]
fig, ax = plt.subplots(figsize = (8,6), dpi = 70)
ax.scatter(x = year, y = data["Income"], color = "brown", alpha = 0.8)
plt.title("Scatterplot between income and consumer age")
plt.xlabel("Consumer age")
plt.ylabel("Income")
plt.axhline(y = 600000, color = "orange")
plt.axvline(x = 100, color = "orange")
ax.annotate(r'$Outliers_{Age}$', xy = (120, 100000), xytext = (120,300000), arrowprops = dict(facecolor = "orange", shrink = 0.05))
ax.annotate(r'$Outlier_{Income}$', xy = (50,666000), xytext = (70, 658000), arrowprops = dict(facecolor = "orange", shrink = 0.05))
plt.show()
y_dummy = np.array([data["AcceptedCmp1"].value_counts()[1],
data["AcceptedCmp2"].value_counts()[1],
data["AcceptedCmp3"].value_counts()[1],
data["AcceptedCmp4"].value_counts()[1],
data["AcceptedCmp5"].value_counts()[1],
data["Response"].value_counts()[1]])
y_dummy
x_dummy = ["AcceptedCmp1", "AcceptedCmp2", "AcceptedCmp3", "AcceptedCmp4", "AcceptedCmp5", "Response"]
print(x_dummy)
print(y_dummy)
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
fig, ax = plt.subplots(figsize = (10,8), dpi = 60)
ax = sns.barplot(x = x_dummy, y = y_dummy)
ax.bar_label(ax.containers[0])
plt.title("Number of consumers who accepted the offer based\non the number of the campaign", size = 18)
plt.xlabel("Promotions")
plt.ylabel("Count")
plt.show()
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
spend_total = data["MntWines"]+data["MntFruits"]+data["MntMeatProducts"]+data["MntFishProducts"]+data["MntSweetProducts"]+data["MntGoldProds"]
income = data["Income"]
fig, ax = plt.subplots(figsize = (8,6), dpi = 70)
ax = sns.scatterplot(x = income, y = spend_total, color = "orange")
plt.ylabel("Total consumer spending")
plt.xlabel("Consumer income")
plt.title("Scatterplot between consumer income\nand total consumer spending", size = 18)
plt.show()
sns.set_theme(style="whitegrid")
sns.catplot(data = data, x = "Education", y = "Income", hue = "Marital_Status_Processed",
kind = "bar", palette = "dark", alpha = 0.7)
plt.title("Impact of education leven on income as\na function of marital status")
plt.show()
complain_labels = ["No complain", "Complain"]
complain_values = [data["Complain"].value_counts()[0], data["Complain"].value_counts()[1]]
fig, ax = plt.subplots(figsize = (8,6), dpi = 70)
sns.barplot(x = complain_labels, y = complain_values)
ax.bar_label(ax.containers[0])
plt.title("Number of consumers who complained\nin the last 2 years", size = 18)
plt.xlabel("Complain")
plt.ylabel("Count")
plt.show()
y_values_type_spend = [
data["MntWines"].sum(),
data["MntFruits"].sum(),
data["MntMeatProducts"].sum(),
data["MntFishProducts"].sum(),
data["MntSweetProducts"].sum(),
data["MntGoldProds"].sum()
]
x_values_type_spend = [
"MntWines", "MntFruits", "MntMeatProducts", "MntFishProducts", "MntSweetProducts", "MntGoldProds"
]
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
fig, ax = plt.subplots(figsize = (8,8), dpi = 70)
sns.barplot(x = y_values_type_spend, y = x_values_type_spend)
plt.title("What is the product that generated the most income\nin the last 2 years?", size = 18)
plt.xlabel("Consumer spending")
plt.ylabel("Type of products")
ax.bar_label(ax.containers[0])
plt.show()
sns.set_theme(style = "whitegrid")
spend_total = data["MntWines"]+data["MntFruits"]+data["MntMeatProducts"]+data["MntFishProducts"]+data["MntSweetProducts"]+data["MntGoldProds"]
deal_purchases = data["NumDealsPurchases"]
fig, ax = plt.subplots(figsize = (8,6), dpi = 70)
ax = sns.scatterplot(x = spend_total, y = deal_purchases)
plt.title("Scatterplot between the total spending and\nnumber of deals purchases", size = 18)
plt.xlabel("Total spending")
plt.show()
y_label_place = ["NumWebPurchases", "NumCatalogPurchases", "NumStorePurchases", "NumWebVisitsMonth"]
x_label_place = [
data["NumWebPurchases"].sum(),
data["NumCatalogPurchases"].sum(),
data["NumStorePurchases"].sum(),
data["NumWebVisitsMonth"].sum()
]
sum_place = np.array(x_label_place).sum()
x_label_place_per = [
data["NumWebPurchases"].sum()/sum_place*100,
data["NumCatalogPurchases"].sum()/sum_place*100,
data["NumStorePurchases"].sum()/sum_place*100,
data["NumWebVisitsMonth"].sum()/sum_place*100,
]
fig, ax = plt.subplots(2, 1, figsize = (12,10), dpi = 65)
sns.barplot(x = y_label_place, y = x_label_place, ax = ax[0])
ax[0].bar_label(ax[0].containers[0])
ax[0].set_title("Where do consumers buy your products?\n", size = 18)
ax[0].set(xlabel = "Places where consumers shop", ylabel = "Number of times consumers bought")
sns.barplot(x = y_label_place, y = x_label_place_per, ax = ax[1])
ax[1].bar_label(ax[1].containers[0])
ax[1].set(xlabel = "Places where consumers shop", ylabel = "Percentage of times consumers bought (%)")
fig.subplots_adjust(hspace = 0.2)
plt.show()
Procesamiento del dataset
data_processed = data.copy()
data_processed
data_processed.drop(["ID", "Education", "Marital_Status", "Marital_Status_Processed", "Dt_Customer",
"AcceptedCmp3", "AcceptedCmp4", "AcceptedCmp5", "AcceptedCmp1", "AcceptedCmp2",
"Complain", "Response"], axis = 1, inplace = True)
data_processed
# Sumar las compras realizadas en cada categoría de los productos
data_processed["Purchases"] = data["MntWines"]+data["MntFruits"]+data["MntMeatProducts"]+data["MntFishProducts"]+data["MntSweetProducts"]+data["MntGoldProds"]
data_processed.drop(["MntWines", "MntFruits", "MntMeatProducts", "MntFishProducts", "MntSweetProducts", "MntGoldProds"], axis = 1, inplace = True)
data_processed.head()
# Eliminación de dos columnas adicionales que no serán necesarios para el análisis.
data_processed.drop(["Z_CostContact", "Z_Revenue"], axis = 1, inplace = True)
# Creación de una columna que especifique la edad del consumidor al año 2022.
data_processed["Years"] = 2022 - data_processed["Year_Birth"]
data_processed.drop(["Year_Birth"], axis = 1, inplace = True)
data_processed.head()
data_processed["Children"] = data_processed["Teenhome"] + data_processed["Kidhome"]
data_processed.drop(["Teenhome", "Kidhome"], axis = 1, inplace = True)
data_processed.head()
Eliminar valores outliers de la variable "Years"
data_processed = data_processed.drop(data_processed[data_processed["Years"]>100].index)
data_processed.head()
data_processed = data_processed.drop(data_processed[data_processed["Income"]>600000].index)
data_processed.head()
data_processed.describe()
Análisis de correlación
fig = plt.figure(figsize = (15,8))
sns.heatmap(data_processed.corr(), annot = True, cmap = "coolwarm")
plt.title("Heatmap of dataset variables\n", size = 18)
plt.show()
Escalamiento de los datos
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(data_processed)
data_processed_scaled = pd.DataFrame(scaler.transform(data_processed), columns = data_processed.columns)
data_processed_scaled
Reducción de dimensionalidad (PCA)
from sklearn.decomposition import PCA
pca = PCA()
pca.fit_transform(data_processed_scaled)
data_processed_scaled_pca = pca.transform(data_processed_scaled)
data_processed_scaled_pca
var = pca.explained_variance_ratio_
print(var)
sns.set(rc={"axes.facecolor":"#FFF9ED", "figure.facecolor":"#FFF9ED"})
cum_var = np.cumsum(np.round(var, decimals=4)*100)
fig, ax = plt.subplots(figsize=(8,6), dpi = 80)
plt.title("Cumulative Variation Graph")
plt.plot(cum_var, "b-*")
plt.ylabel("Cumulative Variation")
plt.xlabel("Dimensions PCA")
plt.axvline(x=7, color = "r")
for i in ['bottom', 'left']:
ax.spines[i].set_color('black')
ax.spines[i].set_linewidth(1.5)
right_side = ax.spines["right"]
right_side.set_visible(False)
top_side = ax.spines["top"]
top_side.set_visible(False)
ax.set_axisbelow(True)
ax.grid(color='gray', linewidth=1, axis='y', alpha=0.4)
plt.show()
data_processed_scaled_pca_dim = pd.DataFrame(data_processed_scaled_pca)
data_processed_scaled_pca_dim.drop([8,9,10,11], axis = 1, inplace = True)
data_processed_scaled_pca_dim
data_processed_scaled_pca_dim.rename(columns = {0:"Income", 1:"Recency", 2:"NumDealsPurchases", 3:"NumWebPurchases", 4:"NumCatalogPurchases",
5:"NumStorePurchases", 6:"NumWebVisitsMonth", 7:"Education_Encoder"})
data_processed_scaled.columns
Importar librerías esenciales para clustering
from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN
from sklearn.metrics import silhouette_score
from scipy.cluster.hierarchy import dendrogram, linkage
Algoritmo de Clustering K-means
from sklearn.metrics.cluster import silhouette_score
sum_of_squared_distances = []
silhouette_scores = []
K = range(2,10)
for k in K:
km = KMeans(n_clusters = k)
y_predict_km = km.fit_predict(data_processed_scaled_pca_dim)
sum_of_squared_distances.append(km.inertia_)
silhouette_scores.append(silhouette_score(data_processed_scaled_pca_dim, y_predict_km))
fig, ax = plt.subplots(figsize = (8,6), dpi = 60)
plt.plot(K, sum_of_squared_distances, "ro-")
plt.ylabel("Inertia")
plt.xlabel("k")
plt.show()
fig, ax = plt.subplots(figsize = (8,6), dpi = 60)
plt.plot(K, silhouette_scores, "bo-")
plt.ylabel("Silhouette scores")
plt.xlabel("k")
plt.show()
km = KMeans(n_clusters = 2)
y_predict_km = km.fit_predict(data_processed_scaled_pca_dim)
print(silhouette_score(data_processed_scaled_pca_dim, y_predict_km))
data_processed["K-Means-PCA"] = y_predict_km
Algoritmo de DBSCAN
from sklearn.neighbors import NearestNeighbors
neighbors = NearestNeighbors(n_neighbors=2)
neighbors_fit = neighbors.fit(data_processed_scaled_pca_dim)
distances, indexs = neighbors_fit.kneighbors(data_processed_scaled_pca_dim)
distances = np.sort(distances, axis = 0)
distances = distances[:,1]
fig = plt.figure(figsize = (8,6), dpi = 60)
plt.plot(distances)
plt.show()
eps_values = np.arange(0.1, 2.5, 0.10)
min_samples = np.arange(2,10)
from itertools import product
dbscan_paramns = list(product(eps_values, min_samples))
n_of_clusters = []
sil_score = []
for p in dbscan_paramns:
y_predict_dbscan = DBSCAN(eps = p[0], min_samples = p[1]).fit_predict(data_processed_scaled_pca_dim)
try:
sil_score.append(silhouette_score(data_processed_scaled_pca_dim, y_predict_dbscan))
except:
sil_score.append(0)
n_of_clusters.append(len(np.unique(y_predict_dbscan)))
df_param_tunning = pd.DataFrame.from_records(dbscan_paramns, columns = ["Eps", "Min_samples"])
df_param_tunning["sil_score"] = sil_score
df_param_tunning["n_cluster"] = n_of_clusters
pivot_1 = pd.pivot_table(df_param_tunning, values = "sil_score", columns = "Eps", index = "Min_samples")
pivot_2 = pd.pivot_table(df_param_tunning, values = "n_cluster", columns = "Eps", index = "Min_samples")
fig, ax=plt.subplots(figsize=(18,6))
sns.heatmap(pivot_1, annot=True, annot_kws={"size":10}, cmap="coolwarm", ax=ax)
plt.show()
fig, ax=plt.subplots(figsize=(18,6))
sns.heatmap(pivot_2, annot=True, annot_kws={"size":10}, cmap="coolwarm", ax=ax)
plt.show()
dbscan_train = DBSCAN(eps=2.4,min_samples=2)
y_dbscan_1 = dbscan_train.fit_predict(data_processed_scaled_pca_dim)
print(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_1))
data_processed["dbscan_pca_1"] = y_dbscan_1
dbscan_train = DBSCAN(eps=2.4,min_samples=3)
y_dbscan_2 = dbscan_train.fit_predict(data_processed_scaled_pca_dim)
print(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_2))
data_processed["dbscan_pca_2"] = y_dbscan_2
data_processed
Visualización de los clústeres, mediante el algoritmo de DBSCAN
sns.set_theme(style = "whitegrid")
fig, axs = plt.subplots(2, 2, figsize = (15,10), dpi = 65)
plt.suptitle(f'Cluster analysis through the K-means algorithm\nSilhouette Score K-means: {round(silhouette_score(data_processed_scaled_pca_dim, y_predict_km),4)} and K = 2', size = 18)
sns.scatterplot(data = data_processed, x = "Income", y = "Purchases", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[0,0])
sns.scatterplot(data = data_processed, x = "Years", y = "Income", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[0,1])
sns.scatterplot(data = data_processed, x = "Education_Encoder", y = "Income", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[1,0])
sns.scatterplot(data = data_processed, x = "Marital_Status_Encoder", y = "Income", hue = "K-Means-PCA", palette = "coolwarm", alpha = 1, ax = axs[1,1])
fig.subplots_adjust(wspace = 0.3, hspace = 0.3)
plt.show()
sns.set_theme(style = "whitegrid")
fig, axs = plt.subplots(2, 2, figsize = (15,10), dpi = 65)
plt.suptitle(f'Cluster analysis through the DBSCAN algorithm\nSilhouette Score DBSCAN: {round(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_1),4)} and K = 4 (-1 is noise)', size = 18)
sns.scatterplot(data = data_processed, x = "Income", y = "Purchases", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[0,0])
sns.scatterplot(data = data_processed, x = "Years", y = "Income", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[0,1])
sns.scatterplot(data = data_processed, x = "Education_Encoder", y = "Income", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[1,0])
sns.scatterplot(data = data_processed, x = "Marital_Status_Encoder", y = "Income", hue = "dbscan_pca_1", palette = "coolwarm", alpha = 1, ax = axs[1,1])
fig.subplots_adjust(wspace = 0.3, hspace = 0.3)
plt.show()
sns.set_theme(style = "whitegrid")
fig, axs = plt.subplots(2, 2, figsize = (15,10), dpi = 65)
plt.suptitle(f'Cluster analysis through the DBSCAN algorithm\nSilhouette Score DBSCAN: {round(silhouette_score(data_processed_scaled_pca_dim,y_dbscan_2),4)} and K = 2 (-1 is noise)', size = 18)
sns.scatterplot(data = data_processed, x = "Income", y = "Purchases", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[0,0])
sns.scatterplot(data = data_processed, x = "Years", y = "Income", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[0,1])
sns.scatterplot(data = data_processed, x = "Education_Encoder", y = "Income", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[1,0])
sns.scatterplot(data = data_processed, x = "Marital_Status_Encoder", y = "Income", hue = "dbscan_pca_2", palette = "coolwarm", alpha = 1, ax = axs[1,1])
fig.subplots_adjust(wspace = 0.3, hspace = 0.3)
plt.show()