#install packages
!pip install lifetimes
Requirement already satisfied: lifetimes in /root/venv/lib/python3.7/site-packages (0.11.3)
Requirement already satisfied: numpy>=1.10.0 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from lifetimes) (1.21.6)
Requirement already satisfied: autograd>=1.2.0 in /root/venv/lib/python3.7/site-packages (from lifetimes) (1.4)
Requirement already satisfied: scipy>=1.0.0 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from lifetimes) (1.7.3)
Requirement already satisfied: dill>=0.2.6 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from lifetimes) (0.3.4)
Requirement already satisfied: pandas>=0.24.0 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from lifetimes) (1.2.5)
Requirement already satisfied: future>=0.15.2 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from autograd>=1.2.0->lifetimes) (0.18.2)
Requirement already satisfied: pytz>=2017.3 in /shared-libs/python3.7/py/lib/python3.7/site-packages (from pandas>=0.24.0->lifetimes) (2022.1)
Requirement already satisfied: python-dateutil>=2.7.3 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from pandas>=0.24.0->lifetimes) (2.8.2)
Requirement already satisfied: six>=1.5 in /shared-libs/python3.7/py-core/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas>=0.24.0->lifetimes) (1.16.0)
WARNING: You are using pip version 20.1.1; however, version 22.0.4 is available.
You should consider upgrading via the '/root/venv/bin/python -m pip install --upgrade pip' command.
# Importing the libraries
import pandas as pd
import numpy as np
import sqlite3 as sql
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
import lifetimes
from lifetimes.utils import summary_data_from_transaction_data
from datetime import datetime
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.simplefilter("ignore")
# connect to the database
db_conn = sql.connect("/datasets/hidrive/base2.db")
#show all tables from the database
pd.read_sql(
"""
select *
from sqlite_master
""", db_conn)
typeobject
table100%
nameobject
geolocation11.1%
order_items11.1%
7 others77.8%
0
table
geolocation
1
table
order_items
2
table
customers
3
table
order_payments
4
table
order_reviews
5
table
orders
6
table
products
7
table
sellers
8
table
translation
# we dont want to evaluate canceled orders, so we are not taking this data
pd.read_sql(
"""
Select DISTINCT order_status
from orders
""", db_conn)
order_statusobject
0
delivered
1
invoiced
2
shipped
3
processing
4
unavailable
5
canceled
6
created
7
approved
SQL_Query = pd.read_sql(
"""
Select
c.customer_unique_id,
o.order_purchase_timestamp,
p.payment_value
from customers as c
join orders as o on o.customer_id = c.customer_id
join order_payments as p on o.order_id = p.order_id
Where NOT o.order_status= 'canceled'
""", db_conn)
df = pd.DataFrame(SQL_Query)
# Checking our dataframe
df.info()
df.head()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 103222 entries, 0 to 103221
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 customer_unique_id 103222 non-null object
1 order_purchase_timestamp 103222 non-null object
2 payment_value 103222 non-null float64
dtypes: float64(1), object(2)
memory usage: 2.4+ MB
customer_unique_idobject
order_purchase_timestampobject
0
7c396fd4830fd04220f754e42b4e5bff
2017-10-02 10:56:33
1
7c396fd4830fd04220f754e42b4e5bff
2017-10-02 10:56:33
2
7c396fd4830fd04220f754e42b4e5bff
2017-10-02 10:56:33
3
af07308b275d755c9edb36a90c618231
2018-07-24 20:41:37
4
3a653a41f6f9fc3d2a113cf8398680e8
2018-08-08 08:38:49
# Checking for null variables
df.isnull().sum(axis=0)
#Counting unique customers
df['customer_unique_id'].nunique()
#This function converts a scalar, array-like, Series or DataFrame/dict-like to a pandas datetime object.
df['order_purchase_timestamp'] = pd.to_datetime(df['order_purchase_timestamp'], format="%Y-%m-%d %H:%M:%S")
#Returns numpy array of python datetime.date objects. It can be used to access the values of the series as datetimelike and return several properties.
df['order_purchase_timestamp'] = df.order_purchase_timestamp.dt.date
# exploring its minimum (first purchase) and maximum (last purchase) values.
df['order_purchase_timestamp'] = pd.to_datetime(df['order_purchase_timestamp'])
df['order_purchase_timestamp'].describe()
# The last purchase made in the dataset was on August 29, 2018, so we will use that date as our current date to simulate an immediate study of the company's transactions.
today = '2018-09-03'
date_today = datetime.strptime(today, '%Y-%m-%d') # datetime.strptime() Returns a datetime corresponding to date_string, parsed according to format.
print(date_today)
print(type(date_today))
2018-09-03 00:00:00
<class 'datetime.datetime'>
# REGENCY: For every customer, create min and max purchase time, then calculate the difference between max and min time of purchase
r = df.groupby('customer_unique_id').agg(['min', 'max'])['order_purchase_timestamp']
r['recency'] = r['max'] - r['min']
# T: New column, calculate from today and the customer's last purchase
r['T'] = date_today - r['min']
r = r[['recency', 'T']]
# Let's take a look at our new variables
r.head()
recencytimedelta64[ns]
Ttimedelta64[ns]
0000366f3b9a7992bf8c76cfdf3221e2
0 days 00:00:00
116 days 00:00:00
0000b849f77a49e4a4ce2b2a4ca5be3f
0 days 00:00:00
119 days 00:00:00
0000f46a3911fa3c0805444483337064
0 days 00:00:00
542 days 00:00:00
0000f6ccb0745a6a4b88665a16c9f078
0 days 00:00:00
326 days 00:00:00
0004aac84e0df4da2b147fca70cf8255
0 days 00:00:00
293 days 00:00:00
# Creating a new dictionary, aggregations
aggregations = {
'order_purchase_timestamp':'count',
'payment_value': 'sum'}
# Using it in the groupby function to obtain the f
f = df.groupby('customer_unique_id').agg(aggregations)
#Creating a new column 'frequency' (becaouse: This means that it’s one less than the total number of purchases)
f['frequency'] = f['order_purchase_timestamp'] - 1
f = f[['frequency']]
# Merging r and f
rf = pd.merge(r,f, left_index=True, right_index=True) # 'right_index' = Use the index from the right DataFrame as the join key.
rf.head()
recencytimedelta64[ns]
Ttimedelta64[ns]
0000366f3b9a7992bf8c76cfdf3221e2
0 days 00:00:00
116 days 00:00:00
0000b849f77a49e4a4ce2b2a4ca5be3f
0 days 00:00:00
119 days 00:00:00
0000f46a3911fa3c0805444483337064
0 days 00:00:00
542 days 00:00:00
0000f6ccb0745a6a4b88665a16c9f078
0 days 00:00:00
326 days 00:00:00
0004aac84e0df4da2b147fca70cf8255
0 days 00:00:00
293 days 00:00:00
rfm = summary_data_from_transaction_data(
df,
customer_id_col='customer_unique_id',
datetime_col='order_purchase_timestamp',
monetary_value_col ='payment_value',
observation_period_end='2018-08-29',
datetime_format='%Y-%m-%d',
freq='W') #Options: Y,M,W,D - year, month, weeks, days
rfm
rfm = summary_data_from_transaction_data(
df,
customer_id_col='customer_unique_id',
datetime_col='order_purchase_timestamp',
monetary_value_col ='payment_value',
observation_period_end='2018-08-29',
datetime_format='%Y-%m-%d',
freq='W') #Options: Y,M,W,D - year, month, weeks, days
rfm.head()
frequencyfloat64
recencyfloat64
0000366f3b9a7992bf8c76cfdf3221e2
0
0
0000b849f77a49e4a4ce2b2a4ca5be3f
0
0
0000f46a3911fa3c0805444483337064
0
0
0000f6ccb0745a6a4b88665a16c9f078
0
0
0004aac84e0df4da2b147fca70cf8255
0
0
import plotly.express as px
px.histogram(rfm, x=rfm['frequency'],title='Frequency of purchase',
labels={'frequency':'Frequency'},
opacity=0.8, marginal='violin',
color_discrete_sequence=['indianred'])
px.histogram(rfm, x=rfm['recency'],title='Recency of purchase',
labels={'recency':'Recency'}, nbins=50,
opacity=0.8, marginal='violin',
color_discrete_sequence=['indianred'])
px.histogram(rfm, x=rfm['T'],title='Time from first purchase',
labels={'T':'Weeks'},
opacity=0.8, marginal='violin',
color_discrete_sequence=['indianred'])
from lifetimes import BetaGeoFitter
bgf = BetaGeoFitter(penalizer_coef=0.001)
#Fit the BG/BB model. Returns: BetaGeoBetaBinomFitter – fitted and with parameters estimated
bgf.fit(rfm['frequency'], rfm['recency'], rfm['T'], verbose=True) #Verbose: Set to true to print out convergence diagnostics.
print(bgf)
Optimization terminated successfully.
Current function value: 0.079785
Iterations: 47
Function evaluations: 48
Gradient evaluations: 48
<lifetimes.BetaGeoFitter: fitted with 95559 subjects, a: 0.28, alpha: 16.94, b: 0.06, r: 0.02>
bgf.summary
#Thats our trained model, it can be used in the lifetimes library for the following visualizations.
coeffloat64
se(coef)float64
r
0.02075049421
0.00130072923
alpha
16.94438221
1.681726748
a
0.2829437777
0.05081711888
b
0.05660246228
0.01117832482
from lifetimes.plotting import plot_frequency_recency_matrix
from lifetimes.plotting import plot_probability_alive_matrix
fig = plt.figure(figsize=(12,8))
plot_frequency_recency_matrix(bgf, T=4) # T = Unit times(weeks)
fig = plt.figure(figsize=(12,8))
plot_probability_alive_matrix(bgf)
from lifetimes.utils import calibration_and_holdout_data
rfm_val = calibration_and_holdout_data(df,
customer_id_col='customer_unique_id',
datetime_col='order_purchase_timestamp',
monetary_value_col ='payment_value',
calibration_period_end='2018-05-29',
observation_period_end='2018-08-29',
datetime_format='%Y-%m-%d',
freq='W')
rfm_val.head(5)
frequency_calfloat64
recency_calfloat64
0000366f3b9a7992bf8c76cfdf3221e2
0
0
0000b849f77a49e4a4ce2b2a4ca5be3f
0
0
0000f46a3911fa3c0805444483337064
0
0
0000f6ccb0745a6a4b88665a16c9f078
0
0
0004aac84e0df4da2b147fca70cf8255
0
0
bgf_val = BetaGeoFitter(penalizer_coef=0.001)
bgf_val.fit(rfm_val['frequency_cal'], rfm_val['recency_cal'], rfm_val['T_cal'], verbose=True)
print(bgf_val)
Optimization terminated successfully.
Current function value: 0.073844
Iterations: 48
Function evaluations: 49
Gradient evaluations: 49
<lifetimes.BetaGeoFitter: fitted with 77125 subjects, a: 0.24, alpha: 17.47, b: 0.05, r: 0.02>
from lifetimes.plotting import plot_calibration_purchases_vs_holdout_purchases
fig = plt.figure(figsize=(12,8))
plot_calibration_purchases_vs_holdout_purchases(model=bgf_val, calibration_holdout_matrix=rfm_val)
rfm_gg = rfm[rfm['frequency'] > 0]
len(rfm_gg) #len(): computes the number of elements
rfm_gg[['monetary_value', 'frequency']].corr() #corr(): computes the correlation
monetary_valuefloat64
frequencyfloat64
monetary_value
1
0.01102058932
frequency
0.01102058932
1
from lifetimes import GammaGammaFitter
ggf = GammaGammaFitter(penalizer_coef = 0.0)
ggf.fit(rfm_gg['frequency'], rfm_gg['monetary_value'])
print(ggf)
<lifetimes.GammaGammaFitter: fitted with 1987 subjects, p: 5.77, q: 2.93, v: 50.39>
rfm['avg_transaction'] = round(ggf.conditional_expected_average_profit(rfm_gg['frequency'],
rfm_gg['monetary_value']), 2)
rfm['avg_transaction'] = rfm['avg_transaction'].fillna(0)
rfm.sort_values(by='avg_transaction', ascending=False)
frequencyfloat64
0.0 - 14.0
recencyfloat64
a1044dd75b74fbc485b040575a14acf0
1
1
012a218df8995d3ec3bb221828360c86
1
6
da1e7179b9c5a1494d78528cbcf05aa0
1
25
6e26bbeaa107ec34112c64e1ee31c0f5
1
26
f0daba1cfcc18d5a3229e1a911313e3e
1
2
5457ae2c352c18e3bf5cba68301d96a8
1
21
ecdbc4bf2201c807294c7d8c4b777032
1
29
4711348768db55c57a67aba080ead566
1
3
73601b1eec55943e90ce8d61253d5c09
1
5
d5419994a02a55d4b7cd00e77fabdbae
1
5