A/B Testing
Measure weekly retention of users who signed up.
Run to view results
Run to view results
Conversion Rate
visits['registered'] = visits.signed_up_at.notna()
visits = visits.sort_values('visited_at').drop_duplicates('user_id',keep='first').copy()
conversion = visits.copy()
conversion['week'] = conversion.visited_at.dt.tz_localize(None).dt.to_period('W').dt.to_timestamp()
conversion = conversion.groupby(['variant','week']).registered.value_counts(normalize=True,dropna=False).reset_index(name='conversion')
conversion = conversion.loc[conversion.registered == True]
Run to view results
Run to view results
conversion.groupby('variant').conversion.mean().reset_index(name='conversion_rate')
Run to view results
from scipy.stats import chi2_contingency
# Calculate the proportion of registered users for variant A vs B
proportions = visits.groupby('variant').registered.value_counts(dropna=False).reset_index(name='nusers')
proportions = proportions.pivot(index='variant',columns='registered',values='nusers')
# Chi-Square test
g,p,dof,expctd = chi2_contingency(proportions)
print(f'p-value is {p}')
if p < 0.05:
print(f'The difference in Conversion Rate between Variants A and B is statistically significant.')
else:
print(f'The difference in Conversion Rate between Variants A and B is NOT statistically significant.')
Run to view results
Retention Rate
Run to view results
Run to view results
Run to view results
retention.loc[retention.Week == 4][['variant','Retention']].reset_index(drop=True)
Run to view results
from scipy.stats import ttest_ind
# Label all users who retained 4+ weeks later
users_retention = sessions_weekly.groupby(['user_id','variant']).apply(lambda x: x.week.max() >= 4).reset_index(name='retained')
# Calculate the proportion of retained users for variant A vs B
proportions = users_retention.groupby('variant').retained.value_counts(dropna=False).reset_index(name='nusers')
proportions = proportions.pivot(index='variant',columns='retained',values='nusers')
# Chi-Square test
g,p,dof,expctd = chi2_contingency(proportions)
print(f'p-value is {p}')
if p < 0.05:
print(f'The difference in Retention between Variants A and B is statistically significant.')
else:
print(f'The difference in Retention between Variants A and B is NOT statistically significant.')
Run to view results
input_1
0 / 10