A/B Testing
This notebook visualizes and measures the results of an A/B test.
Measure weekly conversions from visiting the landing page to signing up.
Measure weekly retention of users who signed up.
Run to view results
Run to view results
from scipy.stats import chi2_contingency
# Calculate the proportion of registered users for variant A vs B
proportions = visits.groupby('variant').registered.value_counts(dropna=False).reset_index(name='nusers')
proportions = proportions.pivot(index='variant',columns='registered',values='nusers')
# Chi-Square test
g,p,dof,expctd = chi2_contingency(proportions)
print(f'p-value is {p}')
if p < 0.05:
print(f'The difference in Conversion Rate between Variants A and B is statistically significant.')
else:
print(f'The difference in Conversion Rate between Variants A and B is NOT statistically significant.')
Run to view results
from scipy.stats import ttest_ind
# Label all users who retained 4+ weeks later
users_retention = sessions_weekly.groupby(['user_id','variant']).apply(lambda x: x.week.max() >= 4).reset_index(name='retained')
# Calculate the proportion of retained users for variant A vs B
proportions = users_retention.groupby('variant').retained.value_counts(dropna=False).reset_index(name='nusers')
proportions = proportions.pivot(index='variant',columns='retained',values='nusers')
# Chi-Square test
g,p,dof,expctd = chi2_contingency(proportions)
print(f'p-value is {p}')
if p < 0.05:
print(f'The difference in Retention between Variants A and B is statistically significant.')
else:
print(f'The difference in Retention between Variants A and B is NOT statistically significant.')
Run to view results
dradada
retention.loc[retention.Week == 4][['variant','Retention']].reset_index(drop=True)
Run to view results
Significance Test
input_1
/ 10