A/B Testing

visits['registered'] = visits.signed_up_at.notna() visits = visits.sort_values('visited_at').drop_duplicates('user_id',keep='first').copy() conversion = visits.copy() conversion['week'] = conversion.visited_at.dt.tz_localize(None).dt.to_period('W').dt.to_timestamp() conversion = conversion.groupby(['variant','week']).registered.value_counts(normalize=True,dropna=False).reset_index(name='conversion') conversion = conversion.loc[conversion.registered == True]

conversion.groupby('variant').conversion.mean().reset_index(name='conversion_rate')

from scipy.stats import chi2_contingency # Calculate the proportion of registered users for variant A vs B proportions = visits.groupby('variant').registered.value_counts(dropna=False).reset_index(name='nusers') proportions = proportions.pivot(index='variant',columns='registered',values='nusers') # Chi-Square test g,p,dof,expctd = chi2_contingency(proportions) print(f'p-value is {p}') if p < 0.05: print(f'The difference in Conversion Rate between Variants A and B is statistically significant.') else: print(f'The difference in Conversion Rate between Variants A and B is NOT statistically significant.')

retention.loc[retention.Week == 4][['variant','Retention']].reset_index(drop=True)

from scipy.stats import ttest_ind # Label all users who retained 4+ weeks later users_retention = sessions_weekly.groupby(['user_id','variant']).apply(lambda x: x.week.max() >= 4).reset_index(name='retained') # Calculate the proportion of retained users for variant A vs B proportions = users_retention.groupby('variant').retained.value_counts(dropna=False).reset_index(name='nusers') proportions = proportions.pivot(index='variant',columns='retained',values='nusers') # Chi-Square test g,p,dof,expctd = chi2_contingency(proportions) print(f'p-value is {p}') if p < 0.05: print(f'The difference in Retention between Variants A and B is statistically significant.') else: print(f'The difference in Retention between Variants A and B is NOT statistically significant.')