import pandas as pd
import pingouin as pg
import scipy.stats as stats
import researchpy as rp
df = pd.read_csv('Experiment Cells - aggregate-all.csv')
df.head()
document_id = []
rater_id = []
scoreA = []
scoreR = []
scoreI = []
for i in range(1, 195):
item = df.loc[df['ID'] == i].iloc[0]
document_id.append(i)
rater_id.append('April')
scoreA.append(int(item['Accuracy (A)']))
scoreR.append(int(item['Readability (A)']))
scoreI.append(int(item['Informativeness (A)']))
document_id.append(i)
rater_id.append('Dakuo')
scoreA.append(int(item['Accuracy (D)']))
scoreR.append(int(item['Readability (D)']))
scoreI.append(int(item['Informativeness (D)']))
data = {'document_id': document_id, 'rater_id': rater_id, 'scoreA': scoreA, 'scoreR': scoreR, 'scoreI': scoreI}
reliability_df = pd.DataFrame(data)
document_id = []
rater_id = []
scoreA = []
scoreR = []
scoreI = []
for i in range(1, 51):
item = df.loc[df['ID'] == i].iloc[0]
document_id.append(i)
rater_id.append('April')
scoreA.append(int(item['Accuracy (A)']))
scoreR.append(int(item['Readability (A)']))
scoreI.append(int(item['Informativeness (A)']))
document_id.append(i)
rater_id.append('Dakuo')
scoreA.append(int(item['Accuracy (D)']))
scoreR.append(int(item['Readability (D)']))
scoreI.append(int(item['Informativeness (D)']))
data = {'document_id': document_id, 'rater_id': rater_id, 'scoreA': scoreA, 'scoreR': scoreR, 'scoreI': scoreI}
reliability50_df = pd.DataFrame(data)
reliability_df
pg.intraclass_corr(data=reliability_df, targets='document_id', raters='rater_id',
ratings='scoreA')
pg.intraclass_corr(data=reliability50_df, targets='document_id', raters='rater_id',
ratings='scoreA')
pg.intraclass_corr(data=reliability_df, targets='document_id', raters='rater_id',
ratings='scoreR')
pg.intraclass_corr(data=reliability50_df, targets='document_id', raters='rater_id',
ratings='scoreR')
pg.intraclass_corr(data=reliability_df, targets='document_id', raters='rater_id',
ratings='scoreI')
pg.intraclass_corr(data=reliability50_df, targets='document_id', raters='rater_id',
ratings='scoreI')
document_id = []
method = []
scoreA = []
scoreR = []
scoreI = []
for i in range(1, 195):
item = df.loc[df['ID'] == i].iloc[0]
document_id.append(i)
method.append(item['Written by (Markdown Cell)'])
scoreA.append((item['Accuracy (A)'] + item['Accuracy (D)']) / 2)
scoreR.append((item['Readability (A)'] + item['Readability (D)']) / 2)
scoreI.append((item['Informativeness (A)'] + item['Informativeness (D)']) / 2)
data = {'document_id': document_id, 'method': method, 'scoreA': scoreA, 'scoreR': scoreR, 'scoreI': scoreI}
score_df = pd.DataFrame(data)
score_df
rp.summary_cont(score_df['scoreA'].groupby(score_df['method']))
stats.f_oneway(score_df['scoreA'][score_df['method'] == 'A'],
score_df['scoreA'][score_df['method'] == 'M'],
score_df['scoreA'][score_df['method'] == 'H'])
rp.summary_cont(score_df['scoreR'].groupby(score_df['method']))
stats.f_oneway(score_df['scoreR'][score_df['method'] == 'A'],
score_df['scoreR'][score_df['method'] == 'M'],
score_df['scoreR'][score_df['method'] == 'H'])
rp.summary_cont(score_df['scoreI'].groupby(score_df['method']))
stats.f_oneway(score_df['scoreI'][score_df['method'] == 'A'],
score_df['scoreI'][score_df['method'] == 'M'],
score_df['scoreI'][score_df['method'] == 'H'])