import pandas as pd import numpy as np %pip install openpyxl

df_e = pd.read_excel("Diabetes-classification.xlsx", sheet_name='Emotions') df_pj = pd.read_excel("Diabetes-classification.xlsx", sheet_name='Patient-journey')

print('Patient journey:',df_pj.columns, '\nEmotion:',df_e.columns)

from sklearn.naive_bayes import MultinomialNB from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split import nltk from nltk.tokenize import word_tokenize nltk.download('punkt') from sklearn.feature_extraction.text import CountVectorizer com_words = 20

def get_document_features(document, word_features): """ This function will convert given document into a feature set. """ document_words = set(document) features = {} for word in word_features: features['contains({})'.format(word)] = (word in document_words) return features

#Extracting a word-tokenized version of columns dt and label. e_dt = df_e.apply(lambda row: nltk.word_tokenize(row['discussion_text']), axis=1) e_lab = df_e.apply(lambda row: nltk.word_tokenize(row['Label']), axis=1)

#Computing the frequency distribution of each word in emotions discussion text, by looping through each row of column, and each word of the rows all_words_e = nltk.FreqDist(w.lower() for l in e_dt for w in l) #Choosing the top n most common words for lower computational requirements most_freq_words_e = all_words_e.most_common(com_words) #Computing a list of lists, each list containing all the words in a given document on first index, with second index being the accompanying label doc_lst_e = [] for i in range(len(e_dt)): doc_lst_e.append([e_dt[i], df_e.loc[i]['Label']]) #Retracting a list of the most common words word_features_e = [word for (word, count) in most_freq_words_e] #Finding out how many words from word_features each document contains featuresets_e = [(get_document_features(d,word_features_e), l) for (d,l) in doc_lst_e]

#Splitting data split_r_e = round(len(featuresets_e)*0.15) train_set_e,test_set_e = featuresets_e[:split_r_e], featuresets_e[split_r_e:] #Training the classifier on the first 75% of the feature set clf_e= nltk.NaiveBayesClassifier.train(train_set_e) print(nltk.classify.accuracy(clf_e, test_set_e))

pj_dt = df_pj.apply(lambda row: nltk.word_tokenize(row['discussion_text']), axis=1) pj_lab = df_pj.apply(lambda row: nltk.word_tokenize(row['Label']), axis=1)

#Finding all words in the entire set of documents all_words_pj = nltk.FreqDist(w.lower() for l in pj_dt for w in l) #Computing top n most common words most_freq_words_pj = all_words_pj.most_common(com_words) #Making list of the most frequent words word_features_pj = [word for (word, count) in most_freq_words_pj] #List of lists, index 1: words in doc, index2: accompanying label doc_lst_pj = [] for i in range(len(pj_dt)): doc_lst_pj.append([pj_dt[i], df_pj.loc[i]['Label']]) #How many words from the most common words are present in each document featuresets_pj = [(get_document_features(d,word_features_pj), l) for (d,l) in doc_lst_pj]

split_r_pj = round(len(featuresets_pj)*0.15) #Splitting into train and test, 15/85 train_set_pj,test_set_pj = featuresets_pj[:split_r_pj], featuresets_pj[split_r_pj:] #Trainig the classifier clf_pj = nltk.NaiveBayesClassifier.train(train_set_pj) #Printing results accuracy print(nltk.classify.accuracy(clf_pj, test_set_pj))

print('MNB on Emotion:',nltk.classify.accuracy(clf_e, test_set_e)) print('MNB on Patient Journey:',nltk.classify.accuracy(clf_pj, test_set_pj))

from sklearn.feature_extraction.text import CountVectorizer from sklearn.pipeline import Pipeline from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score from sklearn.feature_extraction.text import TfidfTransformer from sklearn import metrics import sys

def get_most_informative_features(clf, vectorizer, label_names, max_number_informative_features): """ Prints features with the highest coefficient values, per class """ output = [] try: feature_names = vectorizer.get_feature_names() label_index = len(label_names) if label_index == 2: label_index = 1 print('features for binary classification!') for index in range(label_index): #print(str(index) + label_names[index]) #print('clf.coef_:',len(clf.coef_)) output.append('\n' + label_names[index] + ':\n') coefs_with_fns = sorted(zip(clf.coef_[index], feature_names)) #print(coefs_with_fns) threshold = int(max_number_informative_features / 2) top = zip(coefs_with_fns[:threshold], coefs_with_fns[:-(threshold + 1):-1]) for (coef_1, fn_1), (coef_2, fn_2) in top: feat = "\t%.4f\t%-15s\t\t%.4f\t%-15s" % (coef_1, fn_1, coef_2, fn_2) output.append(feat) #print('\n'.join(output)) except: print("Unexpected error:", sys.exc_info()[0]) raise return '\n'.join(output) return '\n'.join(output)

lab_e_lst = [] for i in df_e['Label']: temp_tok = word_tokenize(i) lab_e_lst.append(temp_tok) lab_e = [] for i in range(len(lab_e_lst)): lab_e.append(lab_e_lst[i][0]) u_lab_e = list(set(lab_e))

xTrain_e, xTest_e, yTrain_e, yTest_e = train_test_split(list(df_e['discussion_text']), lab_e) pipe_clf = Pipeline( [('vect', CountVectorizer(decode_error='ignore', stop_words='english', max_features=100)), ('tfidf', TfidfTransformer()), ('clf', LogisticRegression( penalty='l2', solver='lbfgs', max_iter = 1000, dual=False, multi_class='multinomial', tol=1e-3)), ]) text_clf = pipe_clf.fit(xTrain_e,yTrain_e) predictions = pipe_clf.predict(xTest_e) print('accuracy :', accuracy_score(yTest_e, predictions)) acc_e = accuracy_score(yTest_e, predictions) measures_info = metrics.classification_report(yTest_e, predictions) print('\n\n',measures_info) vectorizer = pipe_clf.named_steps['vect'] clf = pipe_clf.named_steps['clf'] feature_names = vectorizer.get_feature_names() imp_features_e = get_most_informative_features(clf, vectorizer,u_lab_e, 20) #print('\nTop 20 most important features:\n', imp_features)

lab_pj_lst = [] for i in df_pj['Label']: temp_tok = word_tokenize(i) lab_pj_lst.append(temp_tok) lab_pj = [] for i in range(len(lab_pj_lst)): lab_pj.append(lab_pj_lst[i][0]) u_lab_pj = list(set(lab_pj))

xTrain_pj, xTest_pj, yTrain_pj, yTest_pj = train_test_split(list(df_pj['discussion_text']), lab_pj) text_clf = pipe_clf.fit(xTrain_pj,yTrain_pj) predictions = pipe_clf.predict(xTest_pj) print('accuracy :', accuracy_score(yTest_pj, predictions)) measures_info = metrics.classification_report(yTest_pj, predictions) print('\n\n',measures_info) vectorizer_pj = pipe_clf.named_steps['vect'] clf_pj = pipe_clf.named_steps['clf'] feature_names_pj = vectorizer.get_feature_names() imp_features_pj = get_most_informative_features(clf_pj, vectorizer_pj,u_lab_pj, 20) #print('\nTop 20 most important features:\n', imp_features_pj)

print('The accurracy of the Multinomail Naïve Bayes models for the Emotions and Patien Journal are as follows: \n - Emotions: ',nltk.classify.accuracy(clf_e, test_set_e), '\n - Patien Journal: ', nltk.classify.accuracy(clf_pj, test_set_pj),'\n\nThe accurracy of the Logistic Regression models for the Emotions and Patien Journal are as follows: \n - Emotions:',0.29634002361275086,'\n - Patien Journal: ', accuracy_score(yTest_pj, predictions))

print('Emotions: ') clf_e.show_most_informative_features(20) print('\n\nPatient Journal:') clf_pj.show_most_informative_features(20)

print('\nTop 20 most important features:\n', imp_features)

vectorizer_pj = pipe_clf.named_steps['vect'] clf_pj = pipe_clf.named_steps['clf'] feature_names_pj = vectorizer.get_feature_names() imp_features_pj = get_most_informative_features(clf_pj, vectorizer_pj,u_lab_pj, 20) print('\nTop 20 most important features:\n', imp_features_pj)