import numpy as np
import pandas as pd
import string
import matplotlib.pyplot as plt
import seaborn as sns
import ipywidgets
from ipywidgets import interact
import warnings
warnings.filterwarnings("ignore")
plt.rcParams['figure.figsize'] = (15, 5)
plt.style.use('fivethirtyeight')
# reading the Dataset
data = pd.read_csv('drug.csv', error_bad_lines=False)
# lets print the shape of the dataset
print("The Shape of the Dataset :", data.shape)
# lets check the head of the dataset
data.head()
# lets Explore Some of the Important Column in the dataset
print("Number of Unique Drugs present in the Dataset :", data['drugName'].nunique())
print("Number of Unique Medical Conditions present in the Dataset :", data['condition'].nunique())
print("\nThe Time Period of Collecting the Data")
print("Starting Date :", data['date'].min())
print("Ending Date :", data['date'].max())
# lets summarize the Dataset
data[['rating','usefulCount']].describe()
# lets check the Number and Name of the Drugs with 0 Useful Count in Details
print("Analysis on Useless Drugs")
print("----------------------------")
print("The Number of Drugs with No Useful Count :", data[data['usefulCount'] == 0].count()[0])
# Lets Check the Number of Drugs with No Usesful Count with Review Greater than or Equal to 8
print("Number of Good Drugs with Lesser Useful Count :", data[(data['usefulCount'] == 0) &
data['rating'] >= 8].count()[0])
# Lets Check the Average Rating of the Drugs with No Useful Count
print("Average Rating of Drugs with No Useful Count : {0:.2f}".format(data[data['usefulCount'] == 0]['rating'].mean()))
print("\nAnalysis on Useful Drugs")
print("----------------------------")
print("The Number of Drugs with Greater than 1000 Useful Counts :", data[data['usefulCount'] > 1000].count()[0])
print("Average Rating of Drugs with 1000+ Useful Counts :", data[data['usefulCount'] > 1000]['rating'].mean())
print("\nName and Condition of these Drugs: \n\n",
data[data['usefulCount'] > 1000][['drugName','condition']].reset_index(drop = True))
# lets summarize Categorical data also
data[['drugName','condition','review']].describe(include = 'object')
# lets check for Missing Values
data.isnull().sum()
# as we know that condition is an Important Column, so we will delete all the records where Condition is Missing
data = data.dropna()
# lets check the Missing values now
data.isnull().sum().sum()
# lets check the Distribution of Rating and Useful Count
plt.rcParams['figure.figsize'] = (15, 4)
plt.subplot(1, 2, 1)
sns.distplot(data['rating'])
plt.subplot(1, 2, 2)
sns.distplot(data['usefulCount'])
plt.suptitle('Distribution of Rating and Useful Count \n ', fontsize = 20)
plt.show()
# lets check the Impact of Ratings on Usefulness
plt.rcParams['figure.figsize'] = (15, 4)
sns.barplot(data['rating'], data['usefulCount'], palette = 'hot')
plt.grid()
plt.xlabel('\n Ratings')
plt.ylabel('Count\n', fontsize = 20)
plt.title('\n Rating vs Usefulness \n', fontsize = 20)
plt.show()
# Checking whether Length of Review has any Impact on Ratings of the Drugs
# for that we need to create a new column to calculate length of the reviews
data['len'] = data['review'].apply(len)
# lets check the Impact of Length of Reviews on Ratings
data[['rating','len']].groupby(['rating']).agg(['min','mean','max'])
# lets check the Highest Length Review
print("Length of Longest Review", data['len'].max())
data['review'][data['len'] == data['len'].max()].iloc[0]
data['review']
# as it is clear that the reviews have so many unnecassry things such as Stopwords, Punctuations, numbers etc
# First lets remove Punctuations from the Reviews
def punctuation_removal(messy_str):
clean_list = [char for char in messy_str if char not in string.punctuation]
clean_str = ''.join(clean_list)
return clean_str
data['review'] = data['review'].apply(punctuation_removal)
data['review']
# Now lets Remove the Stopwords also
import nltk
#nltk.download('stopwords')
#nltk.download('punkt')
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
stop = stopwords.words('english')
stop.append("i'm")
stop_words = []
for item in stop:
new_item = punctuation_removal(item)
stop_words.append(new_item)
def stopwords_removal(messy_str):
messy_str = word_tokenize(messy_str)
return [word.lower() for word in messy_str
if word.lower() not in stop_words ]
data['review'] = data['review'].apply(stopwords_removal)
data['review']
# lets remove the Numbers also
import re
def drop_numbers(list_text):
list_text_new = []
for i in list_text:
if not re.search('\d', i):
list_text_new.append(i)
return ' '.join(list_text_new)
data['review'] = data['review'].apply(drop_numbers)
data['review']
# for using Sentiment Analyzer we will have to dowload the Vader Lexicon from NLTK
import nltk
nltk.download('vader_lexicon')
# lets calculate the Sentiment from Reviews
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
train_sentiments = []
for i in data['review']:
train_sentiments.append(sid.polarity_scores(i).get('compound'))
train_sentiments = np.asarray(train_sentiments)
data['sentiment'] = pd.Series(data=train_sentiments)
# lets check Impact of Sentiment on Reviews
data[['rating','sentiment']].groupby(['rating']).agg(['min','mean','max'])
# as we can see that Sentiment and length of the review are not related to Reviews, we will drop the sentiment column
# lets remove the unique Id, date, review, len, and sentiment column also
data = data.drop(['date','uniqueID','sentiment','review','len'], axis = 1)
# lets check the name of columns now
data.columns
# Lets Calculate an Effective Rating
min_rating = data['rating'].min()
max_rating = data['rating'].max()
def scale_rating(rating):
rating -= min_rating
rating = rating/(max_rating -1)
rating *= 5
rating = int(round(rating,0))
if(int(rating) == 0 or int(rating)==1 or int(rating)==2):
return 0
else:
return 1
data['eff_score'] = data['rating'].apply(scale_rating)
# lets also calculate Usefulness Score
data['usefulness'] = data['rating']*data['usefulCount']*data['eff_score']
# lets check the Top 10 Most Useful Drugs with their Respective Conditions
data[['drugName','condition','usefulness']][data['usefulness'] >
data['usefulness'].mean()].sort_values(by = 'usefulness',
ascending = False).head(10).reset_index(drop = True)
# lets calculate the Number of Useless and Useful Drugs for Each Condition
def check(condition = list(data['condition'].value_counts().index)):
return data[data['condition'] == condition]['eff_score'].value_counts()
# lets check this in Graph,
popular_conditions = ('Birth Control','Depression','Pain','Anxiety','Acne','Bipolar Disorder','Insomnia','Weight Loss',
'Obesity','ADHD', 'Diabetes, Type 2', 'Emergency Contraception', 'High Blood Pressure','Migrane')
conditions = data.loc[data['condition'].isin(popular_conditions)]
sns.barplot(x = conditions['condition'], y = conditions['rating'], hue = data['eff_score'],
palette = 'autumn')
plt.title('Conditions vs Effective Number of Drugs')
plt.xticks(rotation = 90)
plt.ylabel(' ')
plt.show()
# lets check the Most Common Conditions
print("Number of Unique Conditions :", data['condition'].nunique())
data['condition'].value_counts().head(10)
# lets check Drugs, which were useful to Highest Number of Poeple
data[['drugName','usefulCount']][data['usefulCount'] >
data['usefulCount'].mean()].sort_values(by = 'usefulCount',
ascending = False).head(10).reset_index(drop = True)
# lets remove all the Duplicates from the Dataset
data = data.drop_duplicates()
# lets find the Highest and Lowest Rated Drugs for each Condition
def high_low_rate(condition = list(data['condition'].value_counts().index)):
print("\n Top 5 Drugs")
print(data[data['condition'] == condition][['drugName','usefulness']].sort_values(by = 'usefulness',
ascending = False).head().reset_index(drop = True))
print("\n\n Bottom 5 Drugs")
print(data[data['condition'] == condition][['drugName','usefulness']].sort_values(by = 'usefulness',
ascending = True).head().reset_index(drop = True))