import pandas as pd
import numpy as np
videos_stats = pd.read_csv('videos_stats.csv')
videos_stats
Run to view results
comments = pd.read_csv('comments.csv')
comments
Run to view results
print(f'shape of video_stats',videos_stats.shape)
print(f'no of unique videos',videos_stats['Video ID'].nunique())
print(f'no of unique videos',comments['Video ID'].nunique())
Run to view results
identify duplicate records
duplicate_records_ids
Run to view results
videos_stats=videos_stats[~videos_stats['Video ID'].isin(duplicate_records_ids)]
comments=comments[~comments['Video ID'].isin(duplicate_records_ids)]
Run to view results
print(f'shape of video_stats',videos_stats.shape)
print(f'no of unique videos',videos_stats['Video ID'].nunique())
print(f'no of unique videos',comments['Video ID'].nunique())
Run to view results
Identify missing values
videos_stats.isna().sum()
Run to view results
videos_stats.fillna(0,inplace=True)
comments.isna().sum()
comments.dropna(inplace=True)
Run to view results
Creating a combined Dataset
df_combined=pd.merge(comments,videos_stats,how='left',on='Video ID')
Run to view results
df_combined.head()
Run to view results
Creating groups of video contents
def industry(text):
if text in ["biology","cnn","computer science","data science","education","interview","machine learning","mathchemistry","physics"]:
group="Education"
elif text in ["lofi","marvel","movies","mrbeast","music","reaction","trolling"]:
group='Entertainment'
elif text in ["chess","game development","gaming","minecraft","sports","cubes"]:
group='Gaming'
elif text in ["apple","food"]:
group='Health'
elif text in ["animals","asmr","bed","mukbang","sat"]:
group='Lifestyle'
elif text in ["history",'literature','news']:
group='News & Politics'
elif text in ['crypto','google','how-to','nintendo','tech','tutorial','xbox']:
group='Technology'
else :
group='Business & Finance'
return group
Run to view results
videos_stats['Keyword_group']=videos_stats['Keyword'].apply(industry)
df_combined['Keyword_group']=df_combined['Keyword'].apply(industry)
Run to view results
videos_stats.head()
Run to view results
EDA
import matplotlib.pyplot as plt
import seaborn as sns
Run to view results
Creating pivot tables
table1=pd.pivot_table(data=videos_stats,values='Video ID',index='Keyword_group',aggfunc='nunique',columns='year',fill_value=0)
#print(table1)
Run to view results
table2=videos_stats.groupby(['year','Keyword_group']).agg(
No_of_Videos=('Video ID', 'nunique'),
No_of_views_in_k=('Views','sum'),
No_of_comments_in_k=('Comments','sum'),
No_of_likes_in_k=('Likes','sum')
)
Run to view results
table2['No_of_views_in_k']=table2['No_of_views_in_k']/1000
table2['No_of_comments_in_k']=table2['No_of_comments_in_k']/1000
table2['No_of_likes_in_k']=table2['No_of_likes_in_k']/1000
Run to view results
table2['views_per_video']=table2['No_of_views_in_k']/table2['No_of_Videos']
table2['comments_per_video']=table2['No_of_comments_in_k']/table2['No_of_Videos']
table2['likes_per_video']=table2['No_of_likes_in_k']/table2['No_of_Videos']
Run to view results
table2.reset_index(inplace=True)
Run to view results
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of videos in each industry per year')
sns.heatmap(table1,cmap='coolwarm',annot=True,fmt='.2g')
Run to view results
Visuals displaying no of videos
Run to view results
Run to view results
Visuals showing views and likes of videos
Run to view results
Run to view results
Visuals showing trending contents
Run to view results
Run to view results
Run to view results
Run to view results
Run to view results
Run to view results
Run to view results
Visuals showing total views vs views per video
Run to view results
Run to view results
Run to view results
Heatmaps to show popular contents before and after covid
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of views in each industry in after covid')
sns.heatmap(table3,cmap='plasma',annot=False,fmt='.2g',square=True,cbar=True)
Run to view results
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of likes in each industry after covid')
sns.heatmap(table4,cmap='seismic',annot=False,fmt='.2g',square=True,cbar=True)
Run to view results
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of likes in each industry before covid')
sns.heatmap(table5,cmap='magma',annot=False,fmt='.2g',square=True,cbar=True)
Run to view results
table6=pd.pivot_table(data=videos_stats[(videos_stats['year']<=2020) ],values='Views',index='Keyword_group',aggfunc='sum',columns='Keyword',fill_value=0)
table6=table6/1000000
#print(table4)
Run to view results
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of views in each industry before covid')
sns.heatmap(table6,cmap='bwr',annot=False,fmt='.2g',square=True,cbar=True)
Run to view results
Challenge 1 : Summary
The no videos on youtube has increased exponentially in 2022 speacially techology and entertainment
Videos reated to entertainment has peaked trend post covid.
Under entertainment , trending video are related to marvel,mrbeast and music
Before covid , people liked and viewed mostly videos related to lifestyle and technology .
Sentiment Analysis of Video Comments
comments['Comment']=comments['Comment'].astype('str')
Run to view results
import re
Run to view results
###removing emojis
def remove_emojis(text):
emoji_pattern = re.compile(
"["
"\U0001F600-\U0001F64F" # emoticons
"\U0001F300-\U0001F5FF" # symbols & pictographs
"\U0001F680-\U0001F6FF" # transport & map symbols
"\U0001F700-\U0001F77F" # alchemical symbols
"\U0001F780-\U0001F7FF" # Geometric Shapes Extended
"\U0001F800-\U0001F8FF" # Supplemental Arrows-C
"\U0001F900-\U0001F9FF" # Supplemental Symbols and Pictographs
"\U0001FA00-\U0001FA6F" # Chess Symbols
"\U0001FA70-\U0001FAFF" # Symbols and Pictographs Extended-A
"\U00002702-\U000027B0" # Dingbats
"\U000024C2-\U0001F251"
"]+",
flags=re.UNICODE
)
return emoji_pattern.sub(r'', text)
# Apply the function to the 'transcript' column
comments['Comment'] = comments['Comment'].apply(remove_emojis)
# Print the DataFrame to verify
#print(videos_df)
Run to view results
import nltk
from textblob import TextBlob
from sklearn.feature_extraction.text import TfidfVectorizer
# Example function to calculate sentiment
def calculate_sentiment(transcript):
if not transcript:
return 0
else:
blob = TextBlob(transcript)
return blob.sentiment.polarity
# Apply sentiment analysis
comments['sentiment'] = comments['Comment'].fillna("").astype('str').apply(calculate_sentiment)
# Example function for keyword extraction using TF-IDF
def extract_keywords(transcript):
print(transcript)
if not transcript:
return [] # Return an empty list for empty transcripts
vectorizer = TfidfVectorizer(stop_words='english', max_features=5)
try:
tfidf_matrix = vectorizer.fit_transform([transcript])
keywords = vectorizer.get_feature_names_out()
return list(keywords)
except ValueError:
return []
# Apply keyword extraction
#extract_keywords(comments['Comment'][1])
comments['keywords'] = comments['Comment'].fillna("").astype('str').apply(extract_keywords)
Run to view results
comments.head()
Run to view results
comments_group_df=comments.groupby(['Video ID','Sentiment']).agg(
Comments=('Comment','count'),
likes=('Likes','sum')
)
Run to view results
comments_group_df.reset_index(inplace=True)
Run to view results
comments_group_df.head()
Run to view results
comments_group_df=pd.pivot_table(comments_group_df,index='Video ID',values=['Comments','likes'],columns='Sentiment',aggfunc='sum',fill_value=0)
Run to view results
comments_group_df.columns=['Negative_comments',
'Neutral_comments',
'Positive_comments',
'Negative_likes',
'Neutral_likes',
'Positive_likes']
Run to view results
comments_group_df.reset_index(inplace=True)
Run to view results
comments.shape
Run to view results
comments_group_df.shape
Run to view results
#comments_group_df=pd.merge(comments_group_df,comments[['Video ID','keywords']],how='inner',on='Video ID')
Run to view results
comments_group_df=pd.merge(comments_group_df,videos_stats,how='inner',on='Video ID')
Run to view results
comments_group_df.head()
Run to view results
print(f'no of videos',comments['Video ID'].nunique())
print(f'no of videos',comments_group_df['Video ID'].nunique())
print(f'no of videos',videos_stats['Video ID'].nunique())
print(f'shape of video_stats',videos_stats.shape)
Run to view results
Adding features
from datetime import datetime
comments_group_df['recency']=(datetime.now()-comments_group_df['Published At']).dt.days
Run to view results
comments_group_df['comments_per_days']=comments_group_df['Comments']/comments_group_df['recency']
Run to view results
comments_group_df['comments_per_view']=comments_group_df['Comments']/comments_group_df['Views']
Run to view results
comments_group_df['views_per_days']=comments_group_df['Views']/comments_group_df['recency']
Run to view results
comments_group_df['likes_per_days']=comments_group_df['Likes']/comments_group_df['recency']
Run to view results
comments_group_df.drop(['Keyword','Title','Published At'],axis=1,inplace=True)
Run to view results
comments_group_df.columns.to_list()
Run to view results
comments_group_df.head()
Run to view results
def is_post_covid(x):
if x >2019 :
return 1
else :
return 0
comments_group_df['covid_era']=comments_group_df['year'].apply(is_post_covid)
Run to view results
def is_latest(x):
if x >=2022 :
return 1
else :
return 0
comments_group_df['latest_videos']=comments_group_df['year'].apply(is_latest)
Run to view results
comments_group_df.shape
Run to view results
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.histplot(x=comments_group_df['comments_per_days'], ax=axs[0, 0],bins=55)
sns.histplot(x=comments_group_df['comments_per_view'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['views_per_days'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
#comments_group_df.iloc[:,[12,13,14,15]].boxplot()
Run to view results
Run to view results
Run to view results
Run to view results
comments_group_df.head()
Run to view results
comments_group_df.drop('year',inplace=True,axis=1)
Run to view results
comments_group_df.set_index('Video ID',inplace=True)
Run to view results
data_dummy=pd.get_dummies(comments_group_df['Keyword_group'],dtype='int64')
Run to view results
comments_group_df=pd.concat([comments_group_df,data_dummy],axis=1)
Run to view results
comments_group_df.head()
Run to view results
comments_group_df.drop('Keyword_group',axis=1,inplace=True)
Run to view results
Data Preprocessing
First we will identify the skewed columns and then apply log transformation to reduce skewness
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.histplot(x=comments_group_df['Likes'], ax=axs[0, 0],bins=55)
sns.histplot(x=comments_group_df['Comments'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['Views'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
#comments_group_df.iloc[:,[12,13,14,15]].boxplot()
Run to view results
Apply log transformation
comments_group_df['Likes']=np.log(comments_group_df['Likes'])
Run to view results
comments_group_df['Comments']=np.log(comments_group_df['Comments'])
comments_group_df['Views']=np.log(comments_group_df['Views'])
comments_group_df['likes_per_days']=np.log(comments_group_df['likes_per_days'])
Run to view results
Checking Skewness
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.boxplot(y=comments_group_df['Likes'],x=comments_group_df['Education'].astype('str'),data=comments_group_df,ax=axs[0, 0])
sns.histplot(x=comments_group_df['Comments'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['Views'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
Run to view results
comments_group_df['comments_per_days']=np.log(comments_group_df['comments_per_days'])
comments_group_df['comments_per_view']=np.log(comments_group_df['comments_per_view'])
comments_group_df['views_per_days']=np.log(comments_group_df['views_per_days'])
Run to view results
Checking Skewness
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.histplot(x=comments_group_df['comments_per_days'], ax=axs[0, 0],bins=55)
sns.histplot(x=comments_group_df['comments_per_view'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['views_per_days'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
Run to view results
comments_group_df.describe()
Run to view results
comments_group_df['Likes']=comments_group_df['Likes'].replace(np.inf,np.percentile(comments_group_df['Likes'],99))
Run to view results
comments_group_df['Likes']=comments_group_df['Likes'].replace(-np.inf,np.percentile(comments_group_df['Likes'],5))
Run to view results
comments_group_df['Comments']=comments_group_df['Comments'].replace(np.inf,np.percentile(comments_group_df['Comments'],99))
comments_group_df['Comments']=comments_group_df['Comments'].replace(-np.inf,np.percentile(comments_group_df['Comments'],5))
Run to view results
comments_group_df['Views']=comments_group_df['Views'].replace(np.inf,np.percentile(comments_group_df['Views'],99))
comments_group_df['Views']=comments_group_df['Views'].replace(-np.inf,np.percentile(comments_group_df['Views'],5))
Run to view results
comments_group_df['comments_per_days']=comments_group_df['comments_per_days'].replace(np.inf,np.percentile(comments_group_df['comments_per_days'],99))
comments_group_df['comments_per_days']=comments_group_df['comments_per_days'].replace(-np.inf,np.percentile(comments_group_df['comments_per_days'],5))
Run to view results
comments_group_df['comments_per_view']=comments_group_df['comments_per_view'].replace(np.inf,np.percentile(comments_group_df['comments_per_view'],99))
comments_group_df['comments_per_view']=comments_group_df['comments_per_view'].replace(-np.inf,np.percentile(comments_group_df['comments_per_view'],5))
Run to view results
comments_group_df['likes_per_days']=comments_group_df['likes_per_days'].replace(np.inf,np.percentile(comments_group_df['likes_per_days'],99))
comments_group_df['likes_per_days']=comments_group_df['likes_per_days'].replace(-np.inf,np.percentile(comments_group_df['likes_per_days'],5))
Run to view results
comments_group_df['views_per_days']=comments_group_df['views_per_days'].replace(np.inf,np.percentile(comments_group_df['views_per_days'],99))
comments_group_df['views_per_days']=comments_group_df['views_per_days'].replace(-np.inf,np.percentile(comments_group_df['views_per_days'],5))
Run to view results
Normalization
Run to view results
from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
comments_group_df_normalized=scaler.fit_transform(comments_group_df)
Run to view results
comments_group_df_normalized=pd.DataFrame(comments_group_df_normalized,index=comments_group_df.index,columns=comments_group_df.columns.to_list())
Run to view results
comments_group_df_normalized.isna().sum()
Run to view results
comments_group_df_normalized.fillna(0,inplace=True)
Run to view results
comments_group_df_normalized.head()
Run to view results
Creating clusters
plt.figure(figsize=(8,5))
plt.title('Elbow method')
plt.xlabel('i')
plt.ylabel('SSE')
sns.pointplot(x=sse.keys(),y=sse.values())
Run to view results
kmeans=KMeans(n_clusters=26,random_state=1)
model=kmeans.fit(comments_group_df_normalized)
cluster=model.labels_
comments_group_df_normalized['Cluster']=cluster
Run to view results
comments_group_df_normalized.head()
Run to view results
Run to view results
pivot_cluster=pd.pivot_table(data=comments_group_df_normalized,index='Cluster',values=['Likes','Views','Comments','comments_per_view'],aggfunc='mean')
Run to view results
pivot_cluster=pivot_cluster.transpose()
Run to view results
#pivot_cluster.reset_index(inplace=True)
Run to view results
pivot_cluster
Run to view results
plt.figure(figsize=(20,5))
sns.heatmap(pivot_cluster.iloc[:,:],annot=True,fmt='.2g',cmap='coolwarm',)
Run to view results
Run to view results
premium_cluster=comments_group_df_normalized[(comments_group_df_normalized['Cluster']==5)|(comments_group_df_normalized['Cluster']==11)|(comments_group_df_normalized['Cluster']==25)]
Run to view results
premium_cluster.index.to_list()
Run to view results
comments_group_df_normalized.reset_index(inplace=True)
Run to view results
comments_group_df_normalized.head()
Run to view results
premium_videos=videos_stats[videos_stats['Video ID'].isin(premium_cluster.index.to_list())]
Run to view results
premium_videos
Run to view results
print(f'shape of video_stats',videos_stats.shape)
print(f'shape of comments_group_df_normalized',comments_group_df_normalized.shape)
Run to view results
videos_stats=pd.merge(videos_stats[['Video ID','Keyword','Keyword_group']],comments_group_df_normalized[['Video ID','Cluster']],how='inner',on='Video ID')
Run to view results
videos_stats.shape
Run to view results
Run to view results
premium_videos
Run to view results
pivot_cluster.reset_index(inplace=True)
Run to view results
Run to view results
education_videos=videos_stats[videos_stats['Cluster']==17]
Run to view results
education_videos
Run to view results
Trending_educational_videos=comments_group_df[comments_group_df['Video ID'].isin(education_videos['Video ID'])].sort_values(by='Positive_comments')
Run to view results
Trending_educational_videos.set_index('Video ID',inplace=True)
Run to view results
Trending_educational_videos['value']=Trending_educational_videos.sum(axis=1)
Run to view results
Trending_educational_videos.sort_values(by='value',inplace=True,ascending=False)
Run to view results
Trending_educational_videos.iloc[0:3,:]
Run to view results