import pandas as pd
import numpy as np
videos_stats = pd.read_csv('videos_stats.csv')
videos_stats
comments = pd.read_csv('comments.csv')
comments
print(f'shape of video_stats',videos_stats.shape)
print(f'no of unique videos',videos_stats['Video ID'].nunique())
print(f'no of unique videos',comments['Video ID'].nunique())
identify duplicate records
duplicate_records_ids
videos_stats=videos_stats[~videos_stats['Video ID'].isin(duplicate_records_ids)]
comments=comments[~comments['Video ID'].isin(duplicate_records_ids)]
print(f'shape of video_stats',videos_stats.shape)
print(f'no of unique videos',videos_stats['Video ID'].nunique())
print(f'no of unique videos',comments['Video ID'].nunique())
Identify missing values
videos_stats.isna().sum()
videos_stats.fillna(0,inplace=True)
comments.isna().sum()
comments.dropna(inplace=True)
Creating a combined Dataset
df_combined=pd.merge(comments,videos_stats,how='left',on='Video ID')
df_combined.head()
Creating groups of video contents
def industry(text):
if text in ["biology","cnn","computer science","data science","education","interview","machine learning","mathchemistry","physics"]:
group="Education"
elif text in ["lofi","marvel","movies","mrbeast","music","reaction","trolling"]:
group='Entertainment'
elif text in ["chess","game development","gaming","minecraft","sports","cubes"]:
group='Gaming'
elif text in ["apple","food"]:
group='Health'
elif text in ["animals","asmr","bed","mukbang","sat"]:
group='Lifestyle'
elif text in ["history",'literature','news']:
group='News & Politics'
elif text in ['crypto','google','how-to','nintendo','tech','tutorial','xbox']:
group='Technology'
else :
group='Business & Finance'
return group
videos_stats['Keyword_group']=videos_stats['Keyword'].apply(industry)
df_combined['Keyword_group']=df_combined['Keyword'].apply(industry)
videos_stats.head()
EDA
import matplotlib.pyplot as plt
import seaborn as sns
Creating pivot tables
table1=pd.pivot_table(data=videos_stats,values='Video ID',index='Keyword_group',aggfunc='nunique',columns='year',fill_value=0)
#print(table1)
table2=videos_stats.groupby(['year','Keyword_group']).agg(
No_of_Videos=('Video ID', 'nunique'),
No_of_views_in_k=('Views','sum'),
No_of_comments_in_k=('Comments','sum'),
No_of_likes_in_k=('Likes','sum')
)
table2['No_of_views_in_k']=table2['No_of_views_in_k']/1000
table2['No_of_comments_in_k']=table2['No_of_comments_in_k']/1000
table2['No_of_likes_in_k']=table2['No_of_likes_in_k']/1000
table2['views_per_video']=table2['No_of_views_in_k']/table2['No_of_Videos']
table2['comments_per_video']=table2['No_of_comments_in_k']/table2['No_of_Videos']
table2['likes_per_video']=table2['No_of_likes_in_k']/table2['No_of_Videos']
table2.reset_index(inplace=True)
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of videos in each industry per year')
sns.heatmap(table1,cmap='coolwarm',annot=True,fmt='.2g')
Visuals displaying no of videos
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Visuals showing views and likes of videos
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Visuals showing trending contents
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Visuals showing total views vs views per video
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Heatmaps to show popular contents before and after covid
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of views in each industry in after covid')
sns.heatmap(table3,cmap='plasma',annot=False,fmt='.2g',square=True,cbar=True)
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of likes in each industry after covid')
sns.heatmap(table4,cmap='seismic',annot=False,fmt='.2g',square=True,cbar=True)
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of likes in each industry before covid')
sns.heatmap(table5,cmap='magma',annot=False,fmt='.2g',square=True,cbar=True)
table6=pd.pivot_table(data=videos_stats[(videos_stats['year']<=2020) ],values='Views',index='Keyword_group',aggfunc='sum',columns='Keyword',fill_value=0)
table6=table6/1000000
#print(table4)
plt.figure(figsize=(20,10))
plt.title('Heatmap showing no of views in each industry before covid')
sns.heatmap(table6,cmap='bwr',annot=False,fmt='.2g',square=True,cbar=True)
Challenge 1 : Summary
The no videos on youtube has increased exponentially in 2022 speacially techology and entertainment
Videos reated to entertainment has peaked trend post covid.
Under entertainment , trending video are related to marvel,mrbeast and music
Before covid , people liked and viewed mostly videos related to lifestyle and technology .
Sentiment Analysis of Video Comments
comments['Comment']=comments['Comment'].astype('str')
import re
###removing emojis
def remove_emojis(text):
emoji_pattern = re.compile(
"["
"\U0001F600-\U0001F64F" # emoticons
"\U0001F300-\U0001F5FF" # symbols & pictographs
"\U0001F680-\U0001F6FF" # transport & map symbols
"\U0001F700-\U0001F77F" # alchemical symbols
"\U0001F780-\U0001F7FF" # Geometric Shapes Extended
"\U0001F800-\U0001F8FF" # Supplemental Arrows-C
"\U0001F900-\U0001F9FF" # Supplemental Symbols and Pictographs
"\U0001FA00-\U0001FA6F" # Chess Symbols
"\U0001FA70-\U0001FAFF" # Symbols and Pictographs Extended-A
"\U00002702-\U000027B0" # Dingbats
"\U000024C2-\U0001F251"
"]+",
flags=re.UNICODE
)
return emoji_pattern.sub(r'', text)
# Apply the function to the 'transcript' column
comments['Comment'] = comments['Comment'].apply(remove_emojis)
# Print the DataFrame to verify
#print(videos_df)
import nltk
from textblob import TextBlob
from sklearn.feature_extraction.text import TfidfVectorizer
# Example function to calculate sentiment
def calculate_sentiment(transcript):
if not transcript:
return 0
else:
blob = TextBlob(transcript)
return blob.sentiment.polarity
# Apply sentiment analysis
comments['sentiment'] = comments['Comment'].fillna("").astype('str').apply(calculate_sentiment)
# Example function for keyword extraction using TF-IDF
def extract_keywords(transcript):
print(transcript)
if not transcript:
return [] # Return an empty list for empty transcripts
vectorizer = TfidfVectorizer(stop_words='english', max_features=5)
try:
tfidf_matrix = vectorizer.fit_transform([transcript])
keywords = vectorizer.get_feature_names_out()
return list(keywords)
except ValueError:
return []
# Apply keyword extraction
#extract_keywords(comments['Comment'][1])
comments['keywords'] = comments['Comment'].fillna("").astype('str').apply(extract_keywords)
comments.head()
comments_group_df=comments.groupby(['Video ID','Sentiment']).agg(
Comments=('Comment','count'),
likes=('Likes','sum')
)
comments_group_df.reset_index(inplace=True)
comments_group_df.head()
comments_group_df=pd.pivot_table(comments_group_df,index='Video ID',values=['Comments','likes'],columns='Sentiment',aggfunc='sum',fill_value=0)
comments_group_df.columns=['Negative_comments',
'Neutral_comments',
'Positive_comments',
'Negative_likes',
'Neutral_likes',
'Positive_likes']
comments_group_df.reset_index(inplace=True)
comments.shape
comments_group_df.shape
#comments_group_df=pd.merge(comments_group_df,comments[['Video ID','keywords']],how='inner',on='Video ID')
comments_group_df=pd.merge(comments_group_df,videos_stats,how='inner',on='Video ID')
comments_group_df.head()
print(f'no of videos',comments['Video ID'].nunique())
print(f'no of videos',comments_group_df['Video ID'].nunique())
print(f'no of videos',videos_stats['Video ID'].nunique())
print(f'shape of video_stats',videos_stats.shape)
Adding features
from datetime import datetime
comments_group_df['recency']=(datetime.now()-comments_group_df['Published At']).dt.days
comments_group_df['comments_per_days']=comments_group_df['Comments']/comments_group_df['recency']
comments_group_df['comments_per_view']=comments_group_df['Comments']/comments_group_df['Views']
comments_group_df['views_per_days']=comments_group_df['Views']/comments_group_df['recency']
comments_group_df['likes_per_days']=comments_group_df['Likes']/comments_group_df['recency']
comments_group_df.drop(['Keyword','Title','Published At'],axis=1,inplace=True)
comments_group_df.columns.to_list()
comments_group_df.head()
def is_post_covid(x):
if x >2019 :
return 1
else :
return 0
comments_group_df['covid_era']=comments_group_df['year'].apply(is_post_covid)
def is_latest(x):
if x >=2022 :
return 1
else :
return 0
comments_group_df['latest_videos']=comments_group_df['year'].apply(is_latest)
comments_group_df.shape
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.histplot(x=comments_group_df['comments_per_days'], ax=axs[0, 0],bins=55)
sns.histplot(x=comments_group_df['comments_per_view'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['views_per_days'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
#comments_group_df.iloc[:,[12,13,14,15]].boxplot()
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
Run the app to see this chart
Press the run button in the top right corner
comments_group_df.head()
comments_group_df.drop('year',inplace=True,axis=1)
comments_group_df.set_index('Video ID',inplace=True)
data_dummy=pd.get_dummies(comments_group_df['Keyword_group'],dtype='int64')
comments_group_df=pd.concat([comments_group_df,data_dummy],axis=1)
comments_group_df.head()
comments_group_df.drop('Keyword_group',axis=1,inplace=True)
Data Preprocessing
First we will identify the skewed columns and then apply log transformation to reduce skewness
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.histplot(x=comments_group_df['Likes'], ax=axs[0, 0],bins=55)
sns.histplot(x=comments_group_df['Comments'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['Views'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
#comments_group_df.iloc[:,[12,13,14,15]].boxplot()
Apply log transformation
comments_group_df['Likes']=np.log(comments_group_df['Likes'])
comments_group_df['Comments']=np.log(comments_group_df['Comments'])
comments_group_df['Views']=np.log(comments_group_df['Views'])
comments_group_df['likes_per_days']=np.log(comments_group_df['likes_per_days'])
Checking Skewness
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.boxplot(y=comments_group_df['Likes'],x=comments_group_df['Education'].astype('str'),data=comments_group_df,ax=axs[0, 0])
sns.histplot(x=comments_group_df['Comments'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['Views'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
comments_group_df['comments_per_days']=np.log(comments_group_df['comments_per_days'])
comments_group_df['comments_per_view']=np.log(comments_group_df['comments_per_view'])
comments_group_df['views_per_days']=np.log(comments_group_df['views_per_days'])
Checking Skewness
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
sns.histplot(x=comments_group_df['comments_per_days'], ax=axs[0, 0],bins=55)
sns.histplot(x=comments_group_df['comments_per_view'], ax=axs[0, 1],bins=55)
sns.histplot(x=comments_group_df['views_per_days'], ax=axs[1, 0],bins=55)
sns.histplot(x=comments_group_df['likes_per_days'], ax=axs[1, 1],bins=55)
comments_group_df.describe()
comments_group_df['Likes']=comments_group_df['Likes'].replace(np.inf,np.percentile(comments_group_df['Likes'],99))
comments_group_df['Likes']=comments_group_df['Likes'].replace(-np.inf,np.percentile(comments_group_df['Likes'],5))
comments_group_df['Comments']=comments_group_df['Comments'].replace(np.inf,np.percentile(comments_group_df['Comments'],99))
comments_group_df['Comments']=comments_group_df['Comments'].replace(-np.inf,np.percentile(comments_group_df['Comments'],5))
comments_group_df['Views']=comments_group_df['Views'].replace(np.inf,np.percentile(comments_group_df['Views'],99))
comments_group_df['Views']=comments_group_df['Views'].replace(-np.inf,np.percentile(comments_group_df['Views'],5))
comments_group_df['comments_per_days']=comments_group_df['comments_per_days'].replace(np.inf,np.percentile(comments_group_df['comments_per_days'],99))
comments_group_df['comments_per_days']=comments_group_df['comments_per_days'].replace(-np.inf,np.percentile(comments_group_df['comments_per_days'],5))
comments_group_df['comments_per_view']=comments_group_df['comments_per_view'].replace(np.inf,np.percentile(comments_group_df['comments_per_view'],99))
comments_group_df['comments_per_view']=comments_group_df['comments_per_view'].replace(-np.inf,np.percentile(comments_group_df['comments_per_view'],5))
comments_group_df['likes_per_days']=comments_group_df['likes_per_days'].replace(np.inf,np.percentile(comments_group_df['likes_per_days'],99))
comments_group_df['likes_per_days']=comments_group_df['likes_per_days'].replace(-np.inf,np.percentile(comments_group_df['likes_per_days'],5))
comments_group_df['views_per_days']=comments_group_df['views_per_days'].replace(np.inf,np.percentile(comments_group_df['views_per_days'],99))
comments_group_df['views_per_days']=comments_group_df['views_per_days'].replace(-np.inf,np.percentile(comments_group_df['views_per_days'],5))
Normalization
from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
comments_group_df_normalized=scaler.fit_transform(comments_group_df)
comments_group_df_normalized=pd.DataFrame(comments_group_df_normalized,index=comments_group_df.index,columns=comments_group_df.columns.to_list())
comments_group_df_normalized.isna().sum()
comments_group_df_normalized.fillna(0,inplace=True)
comments_group_df_normalized.head()
Creating clusters
plt.figure(figsize=(8,5))
plt.title('Elbow method')
plt.xlabel('i')
plt.ylabel('SSE')
sns.pointplot(x=sse.keys(),y=sse.values())
kmeans=KMeans(n_clusters=26,random_state=1)
model=kmeans.fit(comments_group_df_normalized)
cluster=model.labels_
comments_group_df_normalized['Cluster']=cluster
comments_group_df_normalized.head()
Run the app to see this chart
Press the run button in the top right corner
pivot_cluster=pd.pivot_table(data=comments_group_df_normalized,index='Cluster',values=['Likes','Views','Comments','comments_per_view'],aggfunc='mean')
pivot_cluster=pivot_cluster.transpose()
#pivot_cluster.reset_index(inplace=True)
pivot_cluster
plt.figure(figsize=(20,5))
sns.heatmap(pivot_cluster.iloc[:,:],annot=True,fmt='.2g',cmap='coolwarm',)
Run the app to see this chart
Press the run button in the top right corner
premium_cluster=comments_group_df_normalized[(comments_group_df_normalized['Cluster']==5)|(comments_group_df_normalized['Cluster']==11)|(comments_group_df_normalized['Cluster']==25)]
premium_cluster.index.to_list()
comments_group_df_normalized.reset_index(inplace=True)
comments_group_df_normalized.head()
premium_videos=videos_stats[videos_stats['Video ID'].isin(premium_cluster.index.to_list())]
premium_videos
print(f'shape of video_stats',videos_stats.shape)
print(f'shape of comments_group_df_normalized',comments_group_df_normalized.shape)
videos_stats=pd.merge(videos_stats[['Video ID','Keyword','Keyword_group']],comments_group_df_normalized[['Video ID','Cluster']],how='inner',on='Video ID')
videos_stats.shape
Run the app to see this chart
Press the run button in the top right corner
premium_videos
pivot_cluster.reset_index(inplace=True)
Run the app to see this chart
Press the run button in the top right corner
education_videos=videos_stats[videos_stats['Cluster']==17]
education_videos
Trending_educational_videos=comments_group_df[comments_group_df['Video ID'].isin(education_videos['Video ID'])].sort_values(by='Positive_comments')
Trending_educational_videos.set_index('Video ID',inplace=True)
Trending_educational_videos['value']=Trending_educational_videos.sum(axis=1)
Trending_educational_videos.sort_values(by='value',inplace=True,ascending=False)
Trending_educational_videos.iloc[0:3,:]