Scraping the r/soccer subreddit to understand the post history of 'Big 6' PL clubs in 2022

Downloading the required data from r/soccer

Importing the essential libraries

import praw from psaw import PushshiftAPI import pandas as pd import datetime import os import numpy as np import matplotlib.pyplot as plt

Using the Reddit app credentials to create a connection

reddit = praw.Reddit(client_id = 'Afv7rdJdpRD3WdRHRvp1aA', client_secret = 'TboLbC6GJZ5ieMe2G4fCcip4d_gt1w', user_agent = 'Soccer Subreddit Web scraper by u/portgas_d_bass') api = PushshiftAPI(reddit)

Doing keyword search for the Big 6 PL clubs in r/soccer

start_time = int(datetime.datetime(2022,1,1).timestamp()) end_time = int(datetime.datetime(2022,9,1).timestamp()) gen = api.search_submissions(q = 'Arsenal|Chelsea|Manchester|Liverpool|Tottenham', after = start_time, before = end_time, subreddit = 'soccer')

Creating a filepath and folder structure to store the data

data_folder = './Scraped_Files_Reddit/' file_name = 'Reddit_Soccer_Data_2022' # Creating a file path if it doesn't already exist check_folder = os.path.isdir(data_folder) if not check_folder: os.makedirs(data_folder) print('Created Folder: ', data_folder)

Creating a pandas dataframe and loading the subreddit data into a csv file

final_data = [] header_list = ['id', 'title', 'author', 'url', 'num_upvotes', 'num_comments', 'num_views', 'hour', 'day', 'month', 'year'] for submission in gen: dt_object = datetime.datetime.fromtimestamp(submission.created_utc) val = [submission.id, submission.title, submission.author, submission.url, submission.score, submission.num_comments, submission.view_count, dt_object.hour, dt_object.day, dt_object.month, dt_object.year] final_data.append(val) df = pd.DataFrame(final_data) df.to_csv(f'{data_folder}{file_name}.csv', encoding = 'UTF-8-sig', index = False, header = header_list)

Final Code

import praw from psaw import PushshiftAPI import pandas as pd import datetime import os import numpy as np import matplotlib.pyplot as plt reddit = praw.Reddit(client_id = 'Afv7rdJdpRD3WdRHRvp1aA', client_secret = 'TboLbC6GJZ5ieMe2G4fCcip4d_gt1w', user_agent = 'Soccer Subreddit Web scraper by u/portgas_d_bass') api = PushshiftAPI(reddit) start_time = int(datetime.datetime(2022,1,1).timestamp()) end_time = int(datetime.datetime(2022,9,1).timestamp()) gen = api.search_submissions(q = 'Arsenal|Chelsea|Manchester|Liverpool|Tottenham', after = start_time, before = end_time, subreddit = 'soccer') data_folder = './Scraped_Files_Reddit/' file_name = 'Reddit_Soccer_Data_2022' check_folder = os.path.isdir(data_folder) if not check_folder: os.makedirs(data_folder) print('Created Folder: ', data_folder) final_data = [] header_list = ['id', 'title', 'author', 'url', 'num_upvotes', 'num_comments', 'num_views', 'hour', 'day', 'month', 'year'] for submission in gen: dt_object = datetime.datetime.fromtimestamp(submission.created_utc) val = [submission.id, submission.title, submission.author, submission.url, submission.score, submission.num_comments, submission.view_count, dt_object.hour, dt_object.day, dt_object.month, dt_object.year] final_data.append(val) df = pd.DataFrame(final_data) df.columns = header_list df.to_csv(f'{data_folder}{file_name}.csv', encoding = 'UTF-8-sig', index = False)

Data Deep-dive using Pandas and Matplotlib

Prelim Checks

df.head()

print('Total number of records: ',len(df.index)) print('Column names are: ', df.columns.tolist()) print('Monthly post counts in 2022 are: \n', df.groupby('month')['id'].count()) print('Average number of upvotes are: ', df['num_upvotes'].mean()) print('Average number of comments are: ', df['num_comments'].mean()) print('Median number of upvotes are: ', df['num_upvotes'].median()) print('Median number of comments are: ', df['num_comments'].median())

Creating a flag for each record based on the PL club

df['title1'] = df['title'].str.lower() pl_clubs = ['Arsenal', 'Chelsea', 'Manchester United', 'Manchester City', 'Liverpool', 'Tottenham Hotspur'] for club in pl_clubs: df[f'{club} Flag'] = np.where(df['title1'].str.count(club.lower()) > 0, 1, 0)

for club in pl_clubs: print(df.groupby(f'{club} Flag')['id'].count())

Creating a date column

df['date'] = pd.to_datetime(df[['day', 'month', 'year']])

df.head()

Plotting the data using Matplotlib

fig, ax = plt.subplots(figsize = (12,6)) data = df.groupby('month')['id'].count() data.plot.bar() ax.set_xticklabels(labels = [1,2,3,4,5,6,7,8], rotation = 0, size = 'medium') ax.bar_label(ax.containers[0]) ax.set_xlabel('Month', rotation=0, labelpad=10) ax.set_ylabel('# of Posts', rotation=90, labelpad=10) plt.title("MoM Reddit posts for the PL 'Big 6' in 2022", fontsize=14) plt.show()

The highest number of posts are understandably coming from the month of Aug'22 which is the final month of the summer transfer window, while you see peaks during May'22 and Jan'22. The peak at May'22 can be attributed to the end of the PL season while the Jan'22 number can be explained by the Winter transfer window.

from collections import defaultdict club_level_count = defaultdict(int) for club in pl_clubs: club_level_count[club] = df[f'{club} Flag'].sum() club_level_count = {k:v for k,v in sorted(club_level_count.items(), key = lambda a: a[1], reverse = True)} fig, ax = plt.subplots(figsize = (12,6)) bars = ax.bar(club_level_count.keys(), club_level_count.values()) ax.bar_label(bars) ax.set_xlabel('Club Name', rotation=0, labelpad=10) ax.set_ylabel('# of Posts', rotation=90, labelpad=10) plt.title("Club wise Reddit posts for the PL 'Big 6' in 2022", fontsize=14) plt.show()

We can clearly see that Chelsea had the most number of reddit posts in r/soccer in the time period we are considering while Tottenham had the least. Manchester United, Liverpool and Arsenal all have a similar post count while Manchester City is in the bottom 2. One thing to look at here would be the number of posts that were from the Summer transfer window to see how that impacted the overall post count as Chelsea and Manchester United both had a very busy transfer window compared to the other clubs (By busy, I mean the number of players that these two clubs were linked with was relatively higher than the other 4 clubs)

from collections import defaultdict club_level_count_tw = defaultdict(int) for club in pl_clubs: club_level_count_tw[club] = df[(df[f'{club} Flag'] == 1) & (df['date'] >= '2022-06-10') & (df['date'] <= '2022-08-31')]['id'].count() club_level_count_tw = {k:v for k,v in sorted(club_level_count_tw.items(), key = lambda a: a[1], reverse = True)} fig, ax = plt.subplots(figsize = (12,6)) bars = ax.bar(club_level_count_tw.keys(), club_level_count_tw.values()) ax.bar_label(bars) ax.set_xlabel('Club Name', rotation=0, labelpad=10) ax.set_ylabel('# of Posts', rotation=90, labelpad=10) plt.title("Total number of Reddit posts for the PL 'Big 6' in 2022 during the summer transfer window", fontsize=14) plt.show()

As you can see, for Chelsea and Manchester United, ~50% of the posts came during the Summer transfer window 10 June - 31 August) which in turn has increased their overall post count and hence became a major factor in them being in the top 2. We could see a better illustration of this point in the chart below.

club_level_count_tw_perc = {} for key,val in club_level_count_tw.items(): club_level_count_tw_perc[key] = round((val*100)/club_level_count[key],2) club_level_count_tw_perc = {k:v for k,v in sorted(club_level_count_tw_perc.items(), key = lambda a: a[1], reverse = False)} fig, ax = plt.subplots(figsize = (12,6)) pl_club = list(club_level_count_tw_perc.keys()) ax.barh(y = pl_club, width = club_level_count_tw_perc.values()) ax.bar_label(ax.containers[0]) ax.set_xlabel('% of Posts', rotation=0, labelpad=10) ax.set_ylabel('Club Name', rotation=90, labelpad=10) plt.title("Percentage of Reddit posts for the PL 'Big 6' in 2022 coming from the summer transfer window", fontsize=14) plt.show()

As mentioned above, Manchester United and Chelsea have ~50% of their posts in 2022 coming during the summer transfer window, while Liverpool are at the bottom with only 27% of their posts coming from the transfer window. Although there were friendlies and the first few matches of the new season also being part of this period, posts related to them would be almost identical for all the clubs and hence the excess can be quite clearly attributed to the transfer activity for the respective clubs.

print(df['num_upvotes'].skew()) print(df['num_comments'].skew())

Since both the number of upvotes and comments are positively skewed, I am showing the median number of upvotes and comments for all the PL clubs in 2022 to better understand the level of interactions happening across clubs.

from collections import defaultdict club_level_median_upvotes = defaultdict(int) for club in pl_clubs: club_level_median_upvotes[club] = df[df[f'{club} Flag'] == 1]['num_upvotes'].median() club_level_median_upvotes = {k:v for k,v in sorted(club_level_median_upvotes.items(), key = lambda a: a[1], reverse = True)} fig, ax = plt.subplots(figsize = (12,6)) bars = ax.bar(club_level_median_upvotes.keys(), club_level_median_upvotes.values()) ax.bar_label(bars) ax.set_xlabel('Club Name', rotation=0, labelpad=10) ax.set_ylabel('Median # of Upvotes', rotation=90, labelpad=10) plt.title("Median Number of upvotes per reddit post for the PL 'Big 6' in 2022", fontsize=14) plt.show()

from collections import defaultdict club_level_median_comments = defaultdict(int) for club in pl_clubs: club_level_median_comments[club] = df[df[f'{club} Flag'] == 1]['num_comments'].median() club_level_median_comments = {k:v for k,v in sorted(club_level_median_comments.items(), key = lambda a: a[1], reverse = True)} fig, ax = plt.subplots(figsize = (12,6)) bars = ax.bar(club_level_median_comments.keys(), club_level_median_comments.values()) ax.bar_label(bars) ax.set_xlabel('Club Name', rotation=0, labelpad=10) ax.set_ylabel('Median # of Comments', rotation=90, labelpad=10) plt.title("Median Number of comments per reddit post for the PL 'Big 6' in 2022", fontsize=14) plt.show()

Despite being in the bottom in terms of total number of posts, Tottenham has the highest median number of upvotes and comments per post in 2022, which signals decent number of interactions in most of the posts involving the club in r/soccer.