# Let's import the packages
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
plt.style.use('ggplot')
from matplotlib.pyplot import figure
%matplotlib inline
matplotlib.rcParams['figure.figsize'] = (12, 8)
df_movies = pd.read_csv('/work/dataset/movies.csv')
df_movies.head(5)
nameobject
ratingobject
0
The Shining
R
1
The Blue Lagoon
R
2
Star Wars: Episode V - The Empire Strikes Back
PG
3
Airplane!
PG
4
Caddyshack
R
df_movies.columns
df_movies.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 7668 entries, 0 to 7667
Data columns (total 15 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 name 7668 non-null object
1 rating 7591 non-null object
2 genre 7668 non-null object
3 year 7668 non-null int64
4 released 7666 non-null object
5 score 7665 non-null float64
6 votes 7665 non-null float64
7 director 7668 non-null object
8 writer 7665 non-null object
9 star 7667 non-null object
10 country 7665 non-null object
11 budget 5497 non-null float64
12 gross 7479 non-null float64
13 company 7651 non-null object
14 runtime 7664 non-null float64
dtypes: float64(5), int64(1), object(9)
memory usage: 898.7+ KB
df_movies.isnull().any()
# Let's loop through the data and see the data missing
for col in df_movies.columns:
pt_missing = np.mean(df_movies[col].isnull())*100
print('{} - {}%'.format(col, round(pt_missing,3)))
name - 0.0%
rating - 1.004%
genre - 0.0%
year - 0.0%
released - 0.026%
score - 0.039%
votes - 0.039%
director - 0.0%
writer - 0.039%
star - 0.013%
country - 0.039%
budget - 28.312%
gross - 2.465%
company - 0.222%
runtime - 0.052%
# The column Budget has the most percentage of missing data
df_movies[df_movies['budget'].isnull()]
nameobject
Venom0.1%
Paradise0.1%
2154 others99.8%
ratingobject
R49.2%
10 others48.2%
Missing2.6%
147
The French Lieutenant's Woman
R
148
The Great Muppet Caper
G
152
Nice Dreams
R
153
Sharky's Machine
R
155
Puberty Blues
R
156
Eye of the Needle
R
158
Student Bodies
R
160
The Hand
R
161
Continental Divide
PG
162
Diva
R
#The Budget column is really important for the analysis. Even if is almost 30% of the data, We need to erased the null values
df_mov = df_movies.dropna(how='any', axis=0)
df_mov.dtypes
# Let's pull apart the column Release
df_mov['released'].astype(str).str.extract('([0-9]{4})')
df_mov['released_year'] = df_mov['released'].astype(str).str.extract('([0-9]{4})')
/shared-libs/python3.7/py-core/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
"""
df_mov['released'].astype(str).str.split(', ').str[0]
df_mov['released_date'] = df_mov['released'].astype(str).str.split(', ').str[0]
/shared-libs/python3.7/py-core/lib/python3.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
This is separate from the ipykernel package so we can avoid doing imports until
df_mov['released'].astype(str).str.split('(').str[1].str.split(')').str[0]
df_mov['released_country'] = df_mov['released'].astype(str).str.split('(').str[1].str.split(')').str[0]
/shared-libs/python3.7/py-core/lib/python3.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
This is separate from the ipykernel package so we can avoid doing imports until
# There is 708 rows where the Year doesn´t match the Release Year
df_mov['released_year'] = df_mov['released_year'].astype('float64')
df_mov[df_mov['released_year'] != df_mov['year']]
/shared-libs/python3.7/py-core/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
nameobject
Trust0.3%
Crash0.3%
704 others99.4%
ratingobject
R64.3%
PG-1319.8%
5 others16%
8
Superman II
PG
11
The Gods Must Be Crazy
PG
21
Heaven's Gate
R
33
Cattle Annie and Little Britches
PG
96
The Evil Dead
NC-17
97
Porky's
R
101
Mad Max 2
R
118
Chariots of Fire
PG
129
Cutter's Way
R
130
Quest for Fire
R
# pd.set_options('display.max_rows', None)
df_mov.drop_duplicates() # there is no duplicates
nameobject
Hercules0.1%
Friday the 13th0%
5334 others99.9%
ratingobject
R47.9%
PG-1331.9%
8 others20.2%
0
The Shining
R
1
The Blue Lagoon
R
2
Star Wars: Episode V - The Empire Strikes Back
PG
3
Airplane!
PG
4
Caddyshack
R
5
Friday the 13th
R
6
The Blues Brothers
R
7
Raging Bull
R
8
Superman II
PG
9
The Long Riders
R
df_mov.boxplot(column=['gross'])
# Looking at the top 15 compaies by gross revenue
company_gross_sum = df_mov.groupby('company')[['gross']].sum()
company_gross_sum_s = company_gross_sum.sort_values('gross', ascending = False)[:15]
company_gross_sum_s = company_gross_sum_s['gross'].astype('int64')
company_gross_sum_s
sns.swarmplot(x="rating", y="gross", data=df_mov)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 93.8% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 77.6% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 23.4% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 63.6% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 84.0% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 35.3% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
# Order our Data
df_mov.sort_values(by=['gross'], inplace=False, ascending=False)
nameobject
Hercules0.1%
The Lion King0%
5334 others99.9%
ratingobject
R47.9%
PG-1331.9%
8 others20.2%
5445
Avatar
PG-13
7445
Avengers: Endgame
PG-13
3045
Titanic
PG-13
6663
Star Wars: Episode VII - The Force Awakens
PG-13
7244
Avengers: Infinity War
PG-13
7480
The Lion King
PG
6653
Jurassic World
PG-13
6043
The Avengers
PG-13
6646
Furious 7
PG-13
7494
Frozen II
PG
#Let's star looking at correlation
sns.pairplot(df_mov)
# Correlation Matrix between all numeric columns
df_mov.corr(method='pearson')
yearfloat64
scorefloat64
year
1
0.05638566436
score
0.05638566436
1
votes
0.2060205008
0.4742562482
budget
0.3277223852
0.07200065706
gross
0.2743211831
0.2225563191
runtime
0.07507715754
0.414067722
released_year
0.9987260945
0.06192340792
df_mov.corr(method='kendall')
yearfloat64
scorefloat64
year
1
0.0393889366
score
0.0393889366
1
votes
0.2965122959
0.3501848539
budget
0.220833391
-0.006406337548
gross
0.2395390682
0.1249428892
runtime
0.06482449328
0.2922536851
released_year
0.9913039886
0.04340033503
df_mov.corr(method='spearman')
yearfloat64
scorefloat64
year
1
0.05774089411
score
0.05774089411
1
votes
0.42762273
0.495408967
budget
0.3128858939
-0.009971091368
gross
0.3510454044
0.1831915822
runtime
0.09544409823
0.412155326
released_year
0.9986935252
0.06367381353
# Correlation heatmap
sns.heatmap(df_mov.corr(), annot=True)
plt.title('Correlation Matrix for Numeric Features')
plt.xlabel('Movie Features')
plt.ylabel('Movie Features')
plt.show()
# Scatter plot with Budget vs Gross
plt.scatter(x=df_mov['budget'], y=df_mov['gross'])
plt.title('Budget vs Gross Earnings')
plt.ylabel('Gross Earnings')
plt.xlabel('Budget for Film')
plt.show()
sns.regplot(x='budget', y='gross', data=df_mov, scatter_kws={'color': 'red'}, line_kws={'color': 'blue'})
sns.regplot(x='score', y='gross', data=df_mov, scatter_kws={'color': 'red'}, line_kws={'color': 'blue'})
df_mov.head()
nameobject
ratingobject
0
The Shining
R
1
The Blue Lagoon
R
2
Star Wars: Episode V - The Empire Strikes Back
PG
3
Airplane!
PG
4
Caddyshack
R
df_mov_c = df_mov.copy(deep=False)
for col_name in df_mov_c.columns:
if(df_mov_c[col_name].dtype == 'object'):
df_mov_c[col_name] = df_mov_c[col_name].astype('category')
df_mov_c[col_name] = df_mov_c[col_name].cat.codes
df_mov_c.head()
nameint16
ratingint8
0
4692
6
1
3929
6
2
3641
4
3
204
4
4
732
6
sns.heatmap(df_mov_c.corr(), annot=True)
plt.title('Correlation Matrix for All Features')
plt.xlabel('Movie Features')
plt.ylabel('Movie Features')
plt.show()
corr_mat = df_mov_c.corr()
corr_pairs = corr_mat.unstack()
print(corr_pairs)
sorted_pairs = corr_pairs.sort_values(kind='quicksort')
print(sorted_pairs)
# We can now take a look at the ones that have a high correlation (> 0.5)
strong_pairs = sorted_pairs[abs(sorted_pairs) > 0.5]
print(strong_pairs)
votes gross 0.614751
gross votes 0.614751
budget gross 0.740247
gross budget 0.740247
released_year year 0.998726
year released_year 0.998726
released_date released 0.999839
released released_date 0.999839
name name 1.000000
released_year released_year 1.000000
rating rating 1.000000
genre genre 1.000000
year year 1.000000
released released 1.000000
score score 1.000000
votes votes 1.000000
released_date released_date 1.000000
director director 1.000000
star star 1.000000
country country 1.000000
budget budget 1.000000
gross gross 1.000000
company company 1.000000
runtime runtime 1.000000
writer writer 1.000000
released_country released_country 1.000000
dtype: float64