import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn as sk
csvUrl = 'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'
colNames = ['Sepal Length','Sepal Width','Petal Length','Petal Width','Class']
irisData = pd.read_csv(csvUrl, names = colNames)
irisData.std()
def minmaxnorm(trData,teData,minV = 0,maxV = 1):
temp = pd.DataFrame(data = trData)
# creating a copy so as not to globally updata the input dataframe
trDataLocal = temp.copy()
check = False # this will be useful to check whether or not we have test data later on
if isinstance(teData, pd.DataFrame):
check = True
teDataLocal = teData.copy()
elif isinstance(teData,type(None)):
teDataLocal = None
else:
check = True
teData = np.array(teData)
if np.ndim(teData) == 1:
temp = np.zeros((1,len(teData)))
for i in range(len(teData)):
temp[0,i] = teData[i]
teData = temp
teDataLocal = pd.DataFrame(data=teData)
trData_numeric = trDataLocal.select_dtypes(include = np.number) # only use numeric data (we cant normalize names)
if check:
teData_numeric = teDataLocal.select_dtypes(include = np.number)
for i in range(len(trData_numeric.columns)):
# looping through thorugh the numeric columns
# First make the data between 0 and 1, then multiply so that it is the right width, then shift into place
trDataLocal.iloc[:,i] = (trData_numeric.iloc[:,i] - trData_numeric.iloc[:,i].min())
trDataLocal.iloc[:,i] = (trDataLocal.iloc[:,i].div(trDataLocal.iloc[:,i].max())).mul(maxV - minV)
trDataLocal.iloc[:,i] = (trDataLocal.iloc[:,i] + minV)
if check:
teDataLocal.iloc[:,i] = (teData_numeric.iloc[:,i] - trData_numeric.iloc[:,i].min())
teDataLocal.iloc[:,i] = (teDataLocal.iloc[:,i].div(trDataLocal.iloc[:,i].max())).mul(maxV - minV)
teDataLocal.iloc[:,i] = (teDataLocal.iloc[:,i] + minV)
return (trDataLocal,teDataLocal)
def zscorenorm(trData,teData,madFlag = False):
temp = pd.DataFrame(data = trData)
trDataLocal = temp.copy()
check = False
if isinstance(teData, pd.DataFrame):
check = True
teDataLocal = teData.copy()
elif isinstance(teData,type(None)):
teDataLocal = None
else:
check = True
teDataLocal = pd.DataFrame(data=teData)
if(madFlag):
sig = trDataLocal.mad()
else:
sig = trDataLocal.std(ddof=0)
mu = trDataLocal.mean()
NormTrData = np.divide((trDataLocal.select_dtypes(include=np.number)-mu),sig)
if(check):
NormTeData = np.divide((teDataLocal.select_dtypes(include=np.number)-mu),sig)
else:
NormTeData = None
return (NormTrData,NormTeData)
TestData = pd.DataFrame([20,37,40,60,85,120])
print(minmaxnorm(TestData,None))
( 0
0 0.00
1 0.17
2 0.20
3 0.40
4 0.65
5 1.00, None)
print(minmaxnorm(TestData,None,-1,1))
( 0
0 -1.00
1 -0.66
2 -0.60
3 -0.20
4 0.30
5 1.00, None)
print(zscorenorm(TestData,None))
( 0
0 -1.202211
1 -0.695494
2 -0.606073
3 -0.009936
4 0.735236
5 1.778477, None)
from sklearn.preprocessing import MinMaxScaler
Scaler = sk.preprocessing.MinMaxScaler()
Scaler.fit(TestData)
print(Scaler.transform(TestData))
[[0. ]
[0.17]
[0.2 ]
[0.4 ]
[0.65]
[1. ]]
Scaler = sk.preprocessing.MinMaxScaler((-1,1))
Scaler.fit(TestData)
print(Scaler.transform(TestData))
[[-1. ]
[-0.66]
[-0.6 ]
[-0.2 ]
[ 0.3 ]
[ 1. ]]
from sklearn.preprocessing import StandardScaler
Scaler = sk.preprocessing.StandardScaler()
Scaler.fit(TestData)
print(Scaler.transform(TestData))
[[-1.20221086]
[-0.69549389]
[-0.60607324]
[-0.00993563]
[ 0.7352364 ]
[ 1.77847723]]
# writing the distance functions
def euclidean(x,y):
x = np.array(x)
y = np.array(y)
return np.sqrt(np.dot(x - y, x - y))
def manhattan(x,y):
x = np.array(x)
y = np.array(y)
temp = 0
for i in range(len(x)):
temp += abs(x[i] - y[i])
return temp
def minkowski(x,y,l):
x = np.array(x)
y = np.array(y)
temp = 0
for i in range(len(x)):
temp += abs(x[i] - y[i])**l
return temp**(1/l)
def supremum(x,y):
x = np.array(x)
y = np.array(y)
return max(abs((x - y)))
def cosinesim(x,y):
x = np.array(x)
y = np.array(y)
return np.dot(x,y)/(np.sqrt(np.dot(x,x))*np.sqrt(np.dot(y,y)))
# Putting in the data
q3_data = np.array([[1.4,1.3,2.9],[1.8,1.1,3.2],[1.3,1.2,2.9],[0.9,3.3,3.1],[1.5,2.1,3.3]])
q3_point = np.array([1.25,1.74,3.01])
dist_table = np.zeros((5,5))
for i in range(5):
dist_table[i,0] = manhattan(q3_point,q3_data[i,:])
dist_table[i,1] = euclidean(q3_point,q3_data[i,:])
dist_table[i,2] = minkowski(q3_point,q3_data[i,:],3)
dist_table[i,3] = supremum(q3_point,q3_data[i,:])
dist_table[i,4] = cosinesim(q3_point,q3_data[i,:])
names = ['manhattan','euclidean','minkowski','supremum','cosine']
dist_table = pd.DataFrame(data=dist_table)
dist_table.columns = names
print(dist_table)
manhattan euclidean minkowski supremum cosine
0 0.70 0.477703 0.447958 0.44 0.993025
1 1.38 0.864986 0.757918 0.64 0.974255
2 0.70 0.553353 0.541659 0.54 0.991329
3 2.00 1.601312 1.565950 1.56 0.949730
4 0.90 0.525547 0.442544 0.36 0.998982
q3_point = np.zeros((1,3))
q3_point[0,0] = 1.25; q3_point[0,1] = 1.74; q3_point[0,2] = 3.01; #making sure the data input is a row, not a column
normalized = minmaxnorm(q3_data,q3_point)
q3_data = np.array(normalized[0]); q3_point = np.array(normalized[1]);
dist_table = np.zeros((5,5))
for i in range(5):
dist_table[i,0] = manhattan(q3_point[0],q3_data[i,:])
dist_table[i,1] = euclidean(q3_point[0],q3_data[i,:])
dist_table[i,2] = minkowski(q3_point[0],q3_data[i,:],3)
dist_table[i,3] = supremum(q3_point[0],q3_data[i,:])
dist_table[i,4] = cosinesim(q3_point[0],q3_data[i,:])
names = ['manhattan','euclidean','minkowski','supremum','cosine']
dist_table = pd.DataFrame(data=dist_table)
dist_table.columns = names
print(dist_table)
manhattan euclidean minkowski supremum cosine
0 0.864646 0.596535 0.559949 0.549091 0.608322
1 1.930000 1.114316 0.927897 0.650000 0.469026
2 0.798990 0.611967 0.596588 0.594545 0.560254
3 1.100000 0.635767 0.529968 0.390000 0.842656
4 1.392121 0.962690 0.905766 0.890000 0.669106
MovieData = pd.read_csv('movies.csv')
NumNa = MovieData.isna().sum()
PercentNa = 100*NumNa[NumNa != 0]/len(MovieData)
print(PercentNa)
runtime 0.154799
studio 1.083591
dvd_rel_year 1.238390
dvd_rel_month 1.238390
dvd_rel_day 1.238390
director 0.309598
actor1 0.309598
actor2 1.083591
actor3 1.393189
actor4 1.857585
actor5 2.167183
dtype: float64
MovieDataCopy = MovieData.copy()
print("number of samples to be removed is",MovieData['director'].isna().sum())
MovieDataCopy.dropna(subset=['director'],inplace=True)
print("number of remaining samples is",len(MovieDataCopy))
number of samples to be removed is 2
number of remaining samples is 644
for i in ['genre','runtime','mpaa_rating','studio','thtr_rel_month',
'imdb_rating','audience_score','best_pic_win']:
print('The type of',i, 'is',type(MovieData[i][0]),end=' ')
if(isinstance(MovieData[i][0],np.number)):
#grab max and min of numeric (is a np.number) attributes
print('The Maximum Value =',MovieData[i].max(),'The Minimum Value =',MovieData[i].min(),end='')
else:
#one hot encoding of nominal/ordinal attribute
dummies = pd.get_dummies(MovieData[i])
#the number of unique elements is the number of columns of the one hot encoded dataframe
print('The size of the domain is',len(dummies.columns),end=' ')
#newline
print()
The type of genre is <class 'str'> The size of the domain is 11
The type of runtime is <class 'numpy.float64'> The Maximum Value = 267.0 The Minimum Value = 39.0
The type of mpaa_rating is <class 'str'> The size of the domain is 6
The type of studio is <class 'str'> The size of the domain is 209
The type of thtr_rel_month is <class 'numpy.int64'> The Maximum Value = 12 The Minimum Value = 1
The type of imdb_rating is <class 'numpy.float64'> The Maximum Value = 9.0 The Minimum Value = 1.9
The type of audience_score is <class 'numpy.int64'> The Maximum Value = 97 The Minimum Value = 11
The type of best_pic_win is <class 'str'> The size of the domain is 2
for i in ['critics_score','runtime']:
FiveNumSum = np.percentile(MovieData[i].dropna(),[0,25,50,75,100])
print(i)
print('Minimum =',FiveNumSum[0],end=' ')
print('25th percentile =',FiveNumSum[1],end=' ')
print('Median =',FiveNumSum[2],end=' ')
print('75th percentile =',FiveNumSum[3],end=' ')
print('Maximum =',FiveNumSum[4])
critics_score
Minimum = 1.0 25th percentile = 33.0 Median = 61.0 75th percentile = 83.0 Maximum = 100.0
runtime
Minimum = 39.0 25th percentile = 92.0 Median = 103.0 75th percentile = 116.0 Maximum = 267.0
for i in ['audience_score','imdb_rating']:
ColData = MovieData[i].dropna()
print(i)
print('Mean =',ColData.mean(),end=' ')
print('Median =',ColData.median(),end=' ')
print('Mode =',ColData.mode().values[0])
audience_score
Mean = 62.40557275541796 Median = 65.0 Mode = 81
imdb_rating
Mean = 6.496594427244582 Median = 6.6 Mode = 5.9
for i in ['audience_score','imdb_rating']:
FiveNumSum = np.percentile(MovieData[i].dropna(),[25,31,75,90])
print(i)
print('25th percentile =',FiveNumSum[0],end=' ')
print('31st percentile =',np.round(FiveNumSum[1],2),end=' ')
print('75th percentile =',FiveNumSum[2],end=' ')
print('90th percentile =',FiveNumSum[3])
audience_score
25th percentile = 46.0 31st percentile = 50.95 75th percentile = 80.0 90th percentile = 87.0
imdb_rating
25th percentile = 5.9 31st percentile = 6.1 75th percentile = 7.3 90th percentile = 7.8
import seaborn as sns
import matplotlib.pyplot as plt
dummies = pd.get_dummies(MovieData['genre']) #one hot encoding to find number of each genre
name_list = []
num_genre = np.zeros(len(dummies.columns))
for i, column in enumerate(dummies.columns):
name_list.append(column) #getting the list of names to order them by
num_genre[i] = dummies[column].sum()
name_list = np.array(name_list)
num_genre = np.array(num_genre)
sorted_index = num_genre.argsort() #arg sort so I can sort the name list the same way I sort the list of counts
ordered_names = np.flip(name_list[sorted_index])
dummies = dummies.reindex(ordered_names, axis=1)
plt.figure(figsize=(10,5))
ax = sns.countplot(x = 'genre',data=MovieData, order=ordered_names)
ax.set_xticklabels(ax.get_xticklabels(), rotation=35, ha="right") #rotating to prevent overlap
plt.title('Number of Movies by Genre')
plt.show()
dummies = pd.get_dummies(MovieData['mpaa_rating'])
name_list = []
num_genre = np.zeros(len(dummies.columns))
for i, column in enumerate(dummies.columns):
name_list.append(column)
num_genre[i] = dummies[column].sum()
name_list = np.array(name_list)
num_genre = np.array(num_genre)
sorted_index = num_genre.argsort() #doing the same thing as before
ordered_names = np.flip(name_list[sorted_index])
dummies = dummies.reindex(ordered_names, axis=1)
plt.figure(figsize=(10,5))
sns.countplot(y = 'mpaa_rating',data=MovieData, order=ordered_names)
plt.title('Number of Movies by mpaa Rating')
plt.show()
sturges = int(1 + 3.322*np.log(len(MovieData['audience_score']))) #using sturges rule to select bin number
plt.figure(figsize=(10,5))
sns.histplot(x="audience_score",data=MovieData,bins=sturges)
plt.title('Histogram of Audience Score')
plt.show()
plt.figure(figsize=(10,5))
sns.displot(x="runtime",data=MovieData,kind = 'kde') # using kernel density estimate
plt.title('Estimated Density of Run Time')
plt.show()
#plt.figure(figsize=(10,5))
sns.displot(data=MovieData, x="critics_rating", hue="genre", multiple = 'stack',stat='proportion',common_norm=False)
#sns.histplot(data=MovieData, x="critics_rating", hue="genre", multiple = 'stack',stat='probability',common_norm=False)
plt.show()
plt.figure(figsize=(10,5))
sns.countplot(x = 'critics_rating',data=MovieData)
plt.title('Number of Movies by Critic Rating')
plt.show()
Top5 = MovieData['genre'].value_counts().head(5).index
MovieData['Bin'] = pd.cut(MovieData['audience_score'],10)
plt.figure(figsize=(10,5))
ax = sns.countplot(x='genre',hue='Bin',data = MovieData,order=Top5)
ax.set_xticklabels(ax.get_xticklabels(), rotation=35, ha="right")
plt.show()
sns.boxplot(x='critics_rating', y='audience_score', data=MovieData)
ax = sns.violinplot(x="genre", y="audience_score", data=MovieData, order = Top5)
ax.set_xticklabels(ax.get_xticklabels(), rotation=35, ha="right")
plt.title('Violin Plot of Audience Score by Genre')
plt.plot()
plt.figure(figsize=(10,5))
sns.displot(x="critics_score",data=MovieData,hue = 'audience_rating',kind = 'kde')
plt.title('Estimated Density of Critics Score')
plt.show()
plt.scatter(MovieData['imdb_rating'],MovieData['audience_score'])
plt.xlabel('imdb rating')
plt.ylabel('audience score')
plt.show()
graphs = sns.FacetGrid(MovieData, col='mpaa_rating')
graphs.map(sns.scatterplot,"imdb_rating","audience_score")
Top3 = MovieData['genre'].value_counts().head(3).index
graphs = sns.FacetGrid(MovieData, col='mpaa_rating',hue='genre',hue_order=Top3)
graphs.map(sns.scatterplot,"imdb_rating","audience_score")
graphs.add_legend()