#!pip install kaggle
#!pip install category_encoders
import os
import math
import kaggle
import warnings
import numpy as np
import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt
import category_encoders as ce
from zipfile import ZipFile
from sklearn.svm import SVR
from sklearn.naive_bayes import GaussianNB
from sklearn.compose import ColumnTransformer
from sklearn.tree import DecisionTreeRegressor,export_graphviz
from sklearn.feature_selection import RFE, RFECV
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor, VotingRegressor
from sklearn.metrics import accuracy_score, classification_report, mean_squared_error
warnings.filterwarnings('ignore')
Download the regression and classification dataset:
!kaggle competitions download -q -c cs9856-spotify-regression-problem-2022
!kaggle competitions download -q -c cs9856-spotify-classification-problem-2022
Unzip both datasets to target folders:
downloadFiles = [("regression", "cs9856-spotify-regression-problem-2022.zip"), ("classification", "cs9856-spotify-classification-problem-2022.zip")]
for folderTarget, downloadedZipFile in downloadFiles:
if os.path.exists(downloadedZipFile):
# Unzip to location
with ZipFile(downloadedZipFile, 'r') as zipFile:
zipFile.extractall(folderTarget)
# If the zip file exists remove it
os.remove(downloadedZipFile)
A function to output model predictions to a csv file and directly import it to Kaggle for comparing scores
def uploadDataToKaggle(testId, y_pred, problemType, upload=False):
if (problemType == "regression"):
pd.DataFrame(data={"Id" : testId, "pop" : y_pred}).to_csv('prediction.csv', index=False)
if(upload):
!kaggle competitions submit -c cs9856-spotify-regression-problem-2022 -f prediction.csv -m "Uploading prediction"
elif(problemType == "classification"):
pd.DataFrame(data={"Id" : testId, "top genre" : y_pred}).to_csv('prediction.csv', index=False)
if(upload):
!kaggle competitions submit -c cs9856-spotify-classification-problem-2022 -f prediction.csv -m "Uploading prediction"
if os.path.exists("prediction.csv"): os.remove("prediction.csv")
A regression problem that aims to predict the popularity score of a song
regressionTrain = pd.read_csv("regression/CS98XRegressionTrain.csv")
regressionTest = pd.read_csv("regression/CS98XRegressionTest.csv")
regressionTrain.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 453 entries, 0 to 452
Data columns (total 15 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 Id 453 non-null int64
1 title 453 non-null object
2 artist 453 non-null object
3 top genre 438 non-null object
4 year 453 non-null int64
5 bpm 453 non-null int64
6 nrgy 453 non-null int64
7 dnce 453 non-null int64
8 dB 453 non-null int64
9 live 453 non-null int64
10 val 453 non-null int64
11 dur 453 non-null int64
12 acous 453 non-null int64
13 spch 453 non-null int64
14 pop 453 non-null int64
dtypes: int64(12), object(3)
memory usage: 53.2+ KB
plt.figure(figsize = (15,8))
sn.heatmap(regressionTrain.corr(), annot=True)
plt.show()
x_train = regressionTrain.loc[:, 'bpm':'spch']
y_train = regressionTrain['pop']
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.20, random_state=42)
lrModel = LinearRegression().fit(x_train, y_train)
lr_RMSE = np.sqrt(mean_squared_error(y_val, lrModel.predict(x_val)))
print("RMSE on validation data:", lr_RMSE)
uploadDataToKaggle(regressionTest['Id'].to_list(), lrModel.predict(regressionTest.loc[:, 'bpm':'spch']), "regression", upload=False)
RMSE on validation data: 12.13173739137971
RMSE on Kaggle test data: 8.91431
x_train = regressionTrain.loc[:, 'bpm':'spch']
y_train = regressionTrain['pop']
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.20, random_state=42)
treeModel = DecisionTreeRegressor(max_depth=2)
treeModel.fit(x_train, y_train)
tree_RMSE = np.sqrt(mean_squared_error(y_val, treeModel.predict(x_val)))
print("RMSE on validation data:", tree_RMSE)
uploadDataToKaggle(regressionTest['Id'].to_list(), treeModel.predict(regressionTest.loc[:, 'bpm':'spch']), "regression", upload=False)
RMSE on validation data: 12.905732435706527
RMSE on Kaggle data: 9.00005
export_graphviz(
treeModel,
out_file= os.path.join("tree_model.dot"),
feature_names=regressionTrain.columns[5:-1],
class_names='pop',
rounded=True,
filled=True)
attributes = ['dur','dB','dnce','acous','nrgy']
x_train = regressionTrain[attributes]
x_test = regressionTest[attributes]
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
x_train = pd.DataFrame(x_train, columns=attributes)
x_test = pd.DataFrame(x_test, columns=attributes)
# After scaling
x_train.head()
pd.plotting.scatter_matrix(x_train[attributes], figsize=(12,7))
plt.show()
# svr_params = {
# 'kernel':['rbf'],
# 'C':[1,10,100,1000],
# #'gamma':['scale','auto']
# 'gamma':[0.0001,0.001,0.01,0.1,0.2,0.25,0.5],
# 'epsilon': [ 0.001,0.1,0.5,1,2,5,6,7,8,10]
# }
# estimator = SVR()
# svr_clf = GridSearchCV(estimator,
# svr_params,
# cv = 5,
# refit = True)
# svr_clf.fit(x_train,y_train)
svr_best = SVR(C=10, epsilon=5, gamma=0.1, kernel='rbf')
svr_clf = AdaBoostRegressor(svr_best, n_estimators=50)
gr_clf = GradientBoostingRegressor(loss='lad', n_estimators=50, max_depth=5)
y_train = regressionTrain['pop']
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)
vt_clf = VotingRegressor([('svr', svr_clf), ('gr', gr_clf)], weights=[0.4,0.6])
vt_clf.fit(x_train, y_train)
vot_RMSE = np.sqrt(mean_squared_error(y_val, vt_clf.predict(x_val)))
print("RMSE on validation data:", vot_RMSE)
uploadDataToKaggle(regressionTest['Id'].to_list(), vt_clf.predict(x_test[attributes]), "regression", upload=False)
RMSE on validation data: 12.061972785704496
RMSE on Kaggle Test Data: 7.59224 (score using full training data for training the model)
Conclusion
A classification problem that aims to predict the top genre that a song belongs to
classificationTrain = pd.read_csv("classification/CS98XClassificationTrain.csv")
classificationTest = pd.read_csv("classification/CS98XClassificationTest.csv")
# May need to upsample or weight classes
classificationTrain['top genre'].value_counts().plot.bar()
plt.rcParams["figure.figsize"] = (15,10)
plt.xlabel('Genre')
plt.title('Number of Instances in Each Genre')
# Look at the distribution of all the features
classificationTrain.hist(bins = 50,figsize=(16,10))
plt.show()
columns = [col for col in classificationTrain.columns if classificationTrain[col].dtypes == 'O']
print(columns)
['title', 'artist', 'top genre']
for var in columns:
print(classificationTrain[var].value_counts())
Take Good Care Of My Baby - 1990 Remastered 2
Please Mr. Postman 2
I Was Made For Lovin' You 1
Mr. Soul 1
(I Just) Died In Your Arms 1
..
My Way 1
I Got 5 On It 1
7 Days 1
Rebel Rebel - 2016 Remaster 1
I'll Be Missing You (feat. 112) 1
Name: title, Length: 451, dtype: int64
Elton John 9
Queen 7
ABBA 7
The Beatles 6
Mariah Carey 5
..
Stargazers 1
Laid Back 1
The Tornados 1
John Denver 1
Gabrielle 1
Name: artist, Length: 345, dtype: int64
adult standards 68
album rock 66
dance pop 61
glam rock 16
brill building pop 16
..
blues 1
glam punk 1
british dance band 1
louisiana blues 1
hip pop 1
Name: top genre, Length: 86, dtype: int64
Adult standard was added for null top genre values because it was the most frequent genre, as identified by an earlier plot.
print(classificationTrain.isnull().sum().sum())
classificationTrain['top genre'].replace(np.NaN, "adult standards", inplace=True)
15
numerical = [var for var in classificationTrain.columns if classificationTrain[var].dtype!='O']
print('There are {} numerical variables'.format(len(numerical)))
print('The numerical variables are :', numerical)
There are 12 numerical variables
The numerical variables are : ['Id', 'year', 'bpm', 'nrgy', 'dnce', 'dB', 'live', 'val', 'dur', 'acous', 'spch', 'pop']
classificationTrain[columns].isnull().sum().sum()
y_train = classificationTrain["top genre"]
classificationTrain.drop(labels="top genre", axis=1, inplace=True)
One hot encoding the title and artist for both the training and test datasets
encoder = ce.OneHotEncoder(cols=['title' , 'artist'], handle_unknown="ignore")
classificationTrain = encoder.fit_transform(classificationTrain)
classificationTest = encoder.transform(classificationTest)
x_train = classificationTrain
x_test = classificationTest
We scaled the data with RobustScaler() as it did perform the best out of all three. StandardScaler follows Standard Normal Distribution (SND); makes mean = 0 and scales the data to unit variance. RobustScaler, however, scales features using statistics that are robust to outliers. This method removes the median and scales the data in the range between 1st quartile and 3rd quartile. i.e., in between 25th quantile and 75th quantile range. This range is also called an Interquartile range.
scaler = RobustScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
X_train = pd.DataFrame(x_train, columns=[classificationTrain.columns])
X_test = pd.DataFrame(x_test, columns=[classificationTrain.columns])
X_train.head()
X_test.replace(np.NaN, 0, inplace=True)
X_test.head()
classificationTrain = pd.read_csv("classification/CS98XClassificationTrain.csv")
classificationTest = pd.read_csv("classification/CS98XClassificationTest.csv")
# Replacing missing genres with adult standards
classificationTrain['top genre'].replace(np.NaN, "adult standards", inplace=True)
x_train = classificationTrain.loc[:, 'bpm':'pop']
y_train = classificationTrain['top genre']
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.20,random_state=42)
lrModel = LogisticRegression().fit(x_train, y_train)
lr_accuracy = accuracy_score(y_val, lrModel.predict(x_val))
print("Accuracy on validation data:", lr_accuracy)
uploadDataToKaggle(classificationTest['Id'].to_list(), lrModel.predict(classificationTest.loc[:, 'bpm':'pop']), "classification", upload=False)
Accuracy on validation data: 0.31868131868131866
Accuracy score on Kaggle test data: 0.23214
# Split into features and classes
train_features = X_train.drop(["Id"], axis=1)
train_classes = classificationTrain["top genre"]
# Use gridseachCV and RFE to find the best combination of hyperparameters and features for use in the model.
# Takes a long time to load so best hyperparameters taken and code commented out for future
# rnd_params = {
# 'estimator__n_estimators':[100,500],
# 'estimator__criterion':['gini'],
# 'estimator__max_depth':[10,50,100],
# 'estimator__bootstrap':[True],
# 'estimator__oob_score':[True]
# }
# estimator = RandomForestClassifier()
# selector = RFE(estimator)
# rnd_clf = GridSearchCV(selector,
# rnd_params,
# cv=5,
# refit = True)
estimator = RandomForestClassifier(n_estimators=100, criterion='gini', max_depth=100, bootstrap=True, oob_score=True)
rnd_clf = RFE(estimator)
x_train, x_val, y_train, y_val = train_test_split(train_features, train_classes, test_size=0.20, random_state=42)
rnd_clf.fit(x_train, y_train)
rnd_accuracy = accuracy_score(y_val, rnd_clf.predict(x_val))
print("Accuracy on validation data:", rnd_accuracy)
Accuracy on validation data: 0.46153846153846156
X_test.drop(["Id"], axis=1, inplace=True)
uploadDataToKaggle(classificationTest['Id'].to_list(), rnd_clf.predict(X_test), "classification", upload=False)
Accuracy on Kaggle test data: 0.50000
Gaussian Naive Bayes is a variant of Naive Bayes that follows Gaussian normal distribution and supports continuous data. Naive Bayes are a group of supervised machine learning classification algorithms based on the Bayes theorem. It is a simple classification technique, but has high functionality.
train_features = X_train.drop(["Id"], axis=1)
train_classes = classificationTrain["top genre"]
x_train, x_val, y_train, y_val = train_test_split(train_features, train_classes, test_size=0.2, random_state=42)
gnb = GaussianNB()
gnb.fit(x_train, y_train)
y_pred = gnb.predict(x_val)
Getting the accuracy score but on the training data that we split earlier
print('Model accuracy score: {0:0.4f}'. format(accuracy_score(y_val, y_pred)))
Model accuracy score: 0.2857
print(classification_report(y_val, y_pred))
precision recall f1-score support
adult standards 1.00 0.22 0.36 23
afropop 0.00 0.00 0.00 1
album rock 1.00 0.60 0.75 10
alternative metal 0.00 0.00 0.00 0
art pop 0.00 0.00 0.00 0
art rock 0.00 0.00 0.00 2
atl hip hop 1.00 0.33 0.50 3
barbadian pop 1.00 0.67 0.80 3
bebop 0.00 0.00 0.00 1
belgian pop 0.00 0.00 0.00 0
blues rock 0.00 0.00 0.00 0
bow pop 0.00 0.00 0.00 1
boy band 1.00 1.00 1.00 2
brill building pop 0.00 0.00 0.00 4
brit funk 0.00 0.00 0.00 1
british blues 0.00 0.00 0.00 1
british folk 0.00 0.00 0.00 1
british invasion 0.00 0.00 0.00 1
british soul 0.00 0.00 0.00 1
britpop 0.00 0.00 0.00 1
classic rock 0.00 0.00 0.00 0
classic soul 0.00 0.00 0.00 0
classic uk pop 0.00 0.00 0.00 2
country rock 0.00 0.00 0.00 1
dance pop 1.00 0.21 0.35 14
dance rock 0.00 0.00 0.00 2
deep adult standards 0.00 0.00 0.00 1
disco house 0.00 0.00 0.00 1
doo-wop 0.00 0.00 0.00 0
east coast hip hop 0.00 0.00 0.00 0
eurodance 0.14 0.50 0.22 2
europop 1.00 0.75 0.86 4
g funk 0.00 0.00 0.00 0
glam punk 0.00 0.00 0.00 1
glam rock 1.00 1.00 1.00 2
hip hop 0.00 0.00 0.00 1
merseybeat 0.00 0.00 0.00 1
new wave pop 0.00 0.00 0.00 1
permanent wave 0.00 0.00 0.00 1
pop 0.25 1.00 0.40 1
soft rock 0.00 0.00 0.00 0
accuracy 0.29 91
macro avg 0.20 0.15 0.15 91
weighted avg 0.68 0.29 0.36 91
uploadDataToKaggle(classificationTest["Id"].tolist(), gnb.predict(X_test), "classification", upload=False)
Accuracy on Kaggle test data: 0.41071 (for full training data used for training)