import gc
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.sparse import csr_matrix, hstack
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
df = pd.read_csv('train.tsv', sep = '\t')
msk = np.random.rand(len(df)) < 0.8
train = df[msk]
test = df[~msk]
train.shape, test.shape
train.head()
train.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 1185866 entries, 0 to 1482533
Data columns (total 8 columns):
train_id 1185866 non-null int64
name 1185866 non-null object
item_condition_id 1185866 non-null int64
category_name 1180783 non-null object
brand_name 679496 non-null object
price 1185866 non-null float64
shipping 1185866 non-null int64
item_description 1185863 non-null object
dtypes: float64(1), int64(3), object(4)
memory usage: 81.4+ MB
train.price.describe()
plt.subplot(1, 2, 1)
(train['price']).plot.hist(bins=50, figsize=(12, 6), edgecolor = 'white', range = [0, 250])
plt.xlabel('price', fontsize=12)
plt.title('Price Distribution', fontsize=12)
plt.subplot(1, 2, 2)
np.log(train['price']+1).plot.hist(bins=50, figsize=(12,6), edgecolor='white')
plt.xlabel('log(price+1)', fontsize=12)
plt.title('Price Distribution', fontsize=12)
train['shipping'].value_counts() / len(train)
shipping_fee_by_buyer = train.loc[df['shipping'] == 0, 'price']
shipping_fee_by_seller = train.loc[df['shipping'] == 1, 'price']
fig, ax = plt.subplots(figsize=(18,8))
ax.hist(shipping_fee_by_seller, color='#8CB4E1', alpha=1.0, bins=50, range = [0, 100],
label='Price when Seller pays Shipping')
ax.hist(shipping_fee_by_buyer, color='#007D00', alpha=0.7, bins=50, range = [0, 100],
label='Price when Buyer pays Shipping')
plt.xlabel('price', fontsize=12)
plt.ylabel('frequency', fontsize=12)
plt.title('Price Distribution by Shipping Type', fontsize=15)
plt.tick_params(labelsize=12)
plt.legend()
plt.show()
print('The average price is {}'.format(round(shipping_fee_by_seller.mean(), 2)), 'if seller pays shipping');
print('The average price is {}'.format(round(shipping_fee_by_buyer.mean(), 2)), 'if buyer pays shipping')
The average price is 22.58 if seller pays shipping
The average price is 30.11 if buyer pays shipping
fig, ax = plt.subplots(figsize=(18,8))
ax.hist(np.log(shipping_fee_by_seller+1), color='#8CB4E1', alpha=1.0, bins=50,
label='Price when Seller pays Shipping')
ax.hist(np.log(shipping_fee_by_buyer+1), color='#007D00', alpha=0.7, bins=50,
label='Price when Buyer pays Shipping')
plt.xlabel('log(price+1)', fontsize=12)
plt.ylabel('frequency', fontsize=12)
plt.title('Price Distribution by Shipping Type', fontsize=15)
plt.tick_params(labelsize=12)
plt.legend()
plt.show()
print('There are', train['category_name'].nunique(), 'unique values in category name column')
There are 1265 unique values in category name column
train['category_name'].value_counts()[:10]
sns.boxplot(x = 'item_condition_id', y = np.log(train['price']+1), data = train, palette = sns.color_palette('RdBu',5))
NUM_BRANDS = 4000
NUM_CATEGORIES = 1000
NAME_MIN_DF = 10
MAX_FEATURES_ITEM_DESCRIPTION = 50000
print('There are %d items that do not have a category name.' %train['category_name'].isnull().sum())
There are 5083 items that do not have a category name.
print('There are %d items that do not have a brand name.' %train['brand_name'].isnull().sum())
There are 506370 items that do not have a brand name.
print('There are %d items that do not have a description.' %train['item_description'].isnull().sum())
There are 3 items that do not have a description.
def handle_missing_inplace(dataset):
dataset['category_name'].fillna(value='missing', inplace=True)
dataset['brand_name'].fillna(value='missing', inplace=True)
dataset['item_description'].replace('No description yet,''missing', inplace=True)
dataset['item_description'].fillna(value='missing', inplace=True)
def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category = dataset['category_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
def to_categorical(dataset):
dataset['category_name'] = dataset['category_name'].astype('category')
dataset['brand_name'] = dataset['brand_name'].astype('category')
dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')
df = pd.read_csv('train.tsv', sep = '\t')
msk = np.random.rand(len(df)) < 0.8
train = df[msk]
test = df[~msk]
test_new = test.drop('price', axis=1)
y_test = np.log1p(test["price"])
train = train[train.price != 0].reset_index(drop=True)
nrow_train = train.shape[0]
y = np.log1p(train["price"])
merge: pd.DataFrame = pd.concat([train, test_new])
handle_missing_inplace(merge)
cutting(merge)
to_categorical(merge)
merge.head()
cv = CountVectorizer(min_df=NAME_MIN_DF)
X_name = cv.fit_transform(merge['name'])
cv = CountVectorizer()
X_category = cv.fit_transform(merge['category_name'])
tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION, ngram_range=(1, 3), stop_words='english')
X_description = tv.fit_transform(merge['item_description'])
lb = LabelBinarizer(sparse_output=True)
X_brand = lb.fit_transform(merge['brand_name'])
X_dummies = csr_matrix(pd.get_dummies(merge[['item_condition_id', 'shipping']], sparse=True).values)
sparse_merge = hstack((X_dummies, X_description, X_brand, X_category, X_name)).tocsr()
sparse_merge.shape
mask = np.array(np.clip(sparse_merge.getnnz(axis=0) - 1, 0, 1), dtype=bool)
sparse_merge = sparse_merge[:, mask]
sparse_merge.shape
X = sparse_merge[:nrow_train]
X_test = sparse_merge[nrow_train:]
train_X = lgb.Dataset(X, label=y)
params = {
'learning_rate': 0.75,
'application': 'regression',
'max_depth': 3,
'num_leaves': 100,
'verbosity': -1,
'metric': 'RMSE',
}
gbm = lgb.train(params, train_set=train_X, num_boost_round=3200, verbose_eval=100)
y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
from sklearn.metrics import mean_squared_error
print('The rmse of prediction is:', mean_squared_error(y_test, y_pred) ** 0.5)
The rmse of prediction is: 0.46164222941613137