Get the data
import pandas as pd
housing = pd.read_csv('/work/housing.csv')
housing.head()
housing.info()
housing["ocean_proximity"].value_counts()
housing.describe()
import matplotlib.pyplot as plt
housing.hist(bins=50,figsize =(20,15))
Many histograms are tail heavy. They extend much farther to the right of the median than to the left. This may make it bit harder for some ML algorithms to detect patterns. https://www.statisticshowto.com/heavy-tailed-distribution/
# creating test set
import numpy as np
def split_train_test(data,test_ratio):
shuffled_indices = np.random.permutation(len(data))
test_set_size = int(len(data)*test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return data.iloc[train_indices],data.iloc[test_indices]
train_set,test_set = split_train_test(housing,0.2)
print(len(train_set), " + ", len(test_set))
y = housing.median_income
x = housing.drop('median_income',axis = 1)
x.head()
!sudo pip install --upgrade pip --quiet
!pip install scikit-learn-extra
# using sckitlearn for splitting
from sklearn.model_selection import train_test_split
train_set ,test_set = train_test_split(housing,test_size=0.2,random_state=40)
# X_train,X_test,Y_train,Y_test = train_test_split(x,y,random_state = 40, test_size =0.2)
housing["median_income"].hist()
housing["income_cat"] = pd.cut(housing["median_income"],
bins=[0., 1.5, 3.0, 4.5, 6., np.inf],
labels=[1, 2, 3, 4, 5])
housing["income_cat"].value_counts()
housing["income_cat"].hist()
#housing["median_income"].hist()
Stratified Sampling is important as it guarantees that your dataset does not have an intrinsic bias and that it does represent the population. Researchers rely on stratified sampling when a population’s characteristics are diverse and they want to ensure that every characteristic is properly represented in the sample. This helps with the generalizability and validity of the study, as well as avoiding research biases like undercoverage bias.
# stratified sampling based on the income category beacuse of it's outliers
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing["income_cat"]):
strat_train_set = housing.loc[train_index]
strat_test_set = housing.loc[test_index]
strat_test_set["income_cat"].value_counts() / len(strat_test_set)
housing["income_cat"].value_counts()/len(housing)
def income_cat_proportions(data):
return data["income_cat"].value_counts() / len(data)
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=40)
compare_props = pd.DataFrame({
"Overall": income_cat_proportions(housing),
"Stratified": income_cat_proportions(strat_test_set),
"Random": income_cat_proportions(test_set),
}).sort_index()
compare_props["Rand. %error"] = 100 * compare_props["Random"] / compare_props["Overall"] - 100
compare_props["Strat. %error"] = 100 * compare_props["Stratified"] / compare_props["Overall"] - 100
compare_props
for set_ in(strat_train_set,strat_test_set):
set_.drop("income_cat",axis = 1,inplace = True)
Data Visualization
# creating copy of training set
housing = strat_test_set.copy()
housing.plot(kind = "scatter",x="longitude",y="latitude")
housing.plot(kind = "scatter",x="longitude",y="latitude",alpha=0.3)
#scatter plot represnting population based on their median house vale.
#radius of circle i.e "s" is based on population density
#color of each circle i.e "c" is based on median house value
housing.plot.scatter(x="longitude",y="latitude",alpha=0.5,
s= housing["population"]/50,label = "population",figsize =(10,5),
c="median_house_value",cmap=plt.get_cmap("jet"),colorbar=True)
above graph explains that housing prices depends on location and also on population density. Which are near to ocean they have high prices.
# correlation between features
corr_matrix = housing.corr()
corr_matrix
corr_matrix["median_house_value"].sort_values(ascending=False)
The correlation coefficient measure only linear correlation. It may completely miss out on nonlinear relationships.
And also in the above values for median house value we can see the median income has high positive correlation coefficient than compared to other.
from pandas.plotting import scatter_matrix
attributes =["median_house_value","median_income","total_rooms","housing_median_age"]
scatter_matrix(housing[attributes],figsize=(12,8))
As you can see median house value is almost linearly related to median income.
housing.plot.scatter(x="median_income",y="median_house_value",alpha =0.1)
Plot between median house value and median income shows that there is linear relation between both but near 500K there is price cap.
Experimenting with attribute combinations
housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"] = housing["population"]/housing["households"]
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending = False)
Compared to total bedrooms new bedrooms per room is more correlated to median house value.
housing = strat_train_set.drop("median_house_value",axis =1)
housing_labels = strat_train_set["median_house_value"].copy()
Data Cleaning
# total bedroom attribute has some missing values
# housing.dropna(subset =["total_bedrooms"])
# housing.drop("total_bedrooms",axis=1)
# median = housing["total_bedrooms"].median()
# housing["total_bedrooms"].fillna(median,inplace=True)
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy ="median")
housing_num = housing.drop("ocean_proximity",axis=1)
imputer.fit(housing_num)
imputer.statistics_
housing_num.median().values
X = imputer.transform(housing_num)
housing_tr = pd.DataFrame(X,columns=housing_num.columns)
from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder()
housing_cat = housing["ocean_proximity"]
housing_cat_encoded = encoder.fit_transform(housing_cat)
housing_cat_encoded
print(encoder.classes_)
one hot encoding, because ML algorithm will assume 0 and 1 are similar compared to 0 and 4.
from sklearn.preprocessing import OneHotEncoder
encoder = OneHotEncoder()
housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1,1))
housing_cat_1hot
housing_cat_1hot.toarray()
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer()
housing_cat_1hot = encoder.fit_transform(housing_cat)
housing_cat_1hot
from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix,bedrooms_ix,population_ix,household_ix = 3,4,5,6
class CombinedAttributesAdder(BaseEstimator,TransformerMixin):
def __init__(self,add_bedrooms_per_room=True):
self.add_bedrooms_per_room = add_bedrooms_per_room
def fit(self,X,y=None):
return self
def transform(self,X,y=None):
rooms_per_household = X[:,rooms_ix]/X[:,household_ix]
population_per_household = X[:,population_ix]/X[:,household_ix]
if self.add_bedrooms_per_room:
bedrooms_per_room =X[:,population_ix]/X[:,rooms_ix]
return np.c_[X,rooms_per_household,population_per_household,bedrooms_per_room]
else:
return np.c_[X,rooms_per_household,population_per_household]
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)
Feature Scaling
Machine Learning Algorithms don't perform well when the input numerical data have very different scale. Min-Max scaling and standardization are two common ways to get all attributes have same scale.
# pipeline constructor takes a list of name/estimator pairs defining a squence of steps.
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
num_pipeline = Pipeline([
('imputer',SimpleImputer(strategy ="median")),
('attribs_adder',CombinedAttributesAdder()),
('std_scaler',StandardScaler())
])
housing_num_tr = num_pipeline.fit_transform(housing_num)
from sklearn.base import BaseEstimator,TransformerMixin
class DataFrameSelector(BaseEstimator,TransformerMixin):
def __init__(self,attribute_names):
self.attribute_names = attribute_names
def fit(self,X,y=None):
return self
def transform(self,X):
return X[self.attribute_names].values
num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]
num_pipeline = Pipeline([
('selector',DataFrameSelector(num_attribs)),
('imputer',SimpleImputer(strategy="median")),
('attribs_adder',CombinedAttributesAdder()),
('std_scaler',StandardScaler())
])
cat_pipeline = Pipeline([
('selector', DataFrameSelector(cat_attribs)),
('one_hot_encoder', OneHotEncoder(sparse=False))
])
from sklearn.pipeline import FeatureUnion
full_pipeline = FeatureUnion(transformer_list=[
("num_pipeline",num_pipeline),
("cat_pipeline",cat_pipeline)
])
housing_prepared = full_pipeline.fit_transform(housing)
housing_prepared
housing_prepared.shape
Model Training
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(housing_prepared,housing_labels)
some_data = housing.iloc[:5]
some_labels = housing_labels.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)
print("Predictions:",lin_reg.predict(some_data_prepared))
print("Labels:",list(some_labels))
from sklearn.metrics import mean_squared_error
housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels,housing_predictions)
lin_rme = np.sqrt(lin_mse)
lin_rme
from sklearn.tree import DecisionTreeRegressor
tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared,housing_labels)
housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels,housing_predictions)
tree_rmse = np.sqrt(tree_mse)
tree_rmse
This can't be the case. Maybe the model overfit. So let's use set of training data for validation set.
Using Cross-Validation
from sklearn.model_selection import cross_val_score
scores = cross_val_score(tree_reg,housing_prepared,housing_labels,
scoring="neg_mean_squared_error",cv=10)
tree_rmse_scores = np.sqrt(-scores)
def display_scores(scores):
print("scores:",scores)
print("Mean:",scores.mean())
print("standard deviation:",scores.std())
display_scores(tree_rmse_scores)
lin_scores = cross_val_score(lin_reg,housing_prepared,housing_labels,
scoring ="neg_mean_squared_error",cv=10)
lin_rmse_scores = np.sqrt(-lin_scores)
display_scores(lin_rmse_scores)
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared,housing_labels)
housing_predictions = forest_reg.predict(housing_prepared)
forest_mse = mean_squared_error(housing_labels,housing_predictions)
forest_rmse = np.sqrt(forest_mse)
forest_rmse
forest_scores = cross_val_score(forest_reg,housing_prepared,housing_labels,
scoring ="neg_mean_squared_error",cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)
display_scores(forest_rmse_scores)
Fine tuning the model
# grid search
from sklearn.model_selection import GridSearchCV
param_grid = [
{'n_estimators':[3,10,30],'max_features':[2,4,6,8]},
{'bootstrap': [False],'n_estimators':[3,10],'max_features':[2,3,4]},
]
forest_reg = RandomForestRegressor()
grid_search = GridSearchCV(forest_reg,param_grid,cv=5,
scoring ='neg_mean_squared_error')
grid_search.fit(housing_prepared,housing_labels)
grid_search.best_params_
grid_search.best_estimator_
RandomForestRegressor(bootstrap=True,criterion='mse',max_depth=None,
max_features=6,max_leaf_nodes=None,
min_samples_leaf=1,min_samples_split=2,
min_weight_fraction_leaf=0.0,n_estimators=30,n_jobs=1,
oob_score=False,random_state=40,verbose=0,warm_start=False)
cvres = grid_search.cv_results_
for mean_score,params in zip(cvres["mean_test_score"],cvres["params"]):
print(np.sqrt(-mean_score),params)
feature_importances = grid_search.best_estimator_.feature_importances_
feature_importances
extra_attribs =["rooms_per_hhold","pop_per_hhold","bedrooms_per_room"]
cat_one_hot_attribs = list(encoder.classes_)
attributes = num_attribs+extra_attribs+cat_one_hot_attribs
sorted(zip(feature_importances,attributes),reverse=True)
final_model = grid_search.best_estimator_
X_test = strat_test_set.drop("median_house_value",axis=1)
y_test = strat_test_set["median_house_value"].copy()
X_test_prepared = full_pipeline.transform(X_test)
final_predictions = final_model.predict(X_test_prepared)
final_mse = mean_squared_error(y_test,final_predictions)
final_rmse = np.sqrt(final_mse)
final_rmse