Untitled Python Project

# To support both python 2 and python 3 from __future__ import division, print_function, unicode_literals # Common imports import numpy as np import os # to make this notebook's output stable across runs np.random.seed(42) # To plot pretty figures %matplotlib inline import matplotlib as mpl import matplotlib.pyplot as plt mpl.rc('axes', labelsize=14) mpl.rc('xtick', labelsize=12) mpl.rc('ytick', labelsize=12) # Where to save the figures PROJECT_ROOT_DIR = "." CHAPTER_ID = "fundamentals" def save_fig(fig_id, tight_layout=True): path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png") print("Saving figure", fig_id) if tight_layout: plt.tight_layout() plt.savefig(path, format='png', dpi=300) # Ignore useless warnings (see SciPy issue #5998) import warnings warnings.filterwarnings(action="ignore", message="^internal gelsd")

def prepare_country_stats(oecd_bli, gdp_per_capita): oecd_bli = oecd_bli[oecd_bli["INEQUALITY"]=="TOT"] oecd_bli = oecd_bli.pivot(index="Country", columns="Indicator", values="Value") gdp_per_capita.rename(columns={"2015": "GDP per capita"}, inplace=True) gdp_per_capita.set_index("Country", inplace=True) full_country_stats = pd.merge(left=oecd_bli, right=gdp_per_capita, left_index=True, right_index=True) full_country_stats.sort_values(by="GDP per capita", inplace=True) remove_indices = [0, 1, 6, 8, 33, 34, 35] keep_indices = list(set(range(36)) - set(remove_indices)) return full_country_stats[["GDP per capita", 'Life satisfaction']].iloc[keep_indices]

import os datapath = os.path.join("datasets", "lifesat", "")

# Code example import matplotlib.pyplot as plt import numpy as np import pandas as pd import sklearn.linear_model # Load the data oecd_bli = pd.read_csv(datapath + "oecd_bli_2015.csv", thousands=',') gdp_per_capita = pd.read_csv(datapath + "gdp_per_capita.csv",thousands=',',delimiter='\t', encoding='latin1', na_values="n/a") # Prepare the data country_stats = prepare_country_stats(oecd_bli, gdp_per_capita) X = np.c_[country_stats["GDP per capita"]] y = np.c_[country_stats["Life satisfaction"]] # Visualize the data country_stats.plot(kind='scatter', x="GDP per capita", y='Life satisfaction') plt.show() # Select a linear model model = sklearn.linear_model.LinearRegression() # Train the model model.fit(X, y) # Make a prediction for Cyprus X_new = [[22587]] # Cyprus' GDP per capita print(model.predict(X_new)) # outputs [[ 5.96242338]]

[[5.96242338]]

oecd_bli = pd.read_csv(datapath + "oecd_bli_2015.csv", thousands=',') oecd_bli = oecd_bli[oecd_bli["INEQUALITY"]=="TOT"] oecd_bli = oecd_bli.pivot(index="Country", columns="Indicator", values="Value") oecd_bli.head(2)

	Air pollutionfloat64	Assault ratefloat64	Consultation on rule-makingfloat64	Dwellings without basic facilitiesfloat64	Educational attainmentfloat64	Employees working very long hoursfloat64	Employment ratefloat64	Homicide ratefloat64	Household net adjusted disposable incomefloat64	Household net financial wealthfloat64	Housing expenditurefloat64	Job securityfloat64	Life expectancyfloat64	Life satisfactionfloat64	Long-term unemployment ratefloat64	Personal earningsfloat64	Quality of support networkfloat64	Rooms per personfloat64	Self-reported healthfloat64	Student skillsfloat64	Time devoted to leisure and personal carefloat64	Voter turnoutfloat64	Water qualityfloat64	Years in educationfloat64
Australia	13	2.1	10.5	1.1	76	14.02	72	0.8	31588	47657	20	4.8	82.1	7.3	1.08	50449	92	2.3	85	512	14.41	93	91	19.4
Austria	27	3.4	7.1	1	83	7.61	72	0.4	31173	49887	21	3.9	81	6.9	1.19	45199	89	1.6	69	500	14.46	75	94	17
2 rows × 24 columns

oecd_bli["Life satisfaction"].head()

gdp_per_capita = pd.read_csv(datapath+"gdp_per_capita.csv", thousands=',', delimiter='\t', encoding='latin1', na_values="n/a") gdp_per_capita.rename(columns={"2015": "GDP per capita"}, inplace=True) gdp_per_capita.set_index("Country", inplace=True) gdp_per_capita.head(2)

	Subject Descriptorobject	Unitsobject	Scaleobject	Country/Series-specific Notesobject	GDP per capitafloat64	Estimates Start Afterfloat64
Afghanistan	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	599.994	2013
Albania	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	3995.383	2010
2 rows × 6 columns

full_country_stats = pd.merge(left=oecd_bli, right=gdp_per_capita, left_index=True, right_index=True) full_country_stats.sort_values(by="GDP per capita", inplace=True) full_country_stats

	Air pollutionfloat649 – 46	Assault ratefloat641.3 – 12.8	Consultation on rule-makingfloat642 – 11.5	Dwellings without basic facilitiesfloat640 – 15.1	Educational attainmentfloat6434 – 94	Employees working very long hoursfloat640.16 – 40.86	Employment ratefloat6449 – 82	Homicide ratefloat640.3 – 25.5	Household net adjusted disposable incomefloat6411664 – 41355	Household net financial wealthfloat643251 – 145769	Housing expenditurefloat6411 – 26	Job securityfloat642.4 – 17.8	Life expectancyfloat6470.2 – 83.2	Life satisfactionfloat644.8 – 7.5	Long-term unemployment ratefloat640.01 – 18.39	Personal earningsfloat6416193 – 56340	Quality of support networkfloat6472 – 96	Rooms per personfloat640.9 – 2.5	Self-reported healthfloat6430 – 90	Student skillsfloat64402 – 542	Time devoted to leisure and personal carefloat6413.42 – 16.06	Voter turnoutfloat6449 – 93	Water qualityfloat6456 – 97	Years in educationfloat6414.4 – 19.8	Subject Descriptorobject Gross domestic product per capita, current prices100%	Unitsobject U.S. dollars100%	Scaleobject Units100%	Country/Series-specific Notesobject See notes for: Gross domestic product, current prices (National currency) Population (Persons).100%	GDP per capitafloat648669.998 – 101994.093	Estimates Start Afterfloat642013 – 2015
Brazil	18	7.9	4	6.7	45	10.41	67	25.5	11664	6844	21	4.6	73.7	7	1.97	17177	90	1.6	69	402	14.97	79	72	16.3	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	8669.998	2014
Mexico	30	12.8	9	4.2	37	28.83	61	23.4	13085	9056	21	4.9	74.6	6.7	0.08	16193	77	1	66	417	13.89	63	67	14.4	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	9009.28	2015
Russia	15	3.8	2.5	15.1	94	0.16	69	12.8	19292	3412	11	4	70.2	6	1.7	20885	90	0.9	37	481	14.97	65	56	16	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	9054.914	2015
Turkey	35	5	5.5	12.7	34	40.86	50	1.2	14095	3251	21	8.1	74.6	5.6	2.37	16919	86	1.1	68	462	13.42	88	62	16.4	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	9437.372	2013
Hungary	15	3.6	7.9	4.8	82	3.19	58	1.3	15442	13277	20	5.7	75.2	4.9	5.1	20948	87	1.1	57	487	15.04	62	77	17.6	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	12239.893999999998	2015
Expand rows 5 - 30
Denmark	15	3.9	7	0.9	78	2.03	73	0.3	26491	44488	24	5.6	80.1	7.5	1.78	48347	95	1.9	72	498	16.06	88	94	19.4	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	52114.165	2015
United States	18	1.5	8.3	0.1	89	11.3	67	5.2	41355	145769	18	5.9	78.7	7.2	1.91	56340	90	2.4	88	492	14.27	68	85	17.2	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	55805.204000000005	2015
Norway	16	3.3	8.1	0.3	82	2.82	75	0.6	33492	8797	17	3.1	81.5	7.4	0.32	50282	94	2	76	496	15.56	78	94	17.9	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	74822.106	2015
Switzerland	20	4.2	8.4	0	86	6.72	80	0.5	33491	108823	22	3	82.8	7.5	1.46	54236	96	1.8	81	518	14.98	49	96	17.3	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	80675.308	2015
Luxembourg	12	4.3	6	0.1	78	3.47	66	0.4	38951	61765	21	4.3	81.5	6.9	1.78	56021	87	2	72	490	15.12	91	86	15.1	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	101994.09300000001	2014
36 rows × 30 columns

full_country_stats[["GDP per capita", 'Life satisfaction']].loc["United States"]

remove_indices = [0, 1, 6, 8, 33, 34, 35] keep_indices = list(set(range(36)) - set(remove_indices)) sample_data = full_country_stats[["GDP per capita", 'Life satisfaction']].iloc[keep_indices] missing_data = full_country_stats[["GDP per capita", 'Life satisfaction']].iloc[remove_indices]

sample_data.plot(kind='scatter', x="GDP per capita", y='Life satisfaction', figsize=(5,3)) plt.axis([0, 60000, 0, 10]) position_text = { "Hungary": (5000, 1), "Korea": (18000, 1.7), "France": (29000, 2.4), "Australia": (40000, 3.0), "United States": (52000, 3.8), } for country, pos_text in position_text.items(): pos_data_x, pos_data_y = sample_data.loc[country] country = "U.S." if country == "United States" else country plt.annotate(country, xy=(pos_data_x, pos_data_y), xytext=pos_text, arrowprops=dict(facecolor='black', width=0.5, shrink=0.1, headwidth=5)) plt.plot(pos_data_x, pos_data_y, "ro") save_fig('money_happy_scatterplot') plt.show()

Saving figure money_happy_scatterplot

sample_data.to_csv(os.path.join("datasets", "lifesat", "lifesat.csv"))

sample_data.loc[list(position_text.keys())]

	GDP per capitafloat64	Life satisfactionfloat64
Hungary	12239.893999999998	4.9
Korea	27195.197	5.8
France	37675.006	6.5
Australia	50961.865	7.3
United States	55805.204000000005	7.2
5 rows × 2 columns

import numpy as np sample_data.plot(kind='scatter', x="GDP per capita", y='Life satisfaction', figsize=(5,3)) plt.axis([0, 60000, 0, 10]) X=np.linspace(0, 60000, 1000) plt.plot(X, 2*X/100000, "r") plt.text(40000, 2.7, r"$\theta_0 = 0$", fontsize=14, color="r") plt.text(40000, 1.8, r"$\theta_1 = 2 \times 10^{-5}$", fontsize=14, color="r") plt.plot(X, 8 - 5*X/100000, "g") plt.text(5000, 9.1, r"$\theta_0 = 8$", fontsize=14, color="g") plt.text(5000, 8.2, r"$\theta_1 = -5 \times 10^{-5}$", fontsize=14, color="g") plt.plot(X, 4 + 5*X/100000, "b") plt.text(5000, 3.5, r"$\theta_0 = 4$", fontsize=14, color="b") plt.text(5000, 2.6, r"$\theta_1 = 5 \times 10^{-5}$", fontsize=14, color="b") save_fig('tweaking_model_params_plot') plt.show()

Saving figure tweaking_model_params_plot

from sklearn import linear_model lin1 = linear_model.LinearRegression() Xsample = np.c_[sample_data["GDP per capita"]] ysample = np.c_[sample_data["Life satisfaction"]] lin1.fit(Xsample, ysample) t0, t1 = lin1.intercept_[0], lin1.coef_[0][0] t0, t1

sample_data.plot(kind='scatter', x="GDP per capita", y='Life satisfaction', figsize=(5,3)) plt.axis([0, 60000, 0, 10]) X=np.linspace(0, 60000, 1000) plt.plot(X, t0 + t1*X, "b") plt.text(5000, 3.1, r"$\theta_0 = 4.85$", fontsize=14, color="b") plt.text(5000, 2.2, r"$\theta_1 = 4.91 \times 10^{-5}$", fontsize=14, color="b") save_fig('best_fit_model_plot') plt.show()

Saving figure best_fit_model_plot

cyprus_gdp_per_capita = gdp_per_capita.loc["Cyprus"]["GDP per capita"] print(cyprus_gdp_per_capita) cyprus_predicted_life_satisfaction = lin1.predict([[cyprus_gdp_per_capita]])[0][0] cyprus_predicted_life_satisfaction

22587.49

sample_data.plot(kind='scatter', x="GDP per capita", y='Life satisfaction', figsize=(5,3), s=1) X=np.linspace(0, 60000, 1000) plt.plot(X, t0 + t1*X, "b") plt.axis([0, 60000, 0, 10]) plt.text(5000, 7.5, r"$\theta_0 = 4.85$", fontsize=14, color="b") plt.text(5000, 6.6, r"$\theta_1 = 4.91 \times 10^{-5}$", fontsize=14, color="b") plt.plot([cyprus_gdp_per_capita, cyprus_gdp_per_capita], [0, cyprus_predicted_life_satisfaction], "r--") plt.text(25000, 5.0, r"Prediction = 5.96", fontsize=14, color="b") plt.plot(cyprus_gdp_per_capita, cyprus_predicted_life_satisfaction, "ro") save_fig('cyprus_prediction_plot') plt.show()

Saving figure cyprus_prediction_plot

sample_data[7:10]

	GDP per capitafloat64	Life satisfactionfloat64
Portugal	19121.592	5.1
Slovenia	20732.482	5.7
Spain	25864.721	6.5
3 rows × 2 columns

(5.1+5.7+6.5)/3

backup = oecd_bli, gdp_per_capita def prepare_country_stats(oecd_bli, gdp_per_capita): oecd_bli = oecd_bli[oecd_bli["INEQUALITY"]=="TOT"] oecd_bli = oecd_bli.pivot(index="Country", columns="Indicator", values="Value") gdp_per_capita.rename(columns={"2015": "GDP per capita"}, inplace=True) gdp_per_capita.set_index("Country", inplace=True) full_country_stats = pd.merge(left=oecd_bli, right=gdp_per_capita, left_index=True, right_index=True) full_country_stats.sort_values(by="GDP per capita", inplace=True) remove_indices = [0, 1, 6, 8, 33, 34, 35] keep_indices = list(set(range(36)) - set(remove_indices)) return full_country_stats[["GDP per capita", 'Life satisfaction']].iloc[keep_indices]

# Code example import matplotlib.pyplot as plt import numpy as np import pandas as pd import sklearn # Load the data oecd_bli = pd.read_csv(datapath + "oecd_bli_2015.csv", thousands=',') gdp_per_capita = pd.read_csv(datapath + "gdp_per_capita.csv",thousands=',',delimiter='\t', encoding='latin1', na_values="n/a") # Prepare the data country_stats = prepare_country_stats(oecd_bli, gdp_per_capita) X = np.c_[country_stats["GDP per capita"]] y = np.c_[country_stats["Life satisfaction"]] # Visualize the data country_stats.plot(kind='scatter', x="GDP per capita", y='Life satisfaction') plt.show() # Select a linear model model = sklearn.linear_model.LinearRegression() # Train the model model.fit(X, y) # Make a prediction for Cyprus X_new = [[22587]] # Cyprus' GDP per capita print(model.predict(X_new)) # outputs [[ 5.96242338]]

[[5.96242338]]

oecd_bli, gdp_per_capita = backup

missing_data

	GDP per capitafloat64	Life satisfactionfloat64
Brazil	8669.998	7
Mexico	9009.28	6.7
Chile	13340.905	6.7
Czech Republic	17256.918	6.5
Norway	74822.106	7.4
Switzerland	80675.308	7.5
Luxembourg	101994.09300000001	6.9
7 rows × 2 columns

position_text2 = { "Brazil": (1000, 9.0), "Mexico": (11000, 9.0), "Chile": (25000, 9.0), "Czech Republic": (35000, 9.0), "Norway": (60000, 3), "Switzerland": (72000, 3.0), "Luxembourg": (90000, 3.0), }

sample_data.plot(kind='scatter', x="GDP per capita", y='Life satisfaction', figsize=(8,3)) plt.axis([0, 110000, 0, 10]) for country, pos_text in position_text2.items(): pos_data_x, pos_data_y = missing_data.loc[country] plt.annotate(country, xy=(pos_data_x, pos_data_y), xytext=pos_text, arrowprops=dict(facecolor='black', width=0.5, shrink=0.1, headwidth=5)) plt.plot(pos_data_x, pos_data_y, "rs") X=np.linspace(0, 110000, 1000) plt.plot(X, t0 + t1*X, "b:") lin_reg_full = linear_model.LinearRegression() Xfull = np.c_[full_country_stats["GDP per capita"]] yfull = np.c_[full_country_stats["Life satisfaction"]] lin_reg_full.fit(Xfull, yfull) t0full, t1full = lin_reg_full.intercept_[0], lin_reg_full.coef_[0][0] X = np.linspace(0, 110000, 1000) plt.plot(X, t0full + t1full * X, "k") save_fig('representative_training_data_scatterplot') plt.show()

Saving figure representative_training_data_scatterplot

full_country_stats.plot(kind='scatter', x="GDP per capita", y='Life satisfaction', figsize=(8,3)) plt.axis([0, 110000, 0, 10]) from sklearn import preprocessing from sklearn import pipeline poly = preprocessing.PolynomialFeatures(degree=60, include_bias=False) scaler = preprocessing.StandardScaler() lin_reg2 = linear_model.LinearRegression() pipeline_reg = pipeline.Pipeline([('poly', poly), ('scal', scaler), ('lin', lin_reg2)]) pipeline_reg.fit(Xfull, yfull) curve = pipeline_reg.predict(X[:, np.newaxis]) plt.plot(X, curve) save_fig('overfitting_model_plot') plt.show()

/opt/venv/lib/python3.7/site-packages/numpy/lib/nanfunctions.py:1544: RuntimeWarning: overflow encountered in multiply
  sqr = np.multiply(arr, arr, out=arr)
/opt/venv/lib/python3.7/site-packages/numpy/core/fromnumeric.py:87: RuntimeWarning: overflow encountered in reduce
  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
Saving figure overfitting_model_plot

full_country_stats.loc[[c for c in full_country_stats.index if "W" in c.upper()]]["Life satisfaction"]

gdp_per_capita.loc[[c for c in gdp_per_capita.index if "W" in c.upper()]].head()

	Subject Descriptorobject	Unitsobject	Scaleobject	Country/Series-specific Notesobject	GDP per capitafloat64	Estimates Start Afterfloat64
Botswana	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	6040.956999999999	2008
Kuwait	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	29363.027000000002	2014
Malawi	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	354.275	2011
New Zealand	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	37044.891	2015
Norway	Gross domestic product per capita, current prices	U.S. dollars	Units	See notes for: Gross domestic product, current prices (National currency) Population (Persons).	74822.106	2015
5 rows × 6 columns

plt.figure(figsize=(8,3)) plt.xlabel("GDP per capita") plt.ylabel('Life satisfaction') plt.plot(list(sample_data["GDP per capita"]), list(sample_data["Life satisfaction"]), "bo") plt.plot(list(missing_data["GDP per capita"]), list(missing_data["Life satisfaction"]), "rs") X = np.linspace(0, 110000, 1000) plt.plot(X, t0full + t1full * X, "r--", label="Linear model on all data") plt.plot(X, t0 + t1*X, "b:", label="Linear model on partial data") ridge = linear_model.Ridge(alpha=10**9.5) Xsample = np.c_[sample_data["GDP per capita"]] ysample = np.c_[sample_data["Life satisfaction"]] ridge.fit(Xsample, ysample) t0ridge, t1ridge = ridge.intercept_[0], ridge.coef_[0][0] plt.plot(X, t0ridge + t1ridge * X, "b", label="Regularized linear model on partial data") plt.legend(loc="lower right") plt.axis([0, 110000, 0, 10]) save_fig('ridge_model_plot') plt.show()

Saving figure ridge_model_plot

backup = oecd_bli, gdp_per_capita def prepare_country_stats(oecd_bli, gdp_per_capita): return sample_data

# Replace this linear model: import sklearn.linear_model model = sklearn.linear_model.LinearRegression()

# with this k-neighbors regression model: import sklearn.neighbors model = sklearn.neighbors.KNeighborsRegressor(n_neighbors=3)

X = np.c_[country_stats["GDP per capita"]] y = np.c_[country_stats["Life satisfaction"]] # Train the model model.fit(X, y) # Make a prediction for Cyprus X_new = np.array([[22587.0]]) # Cyprus' GDP per capita print(model.predict(X_new)) # outputs [[ 5.76666667]]

[[5.76666667]]