Setup
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
import chart_studio
#chart_studio.tools.set_credentials_file(username='econdata777', api_key='ADDhere')
import chart_studio.plotly as save2cs
import statsmodels.api as sm
import statsmodels.formula.api as smf
import warnings
#warnings.filterwarnings('ignore')
Import data
df1 = pd.read_csv("https://raw.githubusercontent.com/quarcs-lab/mendez2020-convergence-clubs-code-data/master/assets/dat.csv")
#df1 = pd.read_csv("https://raw.githubusercontent.com/quarcs-lab/mendez2020-convergence-clubs-code-data/master/assets/dat.csv", parse_dates =['year'])
df1
idint64
1 - 108
countryobject
Zambia0.9%
Italy0.9%
106 others98.1%
2699
108
Zambia
1199
48
Italy
1324
53
Kyrgyz Republic
1299
52
Kenya
1274
51
Kazakhstan
2649
106
Vietnam
1249
50
Japan
1224
49
Jamaica
1174
47
Israel
1374
55
Lithuania
#df1.head()
df1.columns
df1.dtypes
df1['country'].unique()
Dataset definitions
# Import definitions of dataset
df1_def = pd.read_csv("https://raw.githubusercontent.com/quarcs-lab/mendez2020-convergence-clubs-code-data/master//assets/dat-definitions.csv")
df1_def
var_nameobject
country3.6%
year3.6%
26 others92.9%
var_defobject
Standardized country name (from PWT)3.6%
Year3.6%
26 others92.9%
0
country
Standardized country name (from PWT)
1
year
Year
2
Y
GDP
3
K
Physical Capital
4
pop
Population
5
L
Labor Force
6
s
Years of Schooling
7
alpha_it
Variable Capital Share
8
GDPpc
GDP per capita
9
lp
Labor Productivity
Descriptive statistics
df1.describe().round(2)
idfloat64
yearfloat64
count
2700
2700
mean
54.5
2002
std
31.18
7.21
min
1
1990
25%
27.75
1996
50%
54.5
2002
75%
81.25
2008
max
108
2014
Prepare data
select
df1.GDPpc
df2 = df1[['country', 'year', 'GDPpc', 'h', 'ky', 'TFP']]
df2
countryobject
Albania0.9%
Algeria0.9%
106 others98.1%
yearint64
1990 - 2014
0
Albania
1990
1
Albania
1991
2
Albania
1992
3
Albania
1993
4
Albania
1994
5
Albania
1995
6
Albania
1996
7
Albania
1997
8
Albania
1998
9
Albania
1999
query
df_2014 = df1[['country', 'region', 'hi1990', 'year', 'GDPpc', 'h', 'ky', 'TFP']].query("year == 2014")
df_2014
countryobject
Albania0.9%
Algeria0.9%
106 others98.1%
regionobject
Europe27.8%
Asia25.9%
3 others46.3%
24
Albania
Europe
49
Algeria
Africa
74
Argentina
Americas
99
Armenia
Asia
124
Australia
Oceania
149
Austria
Europe
174
Bangladesh
Asia
199
Belgium
Europe
224
Benin
Africa
249
Bolivia
Americas
df_japan = df1[['country', 'year', 'GDPpc', 'h', 'ky', 'TFP']].query("country == 'Japan'")
df_japan
countryobject
Japan100%
yearint64
1990 - 2014
1225
Japan
1990
1226
Japan
1991
1227
Japan
1992
1228
Japan
1993
1229
Japan
1994
1230
Japan
1995
1231
Japan
1996
1232
Japan
1997
1233
Japan
1998
1234
Japan
1999
df1[['country', 'year', 'h']].query("country==['Japan', 'China','Bolivia'] and year == 2014")
countryobject
yearint64
249
Bolivia
2014
474
China
2014
1249
Japan
2014
# Define the logical condictions
is_Asia = df1['region'] == 'Asia'
is_HighIncome = df1['hi1990'] == 'yes'
is_1990 = df1['year'] == 1990
# Apply the logical conditions
df1[is_Asia & is_HighIncome & is_1990]
idint64
countryobject
1000
41
Hong Kong
1150
47
Israel
1225
50
Japan
1975
80
Saudi Arabia
2050
83
Singapore
2300
93
Taiwan
groupby
df_2014.groupby('region').mean().round(2)
yearfloat64
GDPpcfloat64
Africa
2014
3758.65
Americas
2014
14866.46
Asia
2014
17677.28
Europe
2014
32943.2
Oceania
2014
38651.73
df_2014.groupby('region').std().round(2)
yearfloat64
GDPpcfloat64
Africa
0
3675.67
Americas
0
11908.32
Asia
0
16765.34
Europe
0
14573.41
Oceania
0
6984.17
#df_2014.groupby('region')['GDPpc'].agg(['mean', 'std', 'min', 'max']).round(2)
df_2014.groupby('region').GDPpc.agg(['mean', 'std', 'min', 'max']).round(2)
meanfloat64
stdfloat64
Africa
3758.65
3675.67
Americas
14866.46
11908.32
Asia
17677.28
16765.34
Europe
32943.2
14573.41
Oceania
38651.73
6984.17
df1.groupby('year').s.agg(['mean', 'std', 'min', 'max']).round(2)
meanfloat64
6.5 - 8.81
stdfloat64
2.9 - 3.01
1990
6.5
2.9
1991
6.6
2.91
1992
6.71
2.92
1993
6.81
2.93
1994
6.92
2.95
1995
7.02
2.96
1996
7.11
2.96
1997
7.2
2.96
1998
7.29
2.96
1999
7.38
2.96
pivot_table
# Pivot panel data from long form to wide form
df1_s_wide = df1.pivot_table(
index = ['country', 'region', 'hi1990'],
columns = 'year',
values = 's').reset_index(drop=False)
# Make sure the column names are strings
df1_s_wide.columns = df1_s_wide.columns.astype(str)
df1_s_wide
countryobject
Albania0.9%
Algeria0.9%
106 others98.1%
regionobject
Europe27.8%
Asia25.9%
3 others46.3%
0
Albania
Europe
1
Algeria
Africa
2
Argentina
Americas
3
Armenia
Asia
4
Australia
Oceania
5
Austria
Europe
6
Bangladesh
Asia
7
Belgium
Europe
8
Benin
Africa
9
Bolivia
Americas
df1_s_wide.describe().round(2)
1990float64
1991float64
count
108
108
mean
6.5
6.6
std
2.9
2.91
min
0.89
0.92
25%
4.16
4.27
50%
6.98
7.14
75%
8.79
8.94
max
12.2
12.28
df1_s_wide.std().round(2).plot();
<ipython-input-23-ea0ee6fd29ec>:1: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError. Select only valid columns before calling the reduction.
df1_s_wide.std().round(2).plot();
loc
df1_s_wide.loc[:,'1990':'2014']
1990float64
0.89268953 - 12.19856
1991float64
0.91593128 - 12.27736
0
8.497386
8.4427032
1
4.7446432
4.8778758
2
8.3714237
8.4251013
3
10.39716
10.40424
4
11.17822
11.18183
5
8.3587255
8.4494009
6
3.1310649
3.2406831
7
9.4271355
9.5408058
8
2.156024
2.246896
9
7.2618189
7.3760309
df1_s_wide.loc[:,'1990':'2014'].std().round(2).plot();
isin
df1[df1.country.isin(['Vietnam', 'China' 'Indonesia'])]
idint64
106 - 106
countryobject
Vietnam100%
2625
106
Vietnam
2626
106
Vietnam
2627
106
Vietnam
2628
106
Vietnam
2629
106
Vietnam
2630
106
Vietnam
2631
106
Vietnam
2632
106
Vietnam
2633
106
Vietnam
2634
106
Vietnam
Visualize data
Strip plot
px.strip(df_2014, x = 'GDPpc', hover_name= 'country')
#fig1 = px.strip(df_2014, x = 'GDPpc', hover_name= 'country')
#save2cs.plot(fig1, filename = 'figureName', auto_open=True)
px.strip(df1,
x = 'GDPpc',
y = 'region',
hover_name= 'country',
hover_data= ['h', 'ky'],
color= 'region',
animation_frame= 'year')
Line plots
px.line(df1, x='year', y='log_lp', color='country')
px.line(df1, x='year', y='log_lp', color='country', facet_col= 'region', facet_col_wrap= 2, height= 800)
px.line(df1.query("country=='Japan'"), x="year", y="log_lp")
fig1 = px.line(df1.query("country=='Japan'"), x="year", y="log_lp")
fig1.write_html("fig1.html")
px.line(
df1.query("country==['Japan', 'China','Bolivia']"),
x="year",
y="log_lp",
color="country"
)
px.line(
df1.query("region=='Europe'"),
x="year",
y="log_lp",
color="country"
)
Histogram
px.histogram(df_2014, x = 'GDPpc', hover_name= 'country', color= 'hi1990', marginal='rug')
px.histogram(
df1,
x="log_lp",
color="hi1990",
hover_name= 'country',
marginal='box',
animation_frame = 'year'
)
Box plot
px.box(
df1,
x="log_lp",
color="hi1990",
hover_name= 'country',
animation_frame = 'year'
)
Violin plot
px.violin(
df1,
y="hi1990",
x="log_lp",
range_x= [6, 12.5],
color="hi1990",
box=True,
hover_name= 'country',
animation_frame = 'year',
points="all")
ECDF
px.ecdf(
df1,
x="log_lp",
range_x= [7, 12],
color="hi1990",
hover_name= 'country',
animation_frame = 'year'
)
Treemap plot
px.treemap(df1.query("year == 2014"), color = "log_lp", values = "pop", path = ["region", "country"], hover_name = "country")
Sunburst plot
px.sunburst(df1.query("year == 2014"), color = "log_lp", values = "pop", path = ["region", "country"], hover_name = "country")
Scatter plots
Simple
px.scatter(
df1,
x="log_h",
y="log_lp",
color="region",
hover_name="country",
animation_frame = 'year'
)
Regression
px.scatter(
df1.query("year == 1990"),
x="log_h",
y="log_lp",
color="hi1990",
hover_name="country",
hover_data= ['region'],
trendline="ols",
trendline_scope="overall"
)
Regression by groups
px.scatter(
df1,
x="log_h",
y="log_lp",
color="hi1990",
hover_name="country",
animation_frame = 'year',
trendline="ols")
Lowess regression
px.scatter(
df1,
x="log_h",
y="log_lp",
range_x= [0.2, 1.62],
range_y= [7, 12.2],
color="hi1990",
hover_name="country",
animation_frame = 'year',
trendline="lowess")
Margins
px.scatter(
df1.query("year == 1990"),
x="log_h",
y="log_lp",
hover_name="country",
color="hi1990",
trendline="ols",
marginal_x="box",
marginal_y="box")
Multivariate
px.scatter(
df1.query("year == 1990"),
x="log_h",
y="log_lp",
color="region",
size="pop", size_max=60,
hover_name="country",
labels=dict(log_h="Human capital index in 1990 (in logs)",
log_lp="Labor productivity in 1990 (in logs)",
region="Continent",
pop= "Population")
)
Customized
fig = px.scatter(df1.query("year == 1990"),
y="log_lp",
x="log_h",
log_x = False,
color = "region",
size ="pop", size_max=60,
hover_name = "country",
height =500, width=800,
template = "simple_white",
color_discrete_sequence=px.colors.qualitative.G10,
#title = "Year 1990",
labels=dict(region = "Continent",
pop = "Population",
log_lp = "Labor productivity (in logs)",
log_h = "Human capital index (in logs)")
)
fig.update_layout(font_family = "Rockwell",
legend=dict(orientation = "h", title="", y=1.1, x=1, xanchor="right", yanchor="bottom"))
fig.add_hline(df1.query("year == 1990")['log_lp'].mean(), line_width=1, line_dash="dot")
fig.add_vline(df1.query("year == 1990")['log_h'].mean(), line_width=1, line_dash="dot")
fig.show()
px.scatter(df1,
animation_frame="year",
x="log_h",
y="log_lp",
range_x= [0.2, 1.62],
range_y= [7, 12.2],
color="region",
size="pop", size_max=60,
hover_name="country",
hover_data = ['log_ky', 'log_tfp'],
labels=dict(log_h="Human capital index (in logs)",
log_lp="Labor productivity (in logs)",
region="Continent",
pop= "Population")
)
px.scatter(
df1,
x="log_h",
y="log_lp",
color="region",
size="pop", size_max=60,
trendline="ols",
hover_name="country",
facet_col="hi1990",
animation_frame="year"
)
px.density_contour(
df1,
x="log_h",
y="log_lp",
hover_name="country",
marginal_x="histogram",
marginal_y="histogram",
animation_frame="year"
)
px.density_contour(
df1,
x="log_h",
y="log_lp",
marginal_x="box",
marginal_y="box",
animation_frame="year"
)
px.density_contour(
df1,
x="log_h",
y="log_lp",
hover_name="country",
marginal_x="rug",
marginal_y="rug",
animation_frame="year"
)
den = px.density_contour(
df1.query("year == 2014"),
x="log_h",
y="log_lp",
)
den.update_traces(contours_coloring="fill", contours_showlabels = True)
den.show()
den = px.density_contour(
df1.query("year == 2014"),
x="log_h",
y="log_lp",
)
den.update_traces(contours_coloring="fill", contours_showlabels = True, colorscale = 'Viridis')
den.show()
3D
px.scatter_3d(
df1.query("year == 1990"),
x="log_h",
y="log_lp",
z= "pop",
color="region",
hover_name="country"
)
px.choropleth(
df1.query("year == 1990"),
locations="isocode",
color="log_lp",
hover_name="country",
color_continuous_scale=px.colors.sequential.Plasma,
projection="natural earth")
px.choropleth(
df1,
locations="isocode",
color="log_lp",
hover_name="country",
animation_frame="year",
color_continuous_scale=px.colors.sequential.Plasma,
projection="natural earth")