Motivation

Setup

# Load libraries import numpy as np import pandas as pd pd.set_option('display.max_columns', None) import matplotlib.pyplot as plt plt.style.use('ggplot') from matplotlib.ticker import FormatStrFormatter from matplotlib_scalebar.scalebar import ScaleBar from pylab import rcParams import matplotlib as mpl mpl.rcParams['figure.dpi'] = 72 import seaborn as sns #sns.set_style("darkgrid") #sns.set_context(context="paper", font_scale=1.5, rc=None) #sns.set(font="serif") import plotly.express as px import plotly.graph_objects as go import geopandas as gpd import contextily as cx import libpysal from libpysal import weights from libpysal.weights import Queen import esda from esda.moran import Moran, Moran_Local import splot from splot.esda import moran_scatterplot, plot_moran, lisa_cluster, plot_local_autocorrelation from splot.libpysal import plot_spatial_weights from giddy.directional import Rose import statsmodels.api as sm import statsmodels.formula.api as smf from spreg import OLS from spreg import MoranRes from spreg import ML_Lag from spreg import ML_Error from mgwr.gwr import GWR, MGWR from mgwr.sel_bw import Sel_BW from mgwr.utils import shift_colormap, truncate_colormap import warnings warnings.filterwarnings('ignore')

Import data

df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 54668 entries, 1844 to 45530
Data columns (total 32 columns):
 #   Column             Non-Null Count  Dtype  
---  ------             --------------  -----  
 0   country            54668 non-null  object 
 1   year               54668 non-null  int64  
 2   region             54668 non-null  object 
 3   level              54668 non-null  object 
 4   GDLcode            54668 non-null  object 
 5   GDLcode_id         54668 non-null  int64  
 6   continentName      54668 non-null  object 
 7   subcontinent       54558 non-null  object 
 8   shdi               54668 non-null  float64
 9   healthindex        54668 non-null  float64
 10  incindex           54668 non-null  float64
 11  edindex            54668 non-null  float64
 12  esch               54668 non-null  float64
 13  msch               54668 non-null  float64
 14  lifexp             54668 non-null  float64
 15  gnic               54668 non-null  float64
 16  lgnic              54668 non-null  float64
 17  pop                54164 non-null  float64
 18  tr6_lgnic          54668 non-null  float64
 19  tr6_shdi           54668 non-null  float64
 20  tr6_healthindex    54668 non-null  float64
 21  tr6_incindex       54668 non-null  float64
 22  tr6_edindex        54668 non-null  float64
 23  tr6_esch           54668 non-null  float64
 24  tr6_msch           54668 non-null  float64
 25  tr6_lifexp         54668 non-null  float64
 26  countryName_std    54668 non-null  object 
 27  countryName2       54558 non-null  object 
 28  continentName_std  54668 non-null  object 
 29  iso_code           54668 non-null  object 
 30  iso3               54558 non-null  object 
 31  POLY_IDcountry     54478 non-null  float64
dtypes: float64(19), int64(2), object(11)
memory usage: 13.8+ MB

Prepare data

df["year"]= df["year"].astype(str)

# Remove national observations df1 = df[df.level != 'National']

df1in1990 = df.query("year == '1990'")

df1in2019 = df.query("year == '2019'")

# Keep only national observations df2 = df[df.level == 'National']

Descriptive statistics

df1[['shdi', 'healthindex', 'incindex', 'edindex']].describe().round(2)

shdifloat64

healthindexfloat64

count

49606

mean

0.64

0.74

std

0.17

0.15

min

0.17

0.07

25%

0.51

0.65

50%

0.66

0.78

75%

0.77

0.86

max

0.98

1

df1.query("year == '2019'").groupby('subcontinent')['shdi'].agg(['mean', 'std', 'min', 'max']).round(2)

meanfloat64

0.5 - 0.93

stdfloat64

0.02 - 0.11

Australia and New Zealand

0.93

0.02

Northern America

0.92

0.02

Northern Europe

0.91

0.04

Western Europe

0.91

0.04

Eastern Europe

0.84

0.05

Southern Europe

0.84

0.06

Eastern Asia

0.8

0.09

South America

0.75

0.06

Central Asia

0.73

0.06

Western Asia

0.72

0.09

Caribbean

0.72

0.11

Central America

0.71

0.09

Polynesia

0.7

0.02

South-Eastern Asia

0.67

0.09

Southern Asia

0.67

0.09

Northern Africa

0.66

0.1

Southern Africa

0.64

0.09

Micronesia

0.61

0.02

Melanesia

0.61

0.09

Middle Africa

0.53

0.1

Eastern Africa

0.51

0.1

Western Africa

0.5

0.09

df1.query("year == '2019' & subcontinent == 'South America'")[['shdi', 'healthindex', 'incindex', 'edindex']].describe().round(2)

shdifloat64

healthindexfloat64

count

153

mean

0.75

0.85

std

0.06

0.05

min

0.56

0.69

25%

0.71

0.81

50%

0.74

0.86

75%

0.78

0.89

max

0.88

0.95

df1.query("year == '2019' & country == 'Bolivia'")[['region', 'shdi', 'healthindex', 'incindex', 'edindex']].round(2)

regionobject

Santa Cruz11.1%

Tarija11.1%

7 others77.8%

shdifloat64

0.63 - 0.78

5328

Santa Cruz

0.78

5298

Tarija

0.74

5388

Pando

0.74

5238

Oruro

0.73

5358

Beni

0.73

5178

La Paz

0.72

5208

Cochabamba

0.71

5148

Chuquisaca

0.69

5268

Potosi

0.63

Exploratory data analysis (EDA)

Cross-section dynamics

Strip plot

px.strip(df2, x = 'shdi', y = 'continentName', title = "Dynamics of human development across countries classified by continent", range_x= [0, 1], color = 'continentName', hover_name= 'countryName2', hover_data = ['countryName2'], animation_frame= 'year', labels=dict(continentName = "Continent", shdi ="Subnational human development index") )

px.strip(df1, x = 'shdi', y = 'continentName', title = "Dynamics of human development across subnational regions classified by continent", range_x= [0, 1], color = 'continentName', hover_name= 'region', hover_data = ['countryName2'], animation_frame= 'year', labels=dict(continentName = "Continent", countryName2 = "Country", shdi ="Subnational human development index") )

px.strip(df1[df1['country'].isin(['China', 'Bolivia', 'Japan'])], x = 'shdi', y = 'country', color = 'country', title = "Dynamics of human development across subnational regions of selected countries", hover_name= 'region', hover_data = ['countryName2'], range_x = [0.3, 1], animation_frame = 'year', labels=dict(continentName = "Continent", country = "Country", shdi ="Subnational human development index") )

Histogram

px.histogram(df1[df1['subcontinent'].notna()], x = 'shdi', range_x= [0.15, 1], nbins= 80, color = 'subcontinent', title = "Distribution of human development across subnational regions classified by subcontinent", hover_name= 'region', hover_data= ['countryName2'], marginal='rug', # rug and box animation_frame = 'year', labels=dict(subcontinent = "Subcontinent", shdi ="Subnational human development index") )

ECDF

px.ecdf(df1[df1['continentName'].notna()], x="shdi", range_x= [0.15, 1], color="continentName", hover_name= 'region', hover_data= ['country'], title = "Cumulative distribution of human development across subnational regions classified by continent", #markers=True, #lines=False, marginal="rug", # histogram animation_frame = 'year', labels=dict(continentName = "Continent", shdi ="Subnational human development index") )

Boxplot

px.box(df1[df1['subcontinent'].notna()], x = 'shdi', y = 'subcontinent', range_x= [0.15, 1], color = 'subcontinent', hover_name= 'region', hover_data = ['countryName2'], title = "Distribution of human development across subnational regions classified by subcontinent", animation_frame= 'year', labels=dict(subcontinent = "Sub-continent", countryName2 = "Country name", shdi ="Subnational human development index") )

Maps

px.choropleth( df2, locations="iso3", color="shdi", hover_name="country", animation_frame="year", title = "Spatial distribution of human development across countries", color_continuous_scale=px.colors.sequential.Plasma, projection="natural earth")

subnationalMap = gpd.read_file("https://gist.github.com/cmg777/81b6e2b1ea5865c09daf3b2bbf35bbf8/raw/5e0a0370f453421bc503594cfae1e466c89e449f/GDL%2520Shapefiles%2520V4.geojson") subnationalMap.info()

<class 'geopandas.geodataframe.GeoDataFrame'>
RangeIndex: 1745 entries, 0 to 1744
Data columns (total 7 columns):
 #   Column    Non-Null Count  Dtype   
---  ------    --------------  -----   
 0   GDLcode   1745 non-null   object  
 1   constant  1745 non-null   object  
 2   iso_code  1742 non-null   object  
 3   country   1742 non-null   object  
 4   region    1742 non-null   object  
 5   shdi      1742 non-null   float64 
 6   geometry  1745 non-null   geometry
dtypes: float64(1), geometry(1), object(5)
memory usage: 95.6+ KB

gdf2019 = subnationalMap.merge(df1in2019, on = 'GDLcode', how = 'left', suffixes=('', '_y')) # To remove the suffixes that appear after merge we use gdf2019.drop(gdf2019.filter(regex='_y$').columns.tolist(), axis=1, inplace=True)

gdf2019.info()

<class 'geopandas.geodataframe.GeoDataFrame'>
Int64Index: 1745 entries, 0 to 1744
Data columns (total 34 columns):
 #   Column             Non-Null Count  Dtype   
---  ------             --------------  -----   
 0   GDLcode            1745 non-null   object  
 1   constant           1745 non-null   object  
 2   iso_code           1742 non-null   object  
 3   country            1742 non-null   object  
 4   region             1742 non-null   object  
 5   shdi               1742 non-null   float64 
 6   geometry           1745 non-null   geometry
 7   year               1727 non-null   object  
 8   level              1727 non-null   object  
 9   GDLcode_id         1727 non-null   float64 
 10  continentName      1727 non-null   object  
 11  subcontinent       1717 non-null   object  
 12  healthindex        1727 non-null   float64 
 13  incindex           1727 non-null   float64 
 14  edindex            1727 non-null   float64 
 15  esch               1727 non-null   float64 
 16  msch               1727 non-null   float64 
 17  lifexp             1727 non-null   float64 
 18  gnic               1727 non-null   float64 
 19  lgnic              1727 non-null   float64 
 20  pop                1716 non-null   float64 
 21  tr6_lgnic          1727 non-null   float64 
 22  tr6_shdi           1727 non-null   float64 
 23  tr6_healthindex    1727 non-null   float64 
 24  tr6_incindex       1727 non-null   float64 
 25  tr6_edindex        1727 non-null   float64 
 26  tr6_esch           1727 non-null   float64 
 27  tr6_msch           1727 non-null   float64 
 28  tr6_lifexp         1727 non-null   float64 
 29  countryName_std    1727 non-null   object  
 30  countryName2       1717 non-null   object  
 31  continentName_std  1727 non-null   object  
 32  iso3               1717 non-null   object  
 33  POLY_IDcountry     1717 non-null   float64 
dtypes: float64(20), geometry(1), object(13)
memory usage: 477.1+ KB

fig, ax = plt.subplots(figsize=(12,8)) gdf2019.plot(column="shdi", scheme='BoxPlot', cmap='coolwarm', legend=True, ax=ax, legend_kwds={'bbox_to_anchor':(1.00, 0.65)}) plt.title('Spatial distribution of human development in 2019: \nWhich regions are below/above the median?') plt.tight_layout() ax.axis("off") #plt.savefig('myMap.png',dpi=300, bbox_inches='tight') plt.show()

Time series evolution

Continents

fig = px.line( df2, x="year", y="shdi", # Compare tr6_shdi and shdi log_y= True, color="country", hover_name="country", hover_data= ['country'], title = "Evolution of human development across countries classified by continent", labels=dict(shdi="HDI", continentName ="Continent", country = "Country", year = "Year"), facet_col="continentName", facet_col_wrap = 2, facet_row_spacing = 0.01, height= 1800 ) fig.update_layout(showlegend=False)

fig = px.line( df1, x="year", y="shdi", # Compare tr6_shdi and shdi log_y= True, color="region", hover_name="region", hover_data= ['country'], title = "Evolution of human development across subnational regions classified by continent", labels=dict(shdi="HDI", continentName ="Continent", subcontinent = "Subcontinent", year = "Year"), facet_col="continentName", facet_col_wrap = 2, facet_row_spacing = 0.01, height= 1800 ) fig.update_layout(showlegend=False)

Subcontinents

fig = px.line( df2[df2['subcontinent'].notna()], x="year", y="shdi", log_y= True, color="country", hover_name="subcontinent", hover_data= ['country'], title = "Evolution of human development across countries classified by continent", labels=dict(shdi="HDI", subcontinent ="Subcontinent", year = "Year"), facet_col="subcontinent", facet_col_wrap = 3, facet_row_spacing = 0.01, height= 2400 ) fig.update_layout(showlegend=False)

fig = px.line( df1[df1['subcontinent'].notna()], x="year", y="shdi", log_y= True, color="region", hover_name="region", hover_data= ['country', 'continentName', 'subcontinent'], title = "Evolution of human development across subnational regions classified by continent", labels=dict(shdi="HDI", subcontinent ="Subcontinent", year = "Year"), facet_col="subcontinent", facet_col_wrap = 3, facet_row_spacing = 0.01, height= 2400 ) fig.update_layout(showlegend=False)

Countries

fig = px.line( df2[df2['country'].isin(['China', 'Bolivia', 'Japan'])], x="year", y="shdi", log_y= True, color="country", hover_name="subcontinent", hover_data= ['country'], title = "Evolution of human development in selected countries", labels=dict(shdi="Human development index", country ="Country", year = "Year"), facet_col="country", facet_col_wrap = 3, #facet_row_spacing = 0.01, #height= 2400 ) fig.update_layout(showlegend=False)

fig = px.line( df1[df1['country'].isin(['China', 'Bolivia', 'Japan'])], x="year", y="shdi", log_y= True, color="region", hover_name="subcontinent", hover_data= ['country'], title = "Evolution of human development across subnational regions of selected countries", labels=dict(shdi="Human development index", country ="Country", year = "Year"), facet_col="country", facet_col_wrap = 3, #facet_row_spacing = 0.01, #height= 2400 ) fig.update_layout(showlegend=False)

Correlations over time

Continents

px.scatter(df2, y = "lifexp", x = "lgnic", range_y= [30, 90], range_x= [5.5, 12.5], hover_name = "country", hover_data= ['country'], color = "continentName", #size = "pop", size_max = 60, trendline= 'ols', #trendline_scope= "overall", #marginal_x= 'box', #marginal_y= 'box', animation_frame= 'year', title = "The relationship between income and health across countries classified by continent", labels=dict(country = "Country", continentName = "Continent", lgnic ="Log of gross national income per capita", lifexp="Life Expectancy (years)") )

px.scatter(df1, y = "lifexp", x = "lgnic", range_y= [30, 90], range_x= [5.5, 12.5], hover_name = "region", hover_data= ['country'], color = "continentName", #size = "pop", size_max = 60, trendline= 'ols', #trendline_scope= "overall", #marginal_x= 'box', #marginal_y= 'box', animation_frame= 'year', title = "Relationship between income and health across subnational regions classified by continent", labels=dict(country = "Country", continentName = "Continent", lgnic ="Log of gross national income per capita", lifexp="Life Expectancy (years)") )

Subcontinents

px.scatter(df2[df2['subcontinent'].notna()], y = "lifexp", x = "lgnic", range_y= [30, 90], range_x= [5.5, 12.5], hover_name = "country", hover_data= ['country'], color = "subcontinent", #size = "pop", size_max = 60, trendline= 'ols', #trendline_scope= "overall", #marginal_x= 'box', #marginal_y= 'box', animation_frame= 'year', title = "Relationship between income and health across countries classified by subcontinent", labels=dict(country = "Country", subcontinent = "Subcontinent", lgnic ="Log of gross national income per capita", lifexp="Life Expectancy (years)") )

px.scatter(df1[df1['subcontinent'].notna()], y = "lifexp", x = "lgnic", range_y= [30, 90], range_x= [5.5, 12.5], hover_name = "region", hover_data= ['country'], color = "subcontinent", #size = "pop", size_max = 60, trendline= 'ols', #trendline_scope= "overall", #marginal_x= 'box', #marginal_y= 'box', animation_frame= 'year', title = "Relationship between income and health across subnational regions classified by subcontinent", labels=dict(country = "Country", subcontinent = "Subcontinent", lgnic ="Log of gross national income per capita", lifexp="Life Expectancy (years)") )

Countries

px.scatter(df1[df1['country'].isin(['China', 'Bolivia', 'Japan'])], y = "lifexp", x = "lgnic", range_y= [50, 90], range_x= [6, 11], hover_name = "region", hover_data= ['country'], color = "country", #size = "pop", size_max = 60, trendline= 'ols', #marginal_x= 'box', #marginal_y= 'box', animation_frame= 'year', title = "Relationship between income and health across subnational regions in selected countries", labels=dict(country = "Country", lgnic ="Log of gross national income per capita", lifexp="Life Expectancy (years)") )

Export data

Panel data

southAmerica = df1.query("subcontinent == 'South America'").reset_index(drop=True) southAmerica

countryobject

Colombia21.9%

Brazil17.9%

10 others60.2%

yearobject

20043.4%

20053.4%

28 others93.2%

0

Argentina

1990

1

Argentina

1990

2

Argentina

1990

3

Argentina

1990

4

Argentina

1990

5

Argentina

1990

6

Argentina

1990

7

Argentina

1990

8

Argentina

1990

9

Argentina

1990

southAmerica.to_csv('southAmerica.csv', index= False)

Digital map

gdf2019.info()

<class 'geopandas.geodataframe.GeoDataFrame'>
Int64Index: 1745 entries, 0 to 1744
Data columns (total 34 columns):
 #   Column             Non-Null Count  Dtype   
---  ------             --------------  -----   
 0   GDLcode            1745 non-null   object  
 1   constant           1745 non-null   object  
 2   iso_code           1742 non-null   object  
 3   country            1742 non-null   object  
 4   region             1742 non-null   object  
 5   shdi               1742 non-null   float64 
 6   geometry           1745 non-null   geometry
 7   year               1727 non-null   object  
 8   level              1727 non-null   object  
 9   GDLcode_id         1727 non-null   float64 
 10  continentName      1727 non-null   object  
 11  subcontinent       1717 non-null   object  
 12  healthindex        1727 non-null   float64 
 13  incindex           1727 non-null   float64 
 14  edindex            1727 non-null   float64 
 15  esch               1727 non-null   float64 
 16  msch               1727 non-null   float64 
 17  lifexp             1727 non-null   float64 
 18  gnic               1727 non-null   float64 
 19  lgnic              1727 non-null   float64 
 20  pop                1716 non-null   float64 
 21  tr6_lgnic          1727 non-null   float64 
 22  tr6_shdi           1727 non-null   float64 
 23  tr6_healthindex    1727 non-null   float64 
 24  tr6_incindex       1727 non-null   float64 
 25  tr6_edindex        1727 non-null   float64 
 26  tr6_esch           1727 non-null   float64 
 27  tr6_msch           1727 non-null   float64 
 28  tr6_lifexp         1727 non-null   float64 
 29  countryName_std    1727 non-null   object  
 30  countryName2       1717 non-null   object  
 31  continentName_std  1727 non-null   object  
 32  iso3               1717 non-null   object  
 33  POLY_IDcountry     1717 non-null   float64 
dtypes: float64(20), geometry(1), object(13)
memory usage: 477.1+ KB

gdf2019.subcontinent.unique()

southAmerica2019 = gdf2019.query("subcontinent == 'South America'").reset_index(drop=True) southAmerica2019

GDLcodeobject

ARGr1010.7%

ARGr1020.7%

151 others98.7%

constantobject

World100%

0

ARGr101

World

1

ARGr102

World

2

ARGr103

World

3

ARGr104

World

4

ARGr105

World

5

ARGr106

World

6

ARGr107

World

7

ARGr108

World

8

ARGr109

World

9

ARGr110

World

southAmerica2019.to_file("southAmerica2019.geojson", driver='GeoJSON')