import pandas as pd
import altair as alt
url = "https://gist.github.com/scotthmurray/f71065a5694f22259bf9/raw/ce891b9fe7ec3c5cab3308f4cd0c8eeccc36f6c7/Better%2520Life%2520Index%2520Data.csv"
oecd = pd.read_csv(url)
# to keep things consistent with the other examples we turn column names to lowercase
oecd.columns = map(str.lower, oecd.columns)
geonames_full = pd.read_csv("https://www.geonames.org/countryInfoCSV", sep='\t', keep_default_na=False)
geonames_full.head()
0
AD
AND
1
AE
ARE
2
AF
AFG
3
AG
ATG
4
AI
AIA
5 rows, showing
per page
Page of 1
geonames = geonames_full[['name', 'iso alpha3', 'areaInSqKm', 'population']]
geonames.columns = ["country", "code", "area", "population"]
geonames = geonames.set_index("code")
geonames
0.8%
Andorra0.4%
249 others98.8%
0.0 - 17100000.0
AND
Andorra
468
ARE
United Arab Emirates
82880
AFG
Afghanistan
647500
ATG
Antigua and Barbuda
443
AIA
Anguilla
102
ALB
Albania
28748
ARM
Armenia
29800
ANT
960
AGO
Angola
1246700
ATA
Antarctica
14000000
252 rows, showing
per page
Page of 26
worldbank_full = pd.read_csv("worldbank_gdp.csv", header=2)
worldbank = worldbank_full[ ["Country Code", "2020"] ]
worldbank.columns = ["code", "gdp"]
worldbank = worldbank.set_index("code")
worldbank
48855550.2030142 - 84746978784172.8
ABW
nan
AFE
920792331528
AFG
20116137326
AFW
784587603323
AGO
58375976293
ALB
14887629268
AND
nan
ARB
2447584445276
ARE
358868765175
ARG
389288056265
266 rows, showing
per page
Page of 27
# https://www.ecdc.europa.eu/en/publications-data/download-todays-data-geographic-distribution-covid-19-cases-worldwide
ecdc_full = pd.read_csv("https://opendata.ecdc.europa.eu/covid19/casedistribution/csv")
ecdc = ecdc_full[ ["cases", "deaths", "countryterritoryCode"] ]
ecdc.columns = ["covid19_cases", "covid19_deaths", "code"]
ecdc = ecdc.set_index("code")
ecdc = ecdc.groupby("code").sum()
ecdc
1 - 16256754
0 - 299177
ABW
5049
46
AFG
49273
1971
AGO
16188
371
AIA
10
0
ALB
48530
1003
AND
7338
79
ARE
184949
617
ARG
1498160
40766
ARM
148682
2503
ATG
148
4
212 rows, showing
per page
Page of 22
multiple = geonames.join(worldbank).join(ecdc)
# from now on, we do not need the country codes anymore; we can remove them:
multiple = multiple.reset_index(drop=True)
multiple
0.8%
Andorra0.4%
249 others98.8%
0.0 - 17100000.0
0
Andorra
468
1
United Arab Emirates
82880
2
Afghanistan
647500
3
Antigua and Barbuda
443
4
Anguilla
102
5
Albania
28748
6
Armenia
29800
7
960
8
Angola
1246700
9
Antarctica
14000000
252 rows, showing
per page
Page of 26
multiple = multiple.dropna().reset_index(drop=True)
multiple
United Arab Emirates0.5%
Afghanistan0.5%
182 others98.9%
1.95 - 17100000.0
0
United Arab Emirates
82880
1
Afghanistan
647500
2
Antigua and Barbuda
443
3
Albania
28748
4
Armenia
29800
184 rows, showing
per page
Page of 37
import requests
import requests_cache # this package creates a local cache, which speeds things up quite a bit
# set up cache for web requests
requests_cache.install_cache('cache')
endpoint = "https://query.wikidata.org/sparql"
# triple quotes start and end multi-line strings
sparql = """
SELECT ?countryLabel ( MAX(?populations) AS ?population ) ?gdp ( MAX(?areas) AS ?area )
WHERE {
?country wdt:P31 wd:Q3624078;
wdt:P463 wd:Q458;
wdt:P1082 ?populations;
wdt:P8744/wdt:P2131 ?gdp;
p:P2046/psn:P2046/wikibase:quantityAmount ?areas.
?gdp_statement ps:P2131 ?gdp.
FILTER NOT EXISTS {
?country p:P2131/pq:P585 ?gdp_date_ .
FILTER (?gdp_date_ > ?gdp_date)
}
SERVICE wikibase:label {
bd:serviceParam wikibase:language "en" .
}
}
GROUP BY ?countryLabel ?area ?population ?gdp
"""
res = requests.get(endpoint, params = {'format': 'json', 'query': sparql})
response = res.text
Unable to deserialize response: 'Attribute' object has no attribute 'alias'
print(response[:1000])
{
"head" : {
"vars" : [ "countryLabel", "population", "gdp", "area" ]
},
"results" : {
"bindings" : [ {
"countryLabel" : {
"xml:lang" : "en",
"type" : "literal",
"value" : "Finland"
},
"area" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "390905420000"
},
"population" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "5608218"
},
"gdp" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "251884887972.766"
}
}, {
"countryLabel" : {
"xml:lang" : "en",
"type" : "literal",
"value" : "Croatia"
},
"area" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "56594000000"
},
"population" : {
import json
# let's define the parsing steps as a function, so that you can reuse it later
def wikidata_to_dataframe(data):
# parse json data from response and get results:
results = json.loads(data)["results"]["bindings"]
# column names we draw from the first result
cols = [ val for val in results[0] ]
rows = []
# to get the values from this, we need to loop through the results:
for result in results:
values = [ result[val]["value"] for val in result ]
rows.append(values)
# with rows and cols we can create a DataFrame:
return pd.DataFrame(rows, columns=cols)
wikidata = wikidata_to_dataframe(response)
wikidata = wikidata.rename(columns={'countryLabel':'country'})
wikidata
Finland3.8%
Croatia3.8%
24 others92.3%
3909054200003.8%
565940000003.8%
24 others92.3%
0
Finland
390905420000
1
Croatia
56594000000
2
Romania
238397000000
3
Sweden
528861060000
4
Malta
316000000
5
Republic of Ireland
69797000000
6
Estonia
45339000000
7
Austria
83878990000
8
Czech Republic
78866000000
9
Hungary
93036000000
26 rows, showing
per page
Page of 3
wikidata.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 26 entries, 0 to 25
Data columns (total 4 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 country 26 non-null object
1 area 26 non-null object
2 population 26 non-null object
3 gdp 26 non-null object
dtypes: object(4)
memory usage: 960.0+ bytes
cols = wikidata.columns[1:]
wikidata[cols] = wikidata[cols].apply(pd.to_numeric)
wikidata.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 26 entries, 0 to 25
Data columns (total 4 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 country 26 non-null object
1 area 26 non-null int64
2 population 26 non-null int64
3 gdp 26 non-null float64
dtypes: float64(1), int64(2), object(1)
memory usage: 960.0+ bytes
sparql = """
# 1. enter your query here
"""
# 2. uncomment the following lines
# res = requests.get(endpoint, params = {'format': 'json', 'query': sparql})
# your_wikidata = wikidata_to_dataframe(res.text)
# your_wikidata
oecd[oecd.columns[1:]].corr()
-0.743453171979418 - 1.0
-0.4898394590492887 - 1.0
dwellings without basic facilities
1
-0.489839459
housing expenditure
-0.489839459
1
rooms per person
-0.5670415212
0.1081159094
household net adjusted disposable income
-0.5933208225
0.03818512651
household net financial wealth
-0.3882053112
0.07593112825
employment rate
-0.2873319494
-0.0871326234
job security
-0.1233920124
0.1731886451
long-term unemployment rate
-0.1660196279
0.2664336877
personal earnings
-0.6290899838
0.03175170034
quality of support network
-0.4244156075
0.06419716074
24 rows, showing
per page
Page of 3
# import a few components from the machine learning library scikit-learn:
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
# create a data pipeline, which includes scaling to normalize the data
# and by initializing the PCA with the number of principal components: 2
pipe = Pipeline([('scaling', StandardScaler()), ('pca', PCA(n_components=2))])
# run the PCA on all columns (except the first which is the country name)
principal_components = pipe.fit_transform(oecd[oecd.columns[1:]])
# let's have a look
principal_components
# first turn principal_components into a DataFrame
pca_positions = pd.DataFrame(principal_components, columns = ['x', 'y'])
# … and combine it with the source dataframe oecd
oecd_pca = pd.concat([oecd, pca_positions], axis = 1)
alt.Chart(oecd_pca).mark_circle().encode(
x=alt.X('x', axis=alt.Axis(labels=False)),
y=alt.Y('y', axis=alt.Axis(labels=False)),
tooltip='country'
).properties(width=400, height=400)
# 1. run PCA on the numeric columns
# 2. combine the positions with original data
# 3. display a scatterplot
# import the umap library
import umap.umap_ as umap
# there are a bunch of warnings generated by the umap package that we will omit
warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*")
# there are two main parameters, which you need to tweak
reducer = umap.UMAP(
n_neighbors=15, # balances local versus global structure
min_dist=.4 # .01 for tight clumps, large for loose 1
)
# the datasets have a similar structure; you can replace it with multiple or wikidata:
data = oecd
# we use again the StandardScaler for normalization
# and add the UMAP reducer afterwards into the pipeline
pipe = Pipeline([('scaling', StandardScaler()), ('umap', reducer)])
# start the normalization and reduction steps
embedding = pipe.fit_transform(data[data.columns[1:]])
# turn the resulting embedding into dataframe, in which the positions are x and y
umap_positions = pd.DataFrame(embedding, columns=["x", "y"])
# … and merge it with the original dataset
data_umap = pd.concat([data, umap_positions], axis = 1)
# to center the scatterplot around generated positions,
# we adjust the scales according to the smallest and largest x and y values:
x_domain = [data_umap["x"].min(), data_umap["x"].max() ]
y_domain = [data_umap["y"].min(), data_umap["y"].max() ]
# display scatterplot and pass domains for x and y to scale parameter
alt.Chart(data_umap).mark_circle().encode(
alt.X('x', scale=alt.Scale(domain=x_domain), axis=alt.Axis(labels=False)),
alt.Y('y', scale=alt.Scale(domain=y_domain), axis=alt.Axis(labels=False)),
tooltip="country"
).properties(width=400, height=400)
data = wikidata
cols = ["area", "population", "gdp"]
alt.Chart(data).mark_circle().encode(
# the data dimensions used for the encoding are specified below under repeat
# they are all quantitative and we remove the axis labels to avoid clutter
alt.X(alt.repeat("column"), type='quantitative', axis=alt.Axis(labels=False)),
alt.Y(alt.repeat("row"), type='quantitative', axis=alt.Axis(labels=False)),
# we add a tooltip with the country's name
tooltip = "country"
).properties( width=150, height=150).repeat(
# specify which data columns are used
column=cols,
row=cols
)
corr = oecd[oecd.columns[1:]].corr()
# first we reset the index and call it dim1
corr = corr.reset_index().rename(columns={'index': 'dim1'})
# turn correlation data into long form
corr = pd.melt(corr, id_vars="dim1", var_name='dim2', value_name='corr')
# add a label column for rounded correlation values
corr['label'] = corr['corr'].map('{:.1f}'.format)
corr
dwellings without basic facilities4.2%
housing expenditure4.2%
22 others91.7%
dwellings without basic facilities4.2%
housing expenditure4.2%
22 others91.7%
0
dwellings without basic facilities
dwellings without basic facilities
1
housing expenditure
dwellings without basic facilities
2
rooms per person
dwellings without basic facilities
3
household net adjusted disposable income
dwellings without basic facilities
4
household net financial wealth
dwellings without basic facilities
5
employment rate
dwellings without basic facilities
6
job security
dwellings without basic facilities
7
long-term unemployment rate
dwellings without basic facilities
8
personal earnings
dwellings without basic facilities
9
quality of support network
dwellings without basic facilities
576 rows, showing
per page
Page of 58
# we create a layered chart, with the base taking in the correlation data corr
# and the basic layout based on the dimensions
base = alt.Chart(corr).encode(
x='dim1:O',
y='dim2:O'
).properties(width=500, height=500)
# a textual layer displaying rounded correlation values
text = base.mark_text().encode( text='label' )
# heatmap of the correlation values
plot = base.mark_rect().encode(
color='corr:Q'
)
# both layers are combined
plot + text
# to coordinate hover highlights we create a selection
selection = alt.selection_point(on='mouseover', fields=['country'])
# the definitions of the base are used by the three sub-charts
base = alt.Chart(wikidata).mark_bar().encode(
# adjust opacity based on hover selection
opacity=alt.condition(selection, alt.value(1), alt.value(.5)),
x = alt.X("country:O").sort("-y").axis(None),
tooltip=['country','population', 'area', 'gdp']
).properties(
width=600, height=150
).add_params(selection)
# create a chart for each dimension
pop = base.encode(y = "population")
area = base.encode(y = "area")
gdp = base.encode(y = "gdp")
# combine them with ampersands
pop & area & gdp