import pandas as pd
import altair as alt
url = "https://gist.github.com/scotthmurray/f71065a5694f22259bf9/raw/ce891b9fe7ec3c5cab3308f4cd0c8eeccc36f6c7/Better%2520Life%2520Index%2520Data.csv"
oecd = pd.read_csv(url)
# to keep things consistent with the other examples we turn column names to lowercase
oecd.columns = map(str.lower, oecd.columns)
geonames_full = pd.read_csv("https://www.geonames.org/countryInfoCSV", sep='\t', keep_default_na=False)
geonames_full.head()
iso alpha2object
iso alpha3object
0
AD
AND
1
AE
ARE
2
AF
AFG
3
AG
ATG
4
AI
AIA
geonames = geonames_full[['name', 'iso alpha3', 'areaInSqKm', 'population']]
geonames.columns = ["country", "code", "area", "population"]
geonames = geonames.set_index("code")
geonames
countryobject
0.8%
Andorra0.4%
249 others98.8%
areafloat64
0.0 - 17100000.0
AND
Andorra
468
ARE
United Arab Emirates
82880
AFG
Afghanistan
647500
ATG
Antigua and Barbuda
443
AIA
Anguilla
102
ALB
Albania
28748
ARM
Armenia
29800
ANT
960
AGO
Angola
1246700
ATA
Antarctica
14000000
worldbank_full = pd.read_csv("worldbank_gdp.csv", header=2)
worldbank = worldbank_full[ ["Country Code", "2020"] ]
worldbank.columns = ["code", "gdp"]
worldbank = worldbank.set_index("code")
worldbank
gdpfloat64
48855550.2030142 - 84746978784172.8
ABW
nan
AFE
920792331528
AFG
20116137326
AFW
784587603323
AGO
58375976293
ALB
14887629268
AND
nan
ARB
2447584445276
ARE
358868765175
ARG
389288056265
# https://www.ecdc.europa.eu/en/publications-data/download-todays-data-geographic-distribution-covid-19-cases-worldwide
ecdc_full = pd.read_csv("https://opendata.ecdc.europa.eu/covid19/casedistribution/csv")
ecdc = ecdc_full[ ["cases", "deaths", "countryterritoryCode"] ]
ecdc.columns = ["covid19_cases", "covid19_deaths", "code"]
ecdc = ecdc.set_index("code")
ecdc = ecdc.groupby("code").sum()
ecdc
covid19_casesint64
1 - 16256754
covid19_deathsint64
0 - 299177
ABW
5049
46
AFG
49273
1971
AGO
16188
371
AIA
10
0
ALB
48530
1003
AND
7338
79
ARE
184949
617
ARG
1498160
40766
ARM
148682
2503
ATG
148
4
multiple = geonames.join(worldbank).join(ecdc)
# from now on, we do not need the country codes anymore; we can remove them:
multiple = multiple.reset_index(drop=True)
multiple
countryobject
0.8%
Andorra0.4%
249 others98.8%
areafloat64
0.0 - 17100000.0
0
Andorra
468
1
United Arab Emirates
82880
2
Afghanistan
647500
3
Antigua and Barbuda
443
4
Anguilla
102
5
Albania
28748
6
Armenia
29800
7
960
8
Angola
1246700
9
Antarctica
14000000
multiple = multiple.dropna().reset_index(drop=True)
multiple
countryobject
United Arab Emirates0.5%
Afghanistan0.5%
182 others98.9%
areafloat64
1.95 - 17100000.0
0
United Arab Emirates
82880
1
Afghanistan
647500
2
Antigua and Barbuda
443
3
Albania
28748
4
Armenia
29800
import requests
import requests_cache # this package creates a local cache, which speeds things up quite a bit
# set up cache for web requests
requests_cache.install_cache('cache')
endpoint = "https://query.wikidata.org/sparql"
# triple quotes start and end multi-line strings
sparql = """
SELECT ?countryLabel ?population ?gdp ( MAX(?areas) AS ?area )
WHERE {
?country wdt:P31 wd:Q3624078;
wdt:P463 wd:Q458;
wdt:P1082 ?population;
wdt:P8744/wdt:P2131 ?gdp;
p:P2046/psn:P2046/wikibase:quantityAmount ?areas.
?gdp_statement ps:P2131 ?gdp;
pq:P585 ?gdp_date.
FILTER NOT EXISTS {
?country p:P2131/pq:P585 ?gdp_date_ .
FILTER (?gdp_date_ > ?gdp_date)
}
SERVICE wikibase:label {
bd:serviceParam wikibase:language "en" .
}
}
GROUP BY ?countryLabel ?area ?population ?gdp
"""
res = requests.get(endpoint, params = {'format': 'json', 'query': sparql})
response = res.text
print(response[:1000])
{
"head" : {
"vars" : [ "countryLabel", "population", "gdp", "area" ]
},
"results" : {
"bindings" : [ {
"countryLabel" : {
"xml:lang" : "en",
"type" : "literal",
"value" : "Finland"
},
"area" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "390905420000"
},
"population" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "5516224"
},
"gdp" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "251884887972.766"
}
}, {
"countryLabel" : {
"xml:lang" : "en",
"type" : "literal",
"value" : "Croatia"
},
"area" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#decimal",
"type" : "literal",
"value" : "56594000000"
},
"population" : {
import json
# let's define the parsing steps as a function, so that you can reuse it later
def wikidata_to_dataframe(data):
# parse json data from response and get results:
results = json.loads(data)["results"]["bindings"]
# column names we draw from the first result
cols = [ val for val in results[0] ]
rows = []
# to get the values from this, we need to loop through the results:
for result in results:
values = [ result[val]["value"] for val in result ]
rows.append(values)
# with rows and cols we can create a DataFrame:
return pd.DataFrame(rows, columns=cols)
wikidata = wikidata_to_dataframe(response)
wikidata = wikidata.rename(columns={'countryLabel':'country'})
wikidata
countryobject
Finland4.3%
Croatia4.3%
21 others91.3%
areaobject
3909054200004.3%
565940000004.3%
21 others91.3%
0
Finland
390905420000
1
Croatia
56594000000
2
Romania
238397000000
3
Sweden
528861060000
4
Malta
316000000
5
Republic of Ireland
69797000000
6
Latvia
64593790000
7
Estonia
45339000000
8
Austria
83878990000
9
Germany
357587770000
wikidata.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 23 entries, 0 to 22
Data columns (total 4 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 country 23 non-null object
1 area 23 non-null object
2 population 23 non-null object
3 gdp 23 non-null object
dtypes: object(4)
memory usage: 864.0+ bytes
cols = wikidata.columns[1:]
wikidata[cols] = wikidata[cols].apply(pd.to_numeric)
wikidata.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 23 entries, 0 to 22
Data columns (total 4 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 country 23 non-null object
1 area 23 non-null int64
2 population 23 non-null int64
3 gdp 23 non-null float64
dtypes: float64(1), int64(2), object(1)
memory usage: 864.0+ bytes
sparql = """
# 1. enter your query here
"""
# 2. uncomment the following lines
# res = requests.get(endpoint, params = {'format': 'json', 'query': sparql})
# your_wikidata = wikidata_to_dataframe(res.text)
# your_wikidata
oecd.corr()
dwellings without basic facilitiesfloat64
-0.743453171979418 - 1.0
housing expenditurefloat64
-0.4898394590492887 - 1.0
dwellings without basic facilities
1
-0.489839459
housing expenditure
-0.489839459
1
rooms per person
-0.5670415212
0.1081159094
household net adjusted disposable income
-0.5933208225
0.03818512651
household net financial wealth
-0.3882053112
0.07593112825
employment rate
-0.2873319494
-0.0871326234
job security
-0.1233920124
0.1731886451
long-term unemployment rate
-0.1660196279
0.2664336877
personal earnings
-0.6290899838
0.03175170034
quality of support network
-0.4244156075
0.06419716074
# import a few components from the machine learning library scikit-learn:
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
# create a data pipeline, which includes scaling to normalize the data
# and by initializing the PCA with the number of principal components: 2
pipe = Pipeline([('scaling', StandardScaler()), ('pca', PCA(n_components=2))])
# run the PCA on all columns (except the first which is the country name)
principal_components = pipe.fit_transform(oecd[oecd.columns[1:]])
# let's have a look
principal_components
# first turn principal_components into a DataFrame
pca_positions = pd.DataFrame(principal_components, columns = ['x', 'y'])
# … and combine it with the source dataframe oecd
oecd_pca = pd.concat([oecd, pca_positions], axis = 1)
alt.Chart(oecd_pca).mark_circle().encode(
x=alt.X('x', axis=alt.Axis(labels=False)),
y=alt.Y('y', axis=alt.Axis(labels=False)),
tooltip='country'
).properties(width=400, height=400)
# 1. run PCA on the numeric columns
# 2. combine the positions with original data
# 3. display a scatterplot
# import the umap library
import umap.umap_ as umap
# there are a bunch of warnings generated by the umap package that we will omit
warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*")
# there are two main parameters, which you need to tweak
reducer = umap.UMAP(
n_neighbors=15, # balances local versus global structure
min_dist=.4 # .01 for tight clumps, large for loose 1
)
# the datasets have a similar structure; you can replace it with multiple or wikidata:
data = oecd
# we use again the StandardScaler for normalization
# and add the UMAP reducer afterwards into the pipeline
pipe = Pipeline([('scaling', StandardScaler()), ('umap', reducer)])
# start the normalization and reduction steps
embedding = pipe.fit_transform(data[data.columns[1:]])
# turn the resulting embedding into dataframe, in which the positions are x and y
umap_positions = pd.DataFrame(embedding, columns=["x", "y"])
# … and merge it with the original dataset
data_umap = pd.concat([data, umap_positions], axis = 1)
# to center the scatterplot around generated positions,
# we adjust the scales according to the smallest and largest x and y values:
x_domain = [data_umap["x"].min(), data_umap["x"].max() ]
y_domain = [data_umap["y"].min(), data_umap["y"].max() ]
/root/venv/lib/python3.9/site-packages/umap/distances.py:1063: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
@numba.jit()
/root/venv/lib/python3.9/site-packages/umap/distances.py:1071: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
@numba.jit()
/root/venv/lib/python3.9/site-packages/umap/distances.py:1086: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
@numba.jit()
/shared-libs/python3.9/py/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
from .autonotebook import tqdm as notebook_tqdm
/root/venv/lib/python3.9/site-packages/umap/umap_.py:660: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
@numba.jit()
2023-06-01 18:25:31.087165: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-01 18:25:31.397101: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
2023-06-01 18:25:31.397128: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
2023-06-01 18:25:31.440830: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2023-06-01 18:25:33.853385: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory
2023-06-01 18:25:33.853506: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory
2023-06-01 18:25:33.853521: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
# display scatterplot and pass domains for x and y to scale parameter
alt.Chart(data_umap).mark_circle().encode(
alt.X('x', scale=alt.Scale(domain=x_domain), axis=alt.Axis(labels=False)),
alt.Y('y', scale=alt.Scale(domain=y_domain), axis=alt.Axis(labels=False)),
tooltip="country"
).properties(width=400, height=400)
data = wikidata
cols = ["area", "population", "gdp"]
alt.Chart(data).mark_circle().encode(
# the data dimensions used for the encoding are specified below under repeat
# they are all quantitative and we remove the axis labels to avoid clutter
alt.X(alt.repeat("column"), type='quantitative', axis=alt.Axis(labels=False)),
alt.Y(alt.repeat("row"), type='quantitative', axis=alt.Axis(labels=False)),
# we add a tooltip with the country's name
tooltip = "country"
).properties( width=150, height=150).repeat(
# specify which data columns are used
column=cols,
row=cols
)
corr = oecd.corr()
# first we reset the index and call it dim1
corr = corr.reset_index().rename(columns={'index': 'dim1'})
# turn correlation data into long form
corr = pd.melt(corr, id_vars="dim1", var_name='dim2', value_name='corr')
# add a label column for rounded correlation values
corr['label'] = corr['corr'].map('{:.1f}'.format)
corr
dim1object
dwellings without basic facilities4.2%
housing expenditure4.2%
22 others91.7%
dim2object
dwellings without basic facilities4.2%
housing expenditure4.2%
22 others91.7%
0
dwellings without basic facilities
dwellings without basic facilities
1
housing expenditure
dwellings without basic facilities
2
rooms per person
dwellings without basic facilities
3
household net adjusted disposable income
dwellings without basic facilities
4
household net financial wealth
dwellings without basic facilities
5
employment rate
dwellings without basic facilities
6
job security
dwellings without basic facilities
7
long-term unemployment rate
dwellings without basic facilities
8
personal earnings
dwellings without basic facilities
9
quality of support network
dwellings without basic facilities
# we create a layered chart, with the base taking in the correlation data corr
# and the basic layout based on the dimensions
base = alt.Chart(corr).encode(
x='dim1:O',
y='dim2:O'
).properties(width=500, height=500)
# a textual layer displaying rounded correlation values
text = base.mark_text().encode( text='label' )
# heatmap of the correlation values
plot = base.mark_rect().encode(
color='corr:Q'
)
# both layers are combined
plot + text
# to coordinate hover highlights we create a selection
selection = alt.selection_point(on='mouseover', fields=['country'])
# the definitions of the base are used by the three sub-charts
base = alt.Chart(wikidata).mark_bar().encode(
# adjust opacity based on hover selection
opacity=alt.condition(selection, alt.value(1), alt.value(.5)),
x = alt.X("country:O").sort("-y").axis(None),
tooltip=['country','population', 'area', 'gdp']
).properties(
width=600, height=150
).add_params(selection)
# create a chart for each dimension
pop = base.encode(y = "population")
area = base.encode(y = "area")
gdp = base.encode(y = "gdp")
# combine them with ampersands
pop & area & gdp