import janitor
import matplotlib.pyplot as plt
import missingno
import numpy as np
import pandas as pd
import pyreadr
import seaborn as sns
import session_info
import upsetplot
%run pandas-missing-extension.ipynb
%matplotlib inline
sns.set(
rc={
"figure.figsize": (10, 10)
}
)
sns.set_style("whitegrid")
print(
None or True,
None or False,
None == None,
None is None,
# None + True,
# None / False,
type(None),
sep="\n"
)
True
False
True
True
<class 'NoneType'>
print(
np.nan or True,
np.nan or False,
np.nan == np.nan,
np.nan is np.nan,
np.nan / 2,
np.nan * 7,
type(np.nan),
np.isnan(np.nan),
sep="\n"
)
nan
nan
False
True
nan
nan
<class 'float'>
True
test_missing_df = pd.DataFrame.from_dict(
data=dict(
x=[0, 1, np.nan, np.nan, None],
y=[0, 1, pd.NA, np.nan, None]
)
)
test_missing_df
xfloat64
yobject
0
0
0
1
1
1
2
nan
<NA>
3
nan
nan
4
nan
None
test_missing_df.isna()
test_missing_df.isnull()
xbool
ybool
0
false
false
1
false
false
2
true
true
3
true
true
4
true
true
test_missing_df.x.isnull()
pd.Series([1, np.nan])
pd.Series([pd.to_datetime("2022-01-01"), np.nan])
pd.Series([-1]).isnull()
pima_indians_diabetes_url = "https://nrvis.com/data/mldata/pima-indians-diabetes.csv"
!wget -O ./data/pima-indians-diabetes.csv { pima_indians_diabetes_url } -q
diabetes_df = pd.read_csv(
filepath_or_buffer="./data/pima-indians-diabetes.csv", # or pima_indians_diabetes_url
sep=",",
names=[
"pregnancies",
"glucose",
"blood_pressure",
"skin_thickness",
"insulin",
"bmi",
"diabetes_pedigree_function",
"age",
"outcome",
]
)
base_url = "https://github.com/njtierney/naniar/raw/master/data/"
datasets_names = ("oceanbuoys", "pedestrian", "riskfactors")
extension = ".rda"
datasets_dfs = {}
for dataset_name in datasets_names:
dataset_file = f"{ dataset_name }{ extension }"
dataset_output_file = f"./data/{ dataset_file }"
dataset_url = f"{ base_url }{ dataset_file }"
!wget -q -O { dataset_output_file } { dataset_url }
datasets_dfs[f"{ dataset_name }_df"] = pyreadr.read_r(dataset_output_file).get(dataset_name)
datasets_dfs.keys()
locals().update(**datasets_dfs)
del datasets_dfs
oceanbuoys_df.shape, pedestrian_df.shape, riskfactors_df.shape, diabetes_df.shape
riskfactors_df.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 245 entries, 0 to 244
Data columns (total 34 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 state 245 non-null category
1 sex 245 non-null category
2 age 245 non-null int32
3 weight_lbs 235 non-null object
4 height_inch 243 non-null object
5 bmi 234 non-null float64
6 marital 244 non-null category
7 pregnant 30 non-null category
8 children 245 non-null int32
9 education 244 non-null category
10 employment 245 non-null category
11 income 245 non-null category
12 veteran 242 non-null category
13 hispanic 243 non-null category
14 health_general 245 non-null category
15 health_physical 245 non-null int32
16 health_mental 245 non-null int32
17 health_poor 132 non-null object
18 health_cover 245 non-null category
19 provide_care 242 non-null category
20 activity_limited 242 non-null category
21 drink_any 243 non-null category
22 drink_days 111 non-null object
23 drink_average 110 non-null object
24 smoke_100 243 non-null category
25 smoke_days 117 non-null category
26 smoke_stop 33 non-null category
27 smoke_last 84 non-null category
28 diet_fruit 237 non-null object
29 diet_salad 237 non-null object
30 diet_potato 237 non-null object
31 diet_carrot 237 non-null object
32 diet_vegetable 237 non-null object
33 diet_juice 237 non-null object
dtypes: category(18), float64(1), int32(4), object(11)
memory usage: 36.7+ KB
riskfactors_df.isna()
statebool
False100%
sexbool
False100%
0
false
false
1
false
false
2
false
false
3
false
false
4
false
false
5
false
false
6
false
false
7
false
false
8
false
false
9
false
false
riskfactors_df.shape
riskfactors_df.missing.number_complete()
riskfactors_df.missing.number_missing()
riskfactors_df.missing.missing_variable_summary()
variableobject
state2.9%
sex2.9%
32 others94.1%
n_missingint64
0 - 215
0
state
0
1
sex
0
2
age
0
3
weight_lbs
10
4
height_inch
2
5
bmi
11
6
marital
1
7
pregnant
215
8
children
0
9
education
1
riskfactors_df.missing.missing_variable_table()
n_missing_in_variableint64
0 - 215
n_variablesint64
1 - 10
0
0
10
1
8
6
2
2
4
3
3
3
4
1
2
5
10
1
6
11
1
7
113
1
8
128
1
9
134
1
riskfactors_df.missing.missing_case_summary()
caseint64
0 - 244
n_missingint64
1 - 15
0
0
6
1
1
6
2
2
7
3
3
12
4
4
5
5
5
7
6
6
5
7
7
4
8
8
4
9
9
11
riskfactors_df.missing.missing_case_table()
n_missing_in_caseint64
1 - 15
n_casesint64
1 - 49
0
4
49
1
5
45
2
7
39
3
6
36
4
2
31
5
3
30
6
1
4
7
8
3
8
12
3
9
15
2
(
riskfactors_df
.missing
.missing_variable_span(
variable="weight_lbs",
span_every=50
)
)
span_counterint64
n_missingint64
0
0
1
1
1
5
2
2
1
3
3
1
4
4
2
(
riskfactors_df
.missing
.missing_variable_run(
variable="weight_lbs"
)
)
run_lengthint64
1 - 53
is_naobject
complete52.6%
missing47.4%
0
14
complete
1
1
missing
2
45
complete
3
1
missing
4
5
complete
5
1
missing
6
12
complete
7
1
missing
8
10
complete
9
2
missing
riskfactors_df.missing.missing_variable_plot()
riskfactors_df.missing.missing_case_plot()
(
riskfactors_df
.missing
.missing_variable_span_plot(
variable="weight_lbs",
span_every=10,
rot=0
)
)
missingno.bar(df = riskfactors_df)
missingno.matrix(df=riskfactors_df)
(
riskfactors_df
.missing
.missing_upsetplot(
variables = None,
element_size = 60
)
)
common_na_strings = (
"missing",
"NA",
"N A",
"N/A",
"#N/A",
"NA ",
" NA",
"N /A",
"N / A",
" N / A",
"N / A ",
"na",
"n a",
"n/a",
"na ",
" na",
"n /a",
"n / a",
" a / a",
"n / a ",
"NULL",
"null",
"",
"?",
"*",
".",
)
common_na_numbers = (-9, -99, -999, -9999, 9999, 66, 77, 88, -1)
missing_data_example_df = pd.DataFrame.from_dict(
dict(
x = [1, 3, "NA", -99, -98, -99],
y = ["A", "N/A", "NA", "E", "F", "G"],
z = [-100, -99, -98, -101, -1, -1]
)
)
missing_data_example_df
xobject
yobject
0
1
A
1
3
N/A
2
NA
NA
3
-99
E
4
-98
F
5
-99
G
missing_data_example_df.missing.number_missing()
missing_data_example_df.dtypes
missing_data_example_df.x.unique()
(
missing_data_example_df
.select_dtypes(object)
.apply(pd.unique)
)
pd.read_csv(
"./data/missing_data_enconding_example.csv",
na_filter=True,
na_values=[-99, -1]
)
xfloat64
yobject
0
1
A
1
3
nan
2
nan
nan
3
nan
E
4
-98
F
5
nan
G
(
missing_data_example_df
.replace(
to_replace=[-99, "NA"],
value=np.nan
)
)
xfloat64
yobject
0
1
A
1
3
N/A
2
nan
nan
3
nan
E
4
-98
F
5
nan
G
(
missing_data_example_df
.replace(
to_replace={
"x": {
-99: np.nan
}
}
)
)
xobject
yobject
0
1
A
1
3
N/A
2
NA
NA
3
nan
E
4
-98
F
5
nan
G
implicit_to_explicit_df = pd.DataFrame.from_dict(
data={
"name": ["lynn", "lynn", "lynn", "zelda"],
"time": ["morning", "afternoon", "night", "morning"],
"value": [350, 310, np.nan, 320]
}
)
implicit_to_explicit_df
nameobject
timeobject
0
lynn
morning
1
lynn
afternoon
2
lynn
night
3
zelda
morning
(
implicit_to_explicit_df
.pivot_wider(
index="name",
names_from="time",
values_from="value"
)
)
nameobject
afternoonfloat64
0
lynn
310
1
zelda
nan
(
implicit_to_explicit_df
.value_counts(
subset=["name"]
)
.reset_index(name="n")
.query("n < 2")
)
nameobject
nint64
1
zelda
1
(
implicit_to_explicit_df
.complete(
"name",
"time",
)
)
nameobject
timeobject
0
lynn
morning
1
lynn
afternoon
2
lynn
night
3
zelda
morning
4
zelda
afternoon
5
zelda
night
(
implicit_to_explicit_df
# pyjanitor
.complete(
{"name": ["lynn", "zelda"]},
{"time": ["morning", "afternoon"]},
sort=True
)
)
nameobject
timeobject
0
lynn
afternoon
1
lynn
morning
2
zelda
afternoon
3
zelda
morning
4
lynn
night
(
implicit_to_explicit_df
# pyjanitor
.complete(
"name",
"time",
fill_value=np.nan
)
)
nameobject
timeobject
0
lynn
morning
1
lynn
afternoon
2
lynn
night
3
zelda
morning
4
zelda
afternoon
5
zelda
night
(
implicit_to_explicit_df
# pyjanitor
.complete(
"name",
"time",
fill_value=0,
explicit=False
)
)
nameobject
timeobject
0
lynn
morning
1
lynn
afternoon
2
lynn
night
3
zelda
morning
4
zelda
afternoon
5
zelda
night
diabetes_df.missing.missing_variable_plot()
diabetes_df[diabetes_df.columns[1:6]] = diabetes_df[diabetes_df.columns[1:6]].replace(0, np.nan)
diabetes_df.missing.missing_variable_plot()
(
diabetes_df
.missing.sort_variables_by_missingness()
.pipe(missingno.matrix)
)
(
diabetes_df
.missing.sort_variables_by_missingness()
.sort_values(by = "blood_pressure")
.pipe(missingno.matrix)
)
(
diabetes_df
.missing.sort_variables_by_missingness()
.sort_values("insulin")
.pipe(missingno.matrix)
)
(
riskfactors_df
.isna()
.replace({
False: "Not missing",
True: "Missing"
})
.add_suffix("_NA")
.pipe(
lambda shadow_matrix: pd.concat(
[riskfactors_df, shadow_matrix],
axis="columns"
)
)
)
statecategory
64.9%
344.9%
50 others90.2%
sexcategory
Female62.4%
Male37.6%
0
26
Female
1
40
Female
2
72
Female
3
42
Male
4
32
Female
5
19
Male
6
45
Male
7
56
Female
8
18
Male
9
8
Female
(
riskfactors_df
.missing
.bind_shadow_matrix(only_missing = True)
)
statecategory
64.9%
344.9%
50 others90.2%
sexcategory
Female62.4%
Male37.6%
0
26
Female
1
40
Female
2
72
Female
3
42
Male
4
32
Female
5
19
Male
6
45
Male
7
56
Female
8
18
Male
9
8
Female
(
riskfactors_df
.missing
.bind_shadow_matrix(only_missing=True)
.groupby(["weight_lbs_NA"])
["age"]
.describe()
.reset_index()
)
weight_lbs_NAobject
countfloat64
0
Missing
10
1
Not Missing
235
(
riskfactors_df
.missing
.bind_shadow_matrix(only_missing=True)
.pipe(
lambda df: (
sns.displot(
data=df,
x="age",
hue="weight_lbs_NA",
kind="kde"
)
)
)
)
(
riskfactors_df
.missing
.bind_shadow_matrix(only_missing=True)
.pipe(
lambda df: (
sns.boxenplot(
data=df,
x="weight_lbs_NA",
y="age",
)
)
)
)
(
riskfactors_df
.missing
.bind_shadow_matrix(only_missing=True)
.pipe(
lambda df: (
sns.displot(
data=df,
x="age",
col="weight_lbs_NA",
facet_kws={
"sharey": False
}
)
)
)
)
(
riskfactors_df
.missing
.bind_shadow_matrix(only_missing=True)
.pipe(
lambda df: (
sns.displot(
data=df,
x="age",
col="marital_NA",
row="weight_lbs_NA"
)
)
)
)
def column_fill_with_dummies(
column: pd.Series,
proportion_below: float=0.10,
jitter: float=0.075,
seed: int=42
) -> pd.Series:
column = column.copy(deep=True)
# Extract values metadata.
missing_mask = column.isna()
number_missing_values = missing_mask.sum()
column_range = column.max() - column.min()
# Shift data
column_shift = column.min() - column.min() * proportion_below
# Create the "jitter" (noise) to be added around the points.
np.random.seed(seed)
column_jitter = (np.random.rand(number_missing_values) - 2) * column_range * jitter
# Save new dummy data.
column[missing_mask] = column_shift + column_jitter
return column
plt.figure(figsize=(10, 10))
(
riskfactors_df
.select_dtypes(
exclude="category"
)
.pipe(
lambda df: df[df.columns[df.isna().any()]]
)
.missing.bind_shadow_matrix(true_string=True, false_string=False)
.apply(
lambda column: column if "_NA" in column.name else column_fill_with_dummies(column, proportion_below=0.05, jitter=0.075)
)
.assign(
nullity=lambda df: df.weight_lbs_NA | df.height_inch_NA
)
.pipe(
lambda df: (
sns.scatterplot(
data=df,
x="weight_lbs",
y="height_inch",
hue="nullity"
)
)
)
)
missingno.heatmap(
df=riskfactors_df
)
missingno.dendrogram(
df=riskfactors_df
)
riskfactors_df.shape
(
riskfactors_df
.weight_lbs
.mean()
)
riskfactors_df.weight_lbs.size, riskfactors_df.weight_lbs.count()
riskfactors_df.weight_lbs.mean(skipna=False)
(
riskfactors_df
.dropna(
subset=["weight_lbs"],
how="any"
)
.shape
)
(
riskfactors_df
.dropna(
subset=["weight_lbs", "height_inch"],
how="any"
)
.shape
)
(
riskfactors_df
.dropna(
subset=["weight_lbs", "height_inch"],
how="all"
)
.shape
)
(
riskfactors_df
.dropna(
subset=["weight_lbs", "height_inch"],
how="any"
)
.select_columns(["weight_lbs", "height_inch"])
.pipe(
lambda df: missingno.matrix(df)
)
)
(
riskfactors_df
.dropna(
subset=["weight_lbs", "height_inch"],
how="all"
)
.select_columns(["weight_lbs", "height_inch"])
.pipe(
lambda df: missingno.matrix(df)
)
)
implicit_to_explicit_df = pd.DataFrame(
data={
"name": ["lynn", np.nan, "zelda", np.nan, "shadowsong", np.nan],
"time": ["morning", "afternoon", "morning", "afternoon", "morning", "afternoon",],
"value": [350, 310, 320, 350, 310, 320]
}
)
implicit_to_explicit_df
nameobject
timeobject
0
lynn
morning
1
nan
afternoon
2
zelda
morning
3
nan
afternoon
4
shadowsong
morning
5
nan
afternoon
implicit_to_explicit_df.ffill()
nameobject
timeobject
0
lynn
morning
1
lynn
afternoon
2
zelda
morning
3
zelda
afternoon
4
shadowsong
morning
5
shadowsong
afternoon
plt.figure(figsize=(10, 10))
(
riskfactors_df
.select_columns("weight_lbs", "height_inch", "bmi")
.missing.bind_shadow_matrix(true_string=True, false_string=False)
.apply(
axis="rows",
func=lambda column: column.fillna(column.mean()) if "_NA" not in column.name else column
)
.pipe(
lambda df: (
sns.displot(
data=df,
x="weight_lbs",
hue="weight_lbs_NA"
)
)
)
)
plt.figure(figsize=(10, 10))
(
riskfactors_df
.select_columns("weight_lbs", "height_inch", "bmi")
.missing.bind_shadow_matrix(true_string=True, false_string=False)
.apply(
axis="rows",
func=lambda column: column.fillna(column.mean()) if "_NA" not in column.name else column
)
.assign(
imputed=lambda df: df.weight_lbs_NA | df.height_inch_NA
)
.pipe(
lambda df: (
sns.scatterplot(
data=df,
x="weight_lbs",
y="height_inch",
hue="imputed"
)
)
)
)
plt.figure(figsize=(10, 10))
(
riskfactors_df
.select_columns("weight_lbs", "height_inch", "bmi")
.missing.bind_shadow_matrix(true_string=True, false_string=False)
.apply(
axis="rows",
func=lambda column: column.fillna(column.mean())
if "_NA" not in column.name
else column,
)
.pivot_longer(
index="*_NA"
)
.pivot_longer(
index=["variable", 'value'],
names_to="variable_NA",
values_to="value_NA"
)
.assign(
valid=lambda df: df.apply(axis="columns", func=lambda column: column.variable in column.variable_NA)
)
.query("valid")
.pipe(
lambda df: (
sns.displot(
data=df,
x="value",
hue="value_NA",
col="variable",
common_bins=False,
facet_kws={
"sharex": False,
"sharey": False
}
)
)
)
)
session_info.show()