import numpy as np # module for creating and manipulating arrays
import matplotlib.pyplot as plt # plotting
import pandas as pd # dataframes
url = 'https://raw.githubusercontent.com/data-8/materials-sp18/master/lec/san_francisco_2015.csv'
df0 = pd.read_csv(url)
print("Employee compensations in the City of San Francisco in the"\
"year 2015\nShape: {} \nColumns: {}".format(df0.shape, df0.columns))
df0.head()
df0.sort_values(['Total Compensation'], ascending=False)[:3]
df0.sort_values(['Total Compensation'], ascending=True)[:3]
df = df0.loc[df0['Salaries'] > 10000]
TC = df['Total Compensation'] / 1000 # scale the Total compensation
TC
# you can use NumPy to compute the statistics
TC_mean = TC.sum() / TC.size
a = TC.sum() - TC_mean
b = a**2
c = b / TC.size
TC_std = np.sqrt(c)
TC_min = TC.min()
TC_max = TC.max()
# Report the mean 'Total Compensation' using the standard deviation and uncertainties
print("The mean Total Compensation is {:.0f} +- {:.0f} (mean +- SD) "\
"thousand dollars annually, \nwith a range "\
"({:.2f}, {:.2f})".format(TC_mean, TC_std, TC_min, TC_max))
plt.hist(TC, range=(0, 700), bins=20, density=True)
plt.xlabel("Total Compensation (k$)")
TC_median = np.percentile(TC, 50, axis=-1)
print("The median is {:.1f} thousand dollars annually".format(TC_median))
print("10, 50 and 90 percentiles {}".format( np.percentile(TC, [10, 50, 90])))
# draw sample with 500 elements
n = 500
our_sample = np.random.choice(TC, size=n, replace=None, p=None)
# plot an histogram
plt.hist(our_sample, range=(0,700))
# Calcualte the median of the sample
sample_median = np.percentile(TC, 50, axis=-1)
print("Sample median {:.2f}".format(sample_median))
# draw a sample with replacement from the original sample our_sample
resample_1 = np.random.choice(our_sample, n, replace=True)
# plot histogram of the drawed sample
# plt. ... your code here...
def bootstrap_median(original_sample, replications):
"""Returns an array of bootstrapped sample medians
Parameters
----------
original_sample: array containing the original sample
replications: number of bootstrap samples
"""
n = len(original_sample)
medians = np.zeros(replications)
for i in np.arange(replications):
bootstrap_sample = np.random.choice(original_sample, n, replace=True)
resampled_median = np.median(bootstrap_sample)
medians[i] = resampled_median
return medians
bstrap_medians = bootstrap_median(our_sample, 5000)
# calculate the right percentile 2.5 and left percentile 97.5 of bstrap_medians
leftp = np.percentile(TC, 2.5, axis=-1)
rightp = np.percentile(TC, 97.5, axis=-1)
print("The sample median {:.1f}\n95% confidence interval of the median ({:.1f}, {:.1f})".format(sample_median, leftp, rightp))
# plot histogram of the bstrap_medians with the 95% confidence intervals and the sample median
plt.hist(bstrap_medians, density=True)
# indicate the middle 95% with an horizontal line
plt.hlines(0.0, leftp, rightp, 'y', lw=8) # 5 and 95 percentiles
# indicate the sample median with a red dot
plt.plot(sample_median, 0.002, 'ro', ms=10) # median
plt.xlabel('Bootstrapped medians')
