import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime as dt
from datetime import timedelta
from dateutil.parser import parse as parse_date
import pdb
import seaborn as sns
from sklearn.linear_model import LinearRegression
%matplotlib inline
folder= 'data/'
json_files = os.listdir(folder)
data = pd.DataFrame()
for file in json_files:
full_path_file = folder+file
print(full_path_file)
file_df = pd.read_json(full_path_file)
data = pd.concat([file_df, data])
data/UDSFile_2019-07-18_2019-10-26.json
data/UDSFile_2020-05-13_2020-08-21.json
data/UDSFile_2019-10-26_2020-02-03.json
data/UDSFile_2020-08-21_2020-11-29.json
data/UDSFile_2020-02-03_2020-05-13.json
data/UDSFile_2019-04-09_2019-07-18.json
data/UDSFile_2018-12-30_2019-04-09.json
data.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 617 entries, 0 to 99
Data columns (total 44 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 userProfilePK 617 non-null int64
1 calendarDate 617 non-null object
2 uuid 617 non-null object
3 durationInMilliseconds 617 non-null int64
4 rulePk 617 non-null int64
5 totalKilocalories 616 non-null float64
6 activeKilocalories 616 non-null float64
7 bmrKilocalories 611 non-null float64
8 wellnessKilocalories 611 non-null float64
9 remainingKilocalories 616 non-null float64
10 wellnessTotalKilocalories 616 non-null float64
11 wellnessActiveKilocalories 611 non-null float64
12 totalSteps 611 non-null float64
13 dailyStepGoal 611 non-null float64
14 totalDistanceMeters 616 non-null float64
15 wellnessDistanceMeters 611 non-null float64
16 wellnessStartTimeGmt 616 non-null object
17 wellnessEndTimeGmt 616 non-null object
18 wellnessStartTimeLocal 616 non-null object
19 wellnessEndTimeLocal 616 non-null object
20 highlyActiveSeconds 611 non-null float64
21 activeSeconds 611 non-null float64
22 moderateIntensityMinutes 611 non-null float64
23 vigorousIntensityMinutes 611 non-null float64
24 userIntensityMinutesGoal 611 non-null float64
25 userFloorsAscendedGoal 611 non-null float64
26 minHeartRate 611 non-null float64
27 maxHeartRate 611 non-null float64
28 restingHeartRate 610 non-null float64
29 currentDayRestingHeartRate 610 non-null float64
30 restingHeartRateTimestamp 610 non-null object
31 includesWellnessData 617 non-null bool
32 includesActivityData 617 non-null bool
33 includesCalorieConsumedData 617 non-null bool
34 includesSingleMeasurement 611 non-null float64
35 includesContinuousMeasurement 611 non-null float64
36 includesAllDayPulseOx 611 non-null float64
37 includesSleepPulseOx 611 non-null float64
38 source 616 non-null float64
39 allDayStress 617 non-null object
40 minAvgHeartRate 611 non-null float64
41 maxAvgHeartRate 611 non-null float64
42 version 617 non-null int64
43 burnedKilocalories 28 non-null float64
dtypes: bool(3), float64(29), int64(4), object(8)
memory usage: 204.3+ KB
data.head()
data['date'] = data['calendarDate'].apply(lambda x: pd.to_datetime(x['date']))
steps_data = data[['date', 'totalSteps']].set_index('date')
steps_data.head()
| totalStepsfloat64 | ||
|---|---|---|
2019-03-19T00:00:00.000000 | 15 | |
2019-03-20T00:00:00.000000 | 9446 | |
2019-03-21T00:00:00.000000 | 10090 | |
2019-03-22T00:00:00.000000 | 15579 | |
2019-03-23T00:00:00.000000 | 16440 | |
| 5 rows × 1 columns | ||
steps_data.describe()
| totalStepsfloat64 | ||
|---|---|---|
count | 611 | |
mean | 13181.034369885434 | |
std | 8153.262316225169 | |
min | 15 | |
25% | 7751.5 | |
50% | 11821 | |
75% | 17187 | |
max | 49100 | |
| 8 rows × 1 columns | ||
import matplotlib
ax = steps_data.plot(kind='line', fontsize=14, title ="Steps/Day", figsize=(14,7))
ax.set_xlabel('Day')
ax.set_ylabel('# Steps', fontsize=14)
ax.get_legend().remove()
plt.grid(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
ax = steps_data.rolling(window = 7).mean().plot(kind='line', fontsize=14, title ="Rolling 7 Days - Mean Steps", figsize=(14,7))
ax.set_xlabel('Day')
ax.set_ylabel('# Steps', fontsize=14)
ax.get_legend().remove()
plt.grid(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
by_week = steps_data.groupby(pd.Grouper(freq ='W')).sum()
ax = by_week.plot(kind='line', fontsize=14, title ="Steps per Week", figsize=(14,7))
ax.set_xlabel('Week')
ax.set_ylabel('# Steps', fontsize=14)
ax.get_legend().remove()
plt.grid(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
steps_data['month'] = steps_data.index.strftime('%Y-%m')
steps_data[['month', 'totalSteps']]
monthly_grouped=steps_data.groupby('month').mean()
by_month = steps_data.groupby(pd.Grouper(freq ='M')).mean()
by_month['month'] = by_month.reset_index()['date'].dt.strftime('%Y-%m')
ax = monthly_grouped.plot(kind='bar', fontsize=14, title ="Mean Daily Steps by Month", figsize=(14,7))
ax.set_xlabel('Month')
ax.set_ylabel('# Steps', fontsize=14)
ax.get_legend().remove()
plt.grid(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
chart = sns.histplot(steps_data, kde=False, color='green', bins=20)
plt.title('Distribution of Steps', fontsize=18)
plt.xlabel('Daily Steps', fontsize=16)
plt.ylabel('Frequency', fontsize=16)
# figure size in inches
sns.set(rc={'figure.figsize':(11,9)})
sns.set(style="whitegrid")
for key, spine in chart.spines.items():
spine.set_visible(False)
plt.grid(False)
ax.get_xaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
steps_data['day_of_week'] = steps_data.index.day_name()
steps_data['month_name'] = steps_data.index.month_name()
plt.figure(figsize=(10,8))
ax = sns.boxplot(x="day_of_week", y="totalSteps", data=steps_data)
plt.title('Steps / Day of Week', fontsize=18)
plt.xlabel('Day of Week', fontsize=16)
plt.ylabel('Steps', fontsize=16)
sns.set(style="whitegrid")
for key, spine in chart.spines.items():
spine.set_visible(False)
plt.grid(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
plt.figure(figsize=(13,7))
ax = sns.boxplot(x="month_name", y="totalSteps", data=steps_data)
plt.title('Steps / Month', fontsize=18)
plt.xlabel('Month', fontsize=12)
plt.ylabel('Steps', fontsize=16)
sns.set(style="whitegrid")
for key, spine in chart.spines.items():
spine.set_visible(False)
plt.grid(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
steps_data.sort_index(inplace=True)
cum_sum_2020 = steps_data.loc['2020']
cum_sum_2020.reset_index(inplace=True)
cum_sum_2020['DayOfYear'] = cum_sum_2020['date'].dt.dayofyear
cum_sum_2020['cumulative_steps'] = cum_sum_2020['totalSteps'].cumsum().dropna()
cum_sum_2020[['DayOfYear', 'totalSteps', 'cumulative_steps']].head()
/opt/venv/lib/python3.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
after removing the cwd from sys.path.
/opt/venv/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
"""
| DayOfYearint64 | totalStepsfloat64 | cumulative_stepsfloat64 | ||
|---|---|---|---|---|
0 | 1 | 10926 | 10926 | |
1 | 2 | 19469 | 30395 | |
2 | 3 | 10358 | 40753 | |
3 | 4 | 19813 | 60566 | |
4 | 5 | 19952 | 80518 | |
| 5 rows × 3 columns | ||||
cum_series = cum_sum_2020.set_index('DayOfYear')['cumulative_steps'].dropna()
ax = cum_series.plot(kind='line', fontsize=14, title ="Cumulative Steps", figsize=(14,7))
ax.set_xlabel('Day of Year - 2020')
ax.set_ylabel('# Steps', fontsize=14)
plt.grid(False)
for key, spine in chart.spines.items():
spine.set_visible(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
X = cum_series[-60:].index.values.reshape(-1, 1)
Y = cum_series[-60:].values.reshape(-1, 1)
linear_regressor = LinearRegression() # create object for the class
linear_regressor.fit(X, Y) # perform linear regression
start_day = cum_series.index.values.max()
predict_days = np.arange(start_day,366)
prediction = linear_regressor.predict(predict_days.reshape(-1, 1))
cum_series = cum_sum_2020.set_index('DayOfYear')['cumulative_steps'].dropna()
ax = cum_series.plot(kind='line', fontsize=14, title ="Cumulative Steps", figsize=(14,7))
ax.set_xlabel('Day of Year - 2020')
ax.set_ylabel('# Steps', fontsize=14)
plt.grid(False)
for key, spine in chart.spines.items():
spine.set_visible(False)
ax.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
plt.plot(predict_days, prediction, color = 'red', linewidth=4, )
end_of_year_total = int(round(prediction[-1][0],0))
mean_steps_day_prediction = int(end_of_year_total/365)
print('Total predicted steps:')
print(f'{end_of_year_total:,}')
print('Mean steps/day prediction:')
print(f'{mean_steps_day_prediction:,}')
Total predicted steps:
3,496,750
Mean steps/day prediction:
9,580