mini-project-1

import pandas as pd import numpy as np import matplotlib.pyplot as plt from scipy.stats import rv_discrete

url='android.csv' df=pd.read_csv(url) print("Data.shape = ", df.shape) columns = [col for col in df.columns] print("Columns: ", columns)

df.info()

# Helper functions to deal with null values def null_indices(df): #Returns all null indices for each attribute as a dictonary return {col : df[df[col].isnull()].index.tolist() for col in df.columns} def all_null_indices(df): # Returns all the null_indices in the df empty_index_list = [] for col in df.columns: empty_index_list = df[df[col].isnull()].index.tolist() + (empty_index_list) return list(set(empty_index_list)) #Observe the null data indexes null_index = null_indices(df) all_null_indexes = all_null_indices(df) print("Before cleaning:") print(null_index) print(all_null_indexes) #NULL data removed df = df.dropna() #Check null_index = null_indices(df) all_null_indexes = all_null_indices(df) print("After cleaning:") print(null_index) print(all_null_indexes)

for col in df.columns: unq = df[col].unique() unq.sort() print(col, " : \n", unq, "\n")

#creating a dataframe showing frequences of app usages app_usage_df = pd.DataFrame(columns=['app_name', "is_active", "is_passive", "frequency"]) apps = columns[5:-2] for app in apps: appUsage = df[app].value_counts().to_dict() is_active = appUsage.get(1) is_passive = appUsage.get(0) if is_active==None: is_active=0 if is_passive==None: is_passive=0 appFreq = is_active / (is_active + is_passive) #print(app, "\n", appUsage, "appFreq=", appFreq ,"\n") app_usage_df = app_usage_df.append({'app_name':app, "is_active":is_active, "is_passive":is_passive, "frequency":appFreq}, ignore_index=True) app_usage_df = app_usage_df.sort_values(by='frequency', ascending=False) pd.set_option('display.max_rows', 100) print(app_usage_df.to_string()) _=pd.reset_option('all')

#JUST FOR ANALYSING #CLEANED FORM OF app_usage_df cleaned = app_usage_df.copy() indexNames = cleaned[ cleaned['frequency'] > 0.99 ].index cleaned.drop(indexNames , inplace=True) indexNames = cleaned[ cleaned['frequency'] < 0.01 ].index cleaned.drop(indexNames , inplace=True) pd.set_option('display.max_rows', 100) print(cleaned.to_string()) _=pd.reset_option('all')

col_idx_to_be_deleted = app_usage_df.loc[ (app_usage_df['frequency'] > 0.99) | (app_usage_df['frequency'] < 0.01) ].index for i in app_usage_df['app_name'][col_idx_to_be_deleted]: print("Column", i, "dropped! Frequency:", float(app_usage_df[ app_usage_df['app_name']==i ]['frequency'])) df.drop(i, axis=1, inplace=True)

#print(df['battery_level'].value_counts()) batteryLevels = df['battery_level'].value_counts().to_dict() positiveChange = 0 negativeChange = 0 noChange = 0 for key, value in batteryLevels.items() : #print("key = ", key, " value: ", value) if key>0: positiveChange += value elif key<0: negativeChange += value else: noChange += value print("Positive Change = ", positiveChange) print("Negative Change = ", negativeChange) print("No Change = ", noChange)

def visualizingAppUsagePerTimeSlot(): slots = df["slot"].unique() slots.sort() x_axis = slots y_axis = [] app_columns = df.columns[5:-3] for key in slots: filter = df["slot"]==key filtered_df = df.where(filter) filtered_df = filtered_df.dropna() app_usage_per_slot = 0 for app in app_columns: #print(app, " frequency in this time slot = ", df2[app].value_counts().to_dict().get(1)) if filtered_df[app].value_counts().to_dict().get(1)!=None: app_usage_per_slot += filtered_df[app].value_counts().to_dict().get(1) #print("TOTAL app usage for this slot = ", app_usage_per_slot) y_axis.append(app_usage_per_slot) fig = plt.figure(figsize=(30, 10)) ax = fig.gca() ax.set_xticks(x_axis) ax.bar(x_axis, y_axis) ax.set_xlabel("slots") ax.set_ylabel("total app usages") plt.show() visualizingAppUsagePerTimeSlot()

# Drop values with year=1970 df = df[df.year != 1970] # Print {Year: Frequency} counts = df['year'].value_counts().to_dict() for k,v in counts.items(): print("Year:", k, "Frequency:", v)

df['time'] = pd.to_datetime(df[['year','month','day']]) print(df['time'])

battery_plugged_gt_0 = df[df['battery_plugged']>0] battery_plugged_eq_0 = df[df['battery_plugged']==0] battery_plugged_gt_0_fixed = df.loc[(df['battery_plugged']>0) & (df['battery_status']>0)] battery_plugged_eq_0_fixed = df.loc[(df['battery_plugged']==0) & (df['battery_status']>0)] fig, axs = plt.subplots(2,2, figsize=(10, 5)) axs[0,0].hist(battery_plugged_gt_0['battery_level'], rwidth=0.9) axs[0,0].set_xlabel('battery_level') axs[0,0].set_ylabel('frequency') axs[0,0].set_title('battery_plugged>0') axs[1,0].hist(battery_plugged_eq_0['battery_level'], rwidth=0.9) axs[1,0].set_xlabel('battery_level') axs[1,0].set_ylabel('frequency') axs[1,0].set_title('battery_plugged==0') axs[0,1].hist(battery_plugged_gt_0_fixed['battery_level'], rwidth=0.9) axs[0,1].set_xlabel('battery_level') axs[0,1].set_ylabel('frequency') axs[0,1].set_title('battery_plugged>0 & battery_status>0') axs[1,1].hist(battery_plugged_eq_0_fixed['battery_level'], rwidth=0.9) axs[1,1].set_xlabel('battery_level') axs[1,1].set_ylabel('frequency') axs[1,1].set_title('battery_plugged==0 & battery_status>0') fig.tight_layout()

df_state_prediction = df.drop(columns = ['battery_status','battery_plugged']) df_state_prediction.to_csv('state_prediction.csv', index=False)

# clean charging data charging_data_idx = df.loc[(df['battery_plugged']==1) | (df['battery_status']==2) | (df['battery_status']>=4) | (df['battery_level']>0)].index print("[Charging Data]: Rows to be deleted:", len(charging_data_idx)) df = df.drop(charging_data_idx) # after this cleaning battery_status and battery_plugged will be meaningless df = df.drop('battery_status', 1) df = df.drop('battery_plugged', 1)

df.to_csv('clean_android.csv', index=False)

#pd.set_option('display.max_columns', None) #print("SUM OF NULLS: ", df.isnull().sum(), " --- \n\n") #print(df.isnull()) #print(df.loc[1520,:])