import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import rv_discrete
url='android.csv'
df=pd.read_csv(url)
print("Data.shape = ", df.shape)
columns = [col for col in df.columns]
print("Columns: ", columns)
Data.shape = (2469, 72)
Columns: ['battery_level', 'battery_plugged', 'battery_status', 'day', 'month', 'packages_running_android', 'packages_running_ch.smalltech.battery.free', 'packages_running_com.android.bluetooth', 'packages_running_com.android.calculator2', 'packages_running_com.android.chrome', 'packages_running_com.android.contacts', 'packages_running_com.android.defcontainer', 'packages_running_com.android.keychain', 'packages_running_com.android.launcher', 'packages_running_com.android.mms', 'packages_running_com.android.musicfx', 'packages_running_com.android.nfc', 'packages_running_com.android.noisefield', 'packages_running_com.android.packageinstaller', 'packages_running_com.android.phone', 'packages_running_com.android.providers.applications', 'packages_running_com.android.providers.calendar', 'packages_running_com.android.providers.contacts', 'packages_running_com.android.providers.downloads', 'packages_running_com.android.providers.drm', 'packages_running_com.android.providers.media', 'packages_running_com.android.providers.settings', 'packages_running_com.android.providers.telephony', 'packages_running_com.android.providers.userdictionary', 'packages_running_com.android.settings', 'packages_running_com.android.stk', 'packages_running_com.android.systemui', 'packages_running_com.android.vending', 'packages_running_com.google.android.apps.books', 'packages_running_com.google.android.apps.genie.geniewidget', 'packages_running_com.google.android.apps.maps', 'packages_running_com.google.android.apps.plus', 'packages_running_com.google.android.apps.uploader', 'packages_running_com.google.android.backup', 'packages_running_com.google.android.browser', 'packages_running_com.google.android.calendar', 'packages_running_com.google.android.deskclock', 'packages_running_com.google.android.email', 'packages_running_com.google.android.exchange', 'packages_running_com.google.android.gallery3d', 'packages_running_com.google.android.gm', 'packages_running_com.google.android.gms', 'packages_running_com.google.android.googlequicksearchbox', 'packages_running_com.google.android.gsf', 'packages_running_com.google.android.gsf.login', 'packages_running_com.google.android.inputmethod.latin', 'packages_running_com.google.android.inputmethod.latin.dictionarypack', 'packages_running_com.google.android.location', 'packages_running_com.google.android.music', 'packages_running_com.google.android.onetimeinitializer', 'packages_running_com.google.android.partnersetup', 'packages_running_com.google.android.setupwizard', 'packages_running_com.google.android.syncadapters.contacts', 'packages_running_com.google.android.talk', 'packages_running_com.google.android.videos', 'packages_running_com.google.android.youtube', 'packages_running_com.googlecode.android_scripting', 'packages_running_com.googlecode.pythonforandroid', 'packages_running_com.halfbrick.fruitninjafree', 'packages_running_com.rovio.angrybirdsspace.ads', 'packages_running_com.tf.thinkdroid.sg', 'packages_running_de.kicktipp.mbookmark', 'packages_running_edu.udo.cs.ess.mobidac.target', 'packages_running_org.openintents.filemanager', 'packages_running_stream.android', 'slot', 'year']
df.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2469 entries, 0 to 2468
Data columns (total 72 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 battery_level 2468 non-null float64
1 battery_plugged 2468 non-null float64
2 battery_status 2468 non-null float64
3 day 2468 non-null float64
4 month 2468 non-null float64
5 packages_running_android 2468 non-null float64
6 packages_running_ch.smalltech.battery.free 2468 non-null float64
7 packages_running_com.android.bluetooth 2468 non-null float64
8 packages_running_com.android.calculator2 2468 non-null float64
9 packages_running_com.android.chrome 2468 non-null float64
10 packages_running_com.android.contacts 2468 non-null float64
11 packages_running_com.android.defcontainer 2468 non-null float64
12 packages_running_com.android.keychain 2468 non-null float64
13 packages_running_com.android.launcher 2468 non-null float64
14 packages_running_com.android.mms 2468 non-null float64
15 packages_running_com.android.musicfx 2468 non-null float64
16 packages_running_com.android.nfc 2468 non-null float64
17 packages_running_com.android.noisefield 2468 non-null float64
18 packages_running_com.android.packageinstaller 2468 non-null float64
19 packages_running_com.android.phone 2468 non-null float64
20 packages_running_com.android.providers.applications 2468 non-null float64
21 packages_running_com.android.providers.calendar 2468 non-null float64
22 packages_running_com.android.providers.contacts 2468 non-null float64
23 packages_running_com.android.providers.downloads 2468 non-null float64
24 packages_running_com.android.providers.drm 2468 non-null float64
25 packages_running_com.android.providers.media 2468 non-null float64
26 packages_running_com.android.providers.settings 2468 non-null float64
27 packages_running_com.android.providers.telephony 2468 non-null float64
28 packages_running_com.android.providers.userdictionary 2468 non-null float64
29 packages_running_com.android.settings 2468 non-null float64
30 packages_running_com.android.stk 2468 non-null float64
31 packages_running_com.android.systemui 2468 non-null float64
32 packages_running_com.android.vending 2468 non-null float64
33 packages_running_com.google.android.apps.books 2468 non-null float64
34 packages_running_com.google.android.apps.genie.geniewidget 2468 non-null float64
35 packages_running_com.google.android.apps.maps 2468 non-null float64
36 packages_running_com.google.android.apps.plus 2468 non-null float64
37 packages_running_com.google.android.apps.uploader 2468 non-null float64
38 packages_running_com.google.android.backup 2468 non-null float64
39 packages_running_com.google.android.browser 2468 non-null float64
40 packages_running_com.google.android.calendar 2468 non-null float64
41 packages_running_com.google.android.deskclock 2468 non-null float64
42 packages_running_com.google.android.email 2468 non-null float64
43 packages_running_com.google.android.exchange 2468 non-null float64
44 packages_running_com.google.android.gallery3d 2468 non-null float64
45 packages_running_com.google.android.gm 2468 non-null float64
46 packages_running_com.google.android.gms 2468 non-null float64
47 packages_running_com.google.android.googlequicksearchbox 2468 non-null float64
48 packages_running_com.google.android.gsf 2468 non-null float64
49 packages_running_com.google.android.gsf.login 2468 non-null float64
50 packages_running_com.google.android.inputmethod.latin 2468 non-null float64
51 packages_running_com.google.android.inputmethod.latin.dictionarypack 2468 non-null float64
52 packages_running_com.google.android.location 2468 non-null float64
53 packages_running_com.google.android.music 2468 non-null float64
54 packages_running_com.google.android.onetimeinitializer 2468 non-null float64
55 packages_running_com.google.android.partnersetup 2468 non-null float64
56 packages_running_com.google.android.setupwizard 2468 non-null float64
57 packages_running_com.google.android.syncadapters.contacts 2468 non-null float64
58 packages_running_com.google.android.talk 2468 non-null float64
59 packages_running_com.google.android.videos 2468 non-null float64
60 packages_running_com.google.android.youtube 2468 non-null float64
61 packages_running_com.googlecode.android_scripting 2468 non-null float64
62 packages_running_com.googlecode.pythonforandroid 2468 non-null float64
63 packages_running_com.halfbrick.fruitninjafree 2468 non-null float64
64 packages_running_com.rovio.angrybirdsspace.ads 2468 non-null float64
65 packages_running_com.tf.thinkdroid.sg 2468 non-null float64
66 packages_running_de.kicktipp.mbookmark 2468 non-null float64
67 packages_running_edu.udo.cs.ess.mobidac.target 2468 non-null float64
68 packages_running_org.openintents.filemanager 2468 non-null float64
69 packages_running_stream.android 2468 non-null float64
70 slot 2468 non-null float64
71 year 2468 non-null float64
dtypes: float64(72)
memory usage: 1.4 MB
# Helper functions to deal with null values
def null_indices(df):
#Returns all null indices for each attribute as a dictonary
return {col : df[df[col].isnull()].index.tolist() for col in df.columns}
def all_null_indices(df):
# Returns all the null_indices in the df
empty_index_list = []
for col in df.columns:
empty_index_list = df[df[col].isnull()].index.tolist() + (empty_index_list)
return list(set(empty_index_list))
#Observe the null data indexes
null_index = null_indices(df)
all_null_indexes = all_null_indices(df)
print("Before cleaning:")
print(null_index)
print(all_null_indexes)
#NULL data removed
df = df.dropna()
#Check
null_index = null_indices(df)
all_null_indexes = all_null_indices(df)
print("After cleaning:")
print(null_index)
print(all_null_indexes)
Before cleaning:
{'battery_level': [1520], 'battery_plugged': [1520], 'battery_status': [1520], 'day': [1520], 'month': [1520], 'packages_running_android': [1520], 'packages_running_ch.smalltech.battery.free': [1520], 'packages_running_com.android.bluetooth': [1520], 'packages_running_com.android.calculator2': [1520], 'packages_running_com.android.chrome': [1520], 'packages_running_com.android.contacts': [1520], 'packages_running_com.android.defcontainer': [1520], 'packages_running_com.android.keychain': [1520], 'packages_running_com.android.launcher': [1520], 'packages_running_com.android.mms': [1520], 'packages_running_com.android.musicfx': [1520], 'packages_running_com.android.nfc': [1520], 'packages_running_com.android.noisefield': [1520], 'packages_running_com.android.packageinstaller': [1520], 'packages_running_com.android.phone': [1520], 'packages_running_com.android.providers.applications': [1520], 'packages_running_com.android.providers.calendar': [1520], 'packages_running_com.android.providers.contacts': [1520], 'packages_running_com.android.providers.downloads': [1520], 'packages_running_com.android.providers.drm': [1520], 'packages_running_com.android.providers.media': [1520], 'packages_running_com.android.providers.settings': [1520], 'packages_running_com.android.providers.telephony': [1520], 'packages_running_com.android.providers.userdictionary': [1520], 'packages_running_com.android.settings': [1520], 'packages_running_com.android.stk': [1520], 'packages_running_com.android.systemui': [1520], 'packages_running_com.android.vending': [1520], 'packages_running_com.google.android.apps.books': [1520], 'packages_running_com.google.android.apps.genie.geniewidget': [1520], 'packages_running_com.google.android.apps.maps': [1520], 'packages_running_com.google.android.apps.plus': [1520], 'packages_running_com.google.android.apps.uploader': [1520], 'packages_running_com.google.android.backup': [1520], 'packages_running_com.google.android.browser': [1520], 'packages_running_com.google.android.calendar': [1520], 'packages_running_com.google.android.deskclock': [1520], 'packages_running_com.google.android.email': [1520], 'packages_running_com.google.android.exchange': [1520], 'packages_running_com.google.android.gallery3d': [1520], 'packages_running_com.google.android.gm': [1520], 'packages_running_com.google.android.gms': [1520], 'packages_running_com.google.android.googlequicksearchbox': [1520], 'packages_running_com.google.android.gsf': [1520], 'packages_running_com.google.android.gsf.login': [1520], 'packages_running_com.google.android.inputmethod.latin': [1520], 'packages_running_com.google.android.inputmethod.latin.dictionarypack': [1520], 'packages_running_com.google.android.location': [1520], 'packages_running_com.google.android.music': [1520], 'packages_running_com.google.android.onetimeinitializer': [1520], 'packages_running_com.google.android.partnersetup': [1520], 'packages_running_com.google.android.setupwizard': [1520], 'packages_running_com.google.android.syncadapters.contacts': [1520], 'packages_running_com.google.android.talk': [1520], 'packages_running_com.google.android.videos': [1520], 'packages_running_com.google.android.youtube': [1520], 'packages_running_com.googlecode.android_scripting': [1520], 'packages_running_com.googlecode.pythonforandroid': [1520], 'packages_running_com.halfbrick.fruitninjafree': [1520], 'packages_running_com.rovio.angrybirdsspace.ads': [1520], 'packages_running_com.tf.thinkdroid.sg': [1520], 'packages_running_de.kicktipp.mbookmark': [1520], 'packages_running_edu.udo.cs.ess.mobidac.target': [1520], 'packages_running_org.openintents.filemanager': [1520], 'packages_running_stream.android': [1520], 'slot': [1520], 'year': [1520]}
[1520]
After cleaning:
{'battery_level': [], 'battery_plugged': [], 'battery_status': [], 'day': [], 'month': [], 'packages_running_android': [], 'packages_running_ch.smalltech.battery.free': [], 'packages_running_com.android.bluetooth': [], 'packages_running_com.android.calculator2': [], 'packages_running_com.android.chrome': [], 'packages_running_com.android.contacts': [], 'packages_running_com.android.defcontainer': [], 'packages_running_com.android.keychain': [], 'packages_running_com.android.launcher': [], 'packages_running_com.android.mms': [], 'packages_running_com.android.musicfx': [], 'packages_running_com.android.nfc': [], 'packages_running_com.android.noisefield': [], 'packages_running_com.android.packageinstaller': [], 'packages_running_com.android.phone': [], 'packages_running_com.android.providers.applications': [], 'packages_running_com.android.providers.calendar': [], 'packages_running_com.android.providers.contacts': [], 'packages_running_com.android.providers.downloads': [], 'packages_running_com.android.providers.drm': [], 'packages_running_com.android.providers.media': [], 'packages_running_com.android.providers.settings': [], 'packages_running_com.android.providers.telephony': [], 'packages_running_com.android.providers.userdictionary': [], 'packages_running_com.android.settings': [], 'packages_running_com.android.stk': [], 'packages_running_com.android.systemui': [], 'packages_running_com.android.vending': [], 'packages_running_com.google.android.apps.books': [], 'packages_running_com.google.android.apps.genie.geniewidget': [], 'packages_running_com.google.android.apps.maps': [], 'packages_running_com.google.android.apps.plus': [], 'packages_running_com.google.android.apps.uploader': [], 'packages_running_com.google.android.backup': [], 'packages_running_com.google.android.browser': [], 'packages_running_com.google.android.calendar': [], 'packages_running_com.google.android.deskclock': [], 'packages_running_com.google.android.email': [], 'packages_running_com.google.android.exchange': [], 'packages_running_com.google.android.gallery3d': [], 'packages_running_com.google.android.gm': [], 'packages_running_com.google.android.gms': [], 'packages_running_com.google.android.googlequicksearchbox': [], 'packages_running_com.google.android.gsf': [], 'packages_running_com.google.android.gsf.login': [], 'packages_running_com.google.android.inputmethod.latin': [], 'packages_running_com.google.android.inputmethod.latin.dictionarypack': [], 'packages_running_com.google.android.location': [], 'packages_running_com.google.android.music': [], 'packages_running_com.google.android.onetimeinitializer': [], 'packages_running_com.google.android.partnersetup': [], 'packages_running_com.google.android.setupwizard': [], 'packages_running_com.google.android.syncadapters.contacts': [], 'packages_running_com.google.android.talk': [], 'packages_running_com.google.android.videos': [], 'packages_running_com.google.android.youtube': [], 'packages_running_com.googlecode.android_scripting': [], 'packages_running_com.googlecode.pythonforandroid': [], 'packages_running_com.halfbrick.fruitninjafree': [], 'packages_running_com.rovio.angrybirdsspace.ads': [], 'packages_running_com.tf.thinkdroid.sg': [], 'packages_running_de.kicktipp.mbookmark': [], 'packages_running_edu.udo.cs.ess.mobidac.target': [], 'packages_running_org.openintents.filemanager': [], 'packages_running_stream.android': [], 'slot': [], 'year': []}
[]
for col in df.columns:
unq = df[col].unique()
unq.sort()
print(col, " : \n", unq, "\n")
battery_level :
[-10. -9. -8. -7. -6. -5. -4. -3. -2. -1. 0. 1. 2. 3.
4. 5. 6. 7. 8. 9. 10. 11. 12. 13. 14. 15. 16. 17.
18. 19. 20. 21. 22. 23. 24. 25. 26. 27. 28. 29. 30.]
battery_plugged :
[0. 0.5 0.66666667 0.75 1. 2. ]
battery_status :
[0. 2. 2.33333333 2.5 2.66666667 3.
3.5 3.66666667 4. 4.5 4.66666667 5. ]
day :
[ 1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12. 13. 14. 15. 16. 17. 18.
19. 20. 21. 22. 23. 24. 25. 26. 27. 28. 29. 30. 31.]
month :
[ 1. 2. 3. 8. 9. 10. 11. 12.]
packages_running_android :
[0. 1.]
packages_running_ch.smalltech.battery.free :
[0. 1.]
packages_running_com.android.bluetooth :
[0. 1.]
packages_running_com.android.calculator2 :
[0. 1.]
packages_running_com.android.chrome :
[0. 1.]
packages_running_com.android.contacts :
[0. 1.]
packages_running_com.android.defcontainer :
[0. 1.]
packages_running_com.android.keychain :
[0. 1.]
packages_running_com.android.launcher :
[0. 1.]
packages_running_com.android.mms :
[0. 1.]
packages_running_com.android.musicfx :
[0. 1.]
packages_running_com.android.nfc :
[0. 1.]
packages_running_com.android.noisefield :
[0. 1.]
packages_running_com.android.packageinstaller :
[0. 1.]
packages_running_com.android.phone :
[0. 1.]
packages_running_com.android.providers.applications :
[0. 1.]
packages_running_com.android.providers.calendar :
[0. 1.]
packages_running_com.android.providers.contacts :
[0. 1.]
packages_running_com.android.providers.downloads :
[0. 1.]
packages_running_com.android.providers.drm :
[0. 1.]
packages_running_com.android.providers.media :
[0. 1.]
packages_running_com.android.providers.settings :
[0. 1.]
packages_running_com.android.providers.telephony :
[0. 1.]
packages_running_com.android.providers.userdictionary :
[0. 1.]
packages_running_com.android.settings :
[0. 1.]
packages_running_com.android.stk :
[0. 1.]
packages_running_com.android.systemui :
[0. 1.]
packages_running_com.android.vending :
[0. 1.]
packages_running_com.google.android.apps.books :
[0. 1.]
packages_running_com.google.android.apps.genie.geniewidget :
[0. 1.]
packages_running_com.google.android.apps.maps :
[0. 1.]
packages_running_com.google.android.apps.plus :
[0. 1.]
packages_running_com.google.android.apps.uploader :
[0. 1.]
packages_running_com.google.android.backup :
[0. 1.]
packages_running_com.google.android.browser :
[0. 1.]
packages_running_com.google.android.calendar :
[0. 1.]
packages_running_com.google.android.deskclock :
[0. 1.]
packages_running_com.google.android.email :
[0. 1.]
packages_running_com.google.android.exchange :
[0. 1.]
packages_running_com.google.android.gallery3d :
[0. 1.]
packages_running_com.google.android.gm :
[0. 1.]
packages_running_com.google.android.gms :
[0. 1.]
packages_running_com.google.android.googlequicksearchbox :
[0. 1.]
packages_running_com.google.android.gsf :
[0. 1.]
packages_running_com.google.android.gsf.login :
[0. 1.]
packages_running_com.google.android.inputmethod.latin :
[0. 1.]
packages_running_com.google.android.inputmethod.latin.dictionarypack :
[0. 1.]
packages_running_com.google.android.location :
[0. 1.]
packages_running_com.google.android.music :
[0. 1.]
packages_running_com.google.android.onetimeinitializer :
[0.]
packages_running_com.google.android.partnersetup :
[0. 1.]
packages_running_com.google.android.setupwizard :
[0. 1.]
packages_running_com.google.android.syncadapters.contacts :
[0. 1.]
packages_running_com.google.android.talk :
[0. 1.]
packages_running_com.google.android.videos :
[0.]
packages_running_com.google.android.youtube :
[0. 1.]
packages_running_com.googlecode.android_scripting :
[0. 1.]
packages_running_com.googlecode.pythonforandroid :
[0. 1.]
packages_running_com.halfbrick.fruitninjafree :
[0. 1.]
packages_running_com.rovio.angrybirdsspace.ads :
[0. 1.]
packages_running_com.tf.thinkdroid.sg :
[0. 1.]
packages_running_de.kicktipp.mbookmark :
[0. 1.]
packages_running_edu.udo.cs.ess.mobidac.target :
[0. 1.]
packages_running_org.openintents.filemanager :
[0. 1.]
packages_running_stream.android :
[0. 1.]
slot :
[ 0. 1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12. 13. 14. 15. 16. 17.
18. 19. 20. 21. 22. 23. 24. 25. 26. 27. 28. 29. 30. 31. 32. 33. 34. 35.
36. 37. 38. 39. 40. 41. 42. 43. 44. 45. 46. 47.]
year :
[1970. 2012. 2013.]
#creating a dataframe showing frequences of app usages
app_usage_df = pd.DataFrame(columns=['app_name', "is_active", "is_passive", "frequency"])
apps = columns[5:-2]
for app in apps:
appUsage = df[app].value_counts().to_dict()
is_active = appUsage.get(1)
is_passive = appUsage.get(0)
if is_active==None:
is_active=0
if is_passive==None:
is_passive=0
appFreq = is_active / (is_active + is_passive)
#print(app, "\n", appUsage, "appFreq=", appFreq ,"\n")
app_usage_df = app_usage_df.append({'app_name':app, "is_active":is_active, "is_passive":is_passive, "frequency":appFreq}, ignore_index=True)
app_usage_df = app_usage_df.sort_values(by='frequency', ascending=False)
pd.set_option('display.max_rows', 100)
print(app_usage_df.to_string())
_=pd.reset_option('all')
app_name is_active is_passive frequency
0 packages_running_android 2467 1 0.999595
45 packages_running_com.google.android.inputmethod.latin 2467 1 0.999595
25 packages_running_com.android.stk 2467 1 0.999595
14 packages_running_com.android.phone 2467 1 0.999595
43 packages_running_com.google.android.gsf 2467 1 0.999595
12 packages_running_com.android.noisefield 2467 1 0.999595
11 packages_running_com.android.nfc 2467 1 0.999595
26 packages_running_com.android.systemui 2467 1 0.999595
8 packages_running_com.android.launcher 2467 1 0.999595
21 packages_running_com.android.providers.settings 2467 1 0.999595
30 packages_running_com.google.android.apps.maps 2467 1 0.999595
33 packages_running_com.google.android.backup 2467 1 0.999595
52 packages_running_com.google.android.syncadapters.contacts 2467 1 0.999595
53 packages_running_com.google.android.talk 2467 1 0.999595
56 packages_running_com.googlecode.android_scripting 2467 1 0.999595
22 packages_running_com.android.providers.telephony 2467 1 0.999595
62 packages_running_edu.udo.cs.ess.mobidac.target 2463 5 0.997974
37 packages_running_com.google.android.email 2461 7 0.997164
17 packages_running_com.android.providers.contacts 2460 8 0.996759
23 packages_running_com.android.providers.userdictionary 2460 8 0.996759
15 packages_running_com.android.providers.applications 2460 8 0.996759
19 packages_running_com.android.providers.drm 2458 10 0.995948
18 packages_running_com.android.providers.downloads 2458 10 0.995948
20 packages_running_com.android.providers.media 2458 10 0.995948
28 packages_running_com.google.android.apps.books 2421 47 0.980956
44 packages_running_com.google.android.gsf.login 2418 50 0.979741
47 packages_running_com.google.android.location 2290 178 0.927877
16 packages_running_com.android.providers.calendar 1846 622 0.747974
46 packages_running_com.google.android.inputmethod.latin.dictionarypack 1845 623 0.747569
35 packages_running_com.google.android.calendar 1807 661 0.732172
39 packages_running_com.google.android.gallery3d 1789 679 0.724878
1 packages_running_ch.smalltech.battery.free 1723 745 0.698136
29 packages_running_com.google.android.apps.genie.geniewidget 1684 784 0.682334
27 packages_running_com.android.vending 1683 785 0.681929
31 packages_running_com.google.android.apps.plus 1662 806 0.673420
32 packages_running_com.google.android.apps.uploader 1533 935 0.621151
61 packages_running_de.kicktipp.mbookmark 1373 1095 0.556321
34 packages_running_com.google.android.browser 1363 1105 0.552269
6 packages_running_com.android.defcontainer 1342 1126 0.543760
55 packages_running_com.google.android.youtube 1310 1158 0.530794
9 packages_running_com.android.mms 1215 1253 0.492301
60 packages_running_com.tf.thinkdroid.sg 1166 1302 0.472447
5 packages_running_com.android.contacts 1106 1362 0.448136
36 packages_running_com.google.android.deskclock 1088 1380 0.440843
57 packages_running_com.googlecode.pythonforandroid 1078 1390 0.436791
42 packages_running_com.google.android.googlequicksearchbox 784 1684 0.317666
38 packages_running_com.google.android.exchange 661 1807 0.267828
24 packages_running_com.android.settings 534 1934 0.216370
48 packages_running_com.google.android.music 495 1973 0.200567
50 packages_running_com.google.android.partnersetup 460 2008 0.186386
4 packages_running_com.android.chrome 254 2214 0.102917
10 packages_running_com.android.musicfx 240 2228 0.097245
41 packages_running_com.google.android.gms 123 2345 0.049838
51 packages_running_com.google.android.setupwizard 74 2394 0.029984
59 packages_running_com.rovio.angrybirdsspace.ads 66 2402 0.026742
40 packages_running_com.google.android.gm 61 2407 0.024716
58 packages_running_com.halfbrick.fruitninjafree 45 2423 0.018233
2 packages_running_com.android.bluetooth 28 2440 0.011345
7 packages_running_com.android.keychain 24 2444 0.009724
3 packages_running_com.android.calculator2 8 2460 0.003241
13 packages_running_com.android.packageinstaller 1 2467 0.000405
63 packages_running_org.openintents.filemanager 1 2467 0.000405
64 packages_running_stream.android 1 2467 0.000405
49 packages_running_com.google.android.onetimeinitializer 0 2468 0.000000
54 packages_running_com.google.android.videos 0 2468 0.000000
: boolean
use_inf_as_null had been deprecated and will be removed in a future
version. Use `use_inf_as_na` instead.
/opt/venv/lib/python3.7/site-packages/pandas/_config/config.py:620: FutureWarning:
: boolean
use_inf_as_null had been deprecated and will be removed in a future
version. Use `use_inf_as_na` instead.
warnings.warn(d.msg, FutureWarning)
#JUST FOR ANALYSING
#CLEANED FORM OF app_usage_df
cleaned = app_usage_df.copy()
indexNames = cleaned[ cleaned['frequency'] > 0.99 ].index
cleaned.drop(indexNames , inplace=True)
indexNames = cleaned[ cleaned['frequency'] < 0.01 ].index
cleaned.drop(indexNames , inplace=True)
pd.set_option('display.max_rows', 100)
print(cleaned.to_string())
_=pd.reset_option('all')
app_name is_active is_passive frequency
28 packages_running_com.google.android.apps.books 2421 47 0.980956
44 packages_running_com.google.android.gsf.login 2418 50 0.979741
47 packages_running_com.google.android.location 2290 178 0.927877
16 packages_running_com.android.providers.calendar 1846 622 0.747974
46 packages_running_com.google.android.inputmethod.latin.dictionarypack 1845 623 0.747569
35 packages_running_com.google.android.calendar 1807 661 0.732172
39 packages_running_com.google.android.gallery3d 1789 679 0.724878
1 packages_running_ch.smalltech.battery.free 1723 745 0.698136
29 packages_running_com.google.android.apps.genie.geniewidget 1684 784 0.682334
27 packages_running_com.android.vending 1683 785 0.681929
31 packages_running_com.google.android.apps.plus 1662 806 0.673420
32 packages_running_com.google.android.apps.uploader 1533 935 0.621151
61 packages_running_de.kicktipp.mbookmark 1373 1095 0.556321
34 packages_running_com.google.android.browser 1363 1105 0.552269
6 packages_running_com.android.defcontainer 1342 1126 0.543760
55 packages_running_com.google.android.youtube 1310 1158 0.530794
9 packages_running_com.android.mms 1215 1253 0.492301
60 packages_running_com.tf.thinkdroid.sg 1166 1302 0.472447
5 packages_running_com.android.contacts 1106 1362 0.448136
36 packages_running_com.google.android.deskclock 1088 1380 0.440843
57 packages_running_com.googlecode.pythonforandroid 1078 1390 0.436791
42 packages_running_com.google.android.googlequicksearchbox 784 1684 0.317666
38 packages_running_com.google.android.exchange 661 1807 0.267828
24 packages_running_com.android.settings 534 1934 0.216370
48 packages_running_com.google.android.music 495 1973 0.200567
50 packages_running_com.google.android.partnersetup 460 2008 0.186386
4 packages_running_com.android.chrome 254 2214 0.102917
10 packages_running_com.android.musicfx 240 2228 0.097245
41 packages_running_com.google.android.gms 123 2345 0.049838
51 packages_running_com.google.android.setupwizard 74 2394 0.029984
59 packages_running_com.rovio.angrybirdsspace.ads 66 2402 0.026742
40 packages_running_com.google.android.gm 61 2407 0.024716
58 packages_running_com.halfbrick.fruitninjafree 45 2423 0.018233
2 packages_running_com.android.bluetooth 28 2440 0.011345
: boolean
use_inf_as_null had been deprecated and will be removed in a future
version. Use `use_inf_as_na` instead.
/opt/venv/lib/python3.7/site-packages/pandas/_config/config.py:620: FutureWarning:
: boolean
use_inf_as_null had been deprecated and will be removed in a future
version. Use `use_inf_as_na` instead.
warnings.warn(d.msg, FutureWarning)
col_idx_to_be_deleted = app_usage_df.loc[ (app_usage_df['frequency'] > 0.99) | (app_usage_df['frequency'] < 0.01) ].index
for i in app_usage_df['app_name'][col_idx_to_be_deleted]:
print("Column", i, "dropped! Frequency:", float(app_usage_df[ app_usage_df['app_name']==i ]['frequency']))
df.drop(i, axis=1, inplace=True)
Column packages_running_android dropped! Frequency: 0.9995948136142626
Column packages_running_com.google.android.inputmethod.latin dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.stk dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.phone dropped! Frequency: 0.9995948136142626
Column packages_running_com.google.android.gsf dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.noisefield dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.nfc dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.systemui dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.launcher dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.providers.settings dropped! Frequency: 0.9995948136142626
Column packages_running_com.google.android.apps.maps dropped! Frequency: 0.9995948136142626
Column packages_running_com.google.android.backup dropped! Frequency: 0.9995948136142626
Column packages_running_com.google.android.syncadapters.contacts dropped! Frequency: 0.9995948136142626
Column packages_running_com.google.android.talk dropped! Frequency: 0.9995948136142626
Column packages_running_com.googlecode.android_scripting dropped! Frequency: 0.9995948136142626
Column packages_running_com.android.providers.telephony dropped! Frequency: 0.9995948136142626
Column packages_running_edu.udo.cs.ess.mobidac.target dropped! Frequency: 0.9979740680713128
Column packages_running_com.google.android.email dropped! Frequency: 0.997163695299838
Column packages_running_com.android.providers.contacts dropped! Frequency: 0.9967585089141004
Column packages_running_com.android.providers.userdictionary dropped! Frequency: 0.9967585089141004
Column packages_running_com.android.providers.applications dropped! Frequency: 0.9967585089141004
Column packages_running_com.android.providers.drm dropped! Frequency: 0.9959481361426256
Column packages_running_com.android.providers.downloads dropped! Frequency: 0.9959481361426256
Column packages_running_com.android.providers.media dropped! Frequency: 0.9959481361426256
Column packages_running_com.android.keychain dropped! Frequency: 0.009724473257698542
Column packages_running_com.android.calculator2 dropped! Frequency: 0.0032414910858995136
Column packages_running_com.android.packageinstaller dropped! Frequency: 0.0004051863857374392
Column packages_running_org.openintents.filemanager dropped! Frequency: 0.0004051863857374392
Column packages_running_stream.android dropped! Frequency: 0.0004051863857374392
Column packages_running_com.google.android.onetimeinitializer dropped! Frequency: 0.0
Column packages_running_com.google.android.videos dropped! Frequency: 0.0
#print(df['battery_level'].value_counts())
batteryLevels = df['battery_level'].value_counts().to_dict()
positiveChange = 0
negativeChange = 0
noChange = 0
for key, value in batteryLevels.items() :
#print("key = ", key, " value: ", value)
if key>0:
positiveChange += value
elif key<0:
negativeChange += value
else:
noChange += value
print("Positive Change = ", positiveChange)
print("Negative Change = ", negativeChange)
print("No Change = ", noChange)
Positive Change = 135
Negative Change = 1029
No Change = 1304
def visualizingAppUsagePerTimeSlot():
slots = df["slot"].unique()
slots.sort()
x_axis = slots
y_axis = []
app_columns = df.columns[5:-3]
for key in slots:
filter = df["slot"]==key
filtered_df = df.where(filter)
filtered_df = filtered_df.dropna()
app_usage_per_slot = 0
for app in app_columns:
#print(app, " frequency in this time slot = ", df2[app].value_counts().to_dict().get(1))
if filtered_df[app].value_counts().to_dict().get(1)!=None:
app_usage_per_slot += filtered_df[app].value_counts().to_dict().get(1)
#print("TOTAL app usage for this slot = ", app_usage_per_slot)
y_axis.append(app_usage_per_slot)
fig = plt.figure(figsize=(30, 10))
ax = fig.gca()
ax.set_xticks(x_axis)
ax.bar(x_axis, y_axis)
ax.set_xlabel("slots")
ax.set_ylabel("total app usages")
plt.show()
visualizingAppUsagePerTimeSlot()
# Drop values with year=1970
df = df[df.year != 1970]
# Print {Year: Frequency}
counts = df['year'].value_counts().to_dict()
for k,v in counts.items():
print("Year:", k, "Frequency:", v)
Year: 2013.0 Frequency: 1278
Year: 2012.0 Frequency: 1189
df['time'] = pd.to_datetime(df[['year','month','day']])
print(df['time'])
0 2012-08-31
1 2012-08-31
2 2012-08-31
3 2012-08-31
4 2012-08-31
...
2464 2013-03-07
2465 2013-03-07
2466 2013-03-07
2467 2013-03-07
2468 2013-03-07
Name: time, Length: 2467, dtype: datetime64[ns]
battery_plugged_gt_0 = df[df['battery_plugged']>0]
battery_plugged_eq_0 = df[df['battery_plugged']==0]
battery_plugged_gt_0_fixed = df.loc[(df['battery_plugged']>0) & (df['battery_status']>0)]
battery_plugged_eq_0_fixed = df.loc[(df['battery_plugged']==0) & (df['battery_status']>0)]
fig, axs = plt.subplots(2,2, figsize=(10, 5))
axs[0,0].hist(battery_plugged_gt_0['battery_level'], rwidth=0.9)
axs[0,0].set_xlabel('battery_level')
axs[0,0].set_ylabel('frequency')
axs[0,0].set_title('battery_plugged>0')
axs[1,0].hist(battery_plugged_eq_0['battery_level'], rwidth=0.9)
axs[1,0].set_xlabel('battery_level')
axs[1,0].set_ylabel('frequency')
axs[1,0].set_title('battery_plugged==0')
axs[0,1].hist(battery_plugged_gt_0_fixed['battery_level'], rwidth=0.9)
axs[0,1].set_xlabel('battery_level')
axs[0,1].set_ylabel('frequency')
axs[0,1].set_title('battery_plugged>0 & battery_status>0')
axs[1,1].hist(battery_plugged_eq_0_fixed['battery_level'], rwidth=0.9)
axs[1,1].set_xlabel('battery_level')
axs[1,1].set_ylabel('frequency')
axs[1,1].set_title('battery_plugged==0 & battery_status>0')
fig.tight_layout()
df_state_prediction = df.drop(columns = ['battery_status','battery_plugged'])
df_state_prediction.to_csv('state_prediction.csv', index=False)
# clean charging data
charging_data_idx = df.loc[(df['battery_plugged']==1) | (df['battery_status']==2) | (df['battery_status']>=4) | (df['battery_level']>0)].index
print("[Charging Data]: Rows to be deleted:", len(charging_data_idx))
df = df.drop(charging_data_idx)
# after this cleaning battery_status and battery_plugged will be meaningless
df = df.drop('battery_status', 1)
df = df.drop('battery_plugged', 1)
[Charging Data]: Rows to be deleted: 209
df.to_csv('clean_android.csv', index=False)
#pd.set_option('display.max_columns', None)
#print("SUM OF NULLS: ", df.isnull().sum(), " --- \n\n")
#print(df.isnull())
#print(df.loc[1520,:])