Starter Project

import pandas as pd

# read a dataset of movie reviewers into a DataFrame user_cols = ['user_id', 'age', 'gender', 'occupation', 'zip_code'] users = pd.read_table('http://bit.ly/movieusers', sep='|', header=None, names=user_cols, index_col='user_id') users.head()

	ageint64	genderobject	occupationobject	zip_codeobject
1	24	M	technician	85711
2	53	F	other	94043
3	23	M	writer	32067
4	24	M	technician	43537
5	33	F	other	15213
5 rows × 4 columns

users.shape

users.zip_code.duplicated().sum() # Numbers of duplicated zip code

users.duplicated().sum()

users.loc[users.duplicated(keep='first'),:] # keep only the first rows in the duplication

	ageint64	genderobject	occupationobject	zip_codeobject
496	21	F	student	55414
572	51	M	educator	20003
621	17	M	student	60402
684	28	M	student	55414
733	44	F	other	60630
805	27	F	other	20009
890	32	M	student	97301
7 rows × 4 columns

users.loc[users.duplicated(keep='last'),:] # keep only the last rows in the duplication

	ageint64	genderobject	occupationobject	zip_codeobject
67	17	M	student	60402
85	51	M	educator	20003
198	21	F	student	55414
350	32	M	student	97301
428	28	M	student	55414
437	27	F	other	20009
460	44	F	other	60630
7 rows × 4 columns

users.loc[users.duplicated(keep=False),:] #Shows all the duplicated rows

	ageint6417 – 51	genderobject M57.1% F42.9%	occupationobject student57.1% other28.6% educator14.3%	zip_codeobject 5541428.6% 6040214.3% 4 others57.1%
67	17	M	student	60402
85	51	M	educator	20003
198	21	F	student	55414
350	32	M	student	97301
428	28	M	student	55414
Expand rows 5 - 8
621	17	M	student	60402
684	28	M	student	55414
733	44	F	other	60630
805	27	F	other	20009
890	32	M	student	97301
14 rows × 4 columns

users.drop_duplicates(keep='first').shape # drop 7 rows but keep the first version of the duplicate

users.duplicated(subset=['age','zip_code']).sum()

users.loc[users.duplicated(subset=['age','zip_code'],keep=False),:].sort_values('zip_code')

	ageint6417 – 63	genderobject M71.9% F28.1%	occupationobject student34.4% other18.8% 10 others46.9%	zip_codeobject 5541418.8% 018106.3% 12 others75%
777	63	M	programmer	01810
364	63	M	engineer	01810
31	24	M	artist	10003
567	24	M	entertainment	10003
85	51	M	educator	20003
Expand rows 5 - 26
741	25	M	writer	63108
350	32	M	student	97301
890	32	M	student	97301
634	39	M	engineer	T8H1N
74	39	M	scientist	T8H1N
32 rows × 4 columns