#from src.utils.clusters import elbowmethod #from src.utils.clusters import construct_model #from src.utils.clusters import countrymatch from tslearn.metrics import soft_dtw from tslearn.clustering import TimeSeriesKMeans from tslearn.clustering import silhouette_score import seaborn as sns import matplotlib.pyplot as plt import pandas as pd def elbowmethod(df:pd.DataFrame): """ Function that finds the best number of clusters and plots a graph of the distortion with the number of clusters Args: pd.DataFrame: Columns are (country(index), year_week, value) Returns: int: Best number of cluster """ import numpy as np from tslearn.clustering import silhouette_score import matplotlib.pyplot as plt distortions = [] inertias = [] mapping1 = {} mapping2 = {} K=range(2,df.shape[0]) for k in K: # Building and fitting the model model = TimeSeriesKMeans(n_clusters=k, metric="softdtw", max_iter=50) model.fit(df.values[...,np.newaxis]) distortions.append(silhouette_score(df, model.labels_, metric="softdtw")) plt.plot(K, distortions, 'bx-') plt.xlabel('Values of K') plt.ylabel('Distortion') plt.title('The Elbow Method using Distortion') plt.show() best_num_cluster = np.argmax(distortions)+2 return best_num_cluster def construct_model(best_num_cluster:int,df:pd.DataFrame): """ Function that constructs the model, tests the accuracy of the model and produces cluster plots using the model Args: pd.DataFrame: Columns are (country(index), year_week, value) int: Best number of cluster Returns: model """ import numpy as np from tslearn.clustering import silhouette_score import matplotlib.pyplot as plt model = TimeSeriesKMeans(n_clusters=best_num_cluster, metric="softdtw", max_iter=50) #### # Test whether the clusters are stable or not model1 = TimeSeriesKMeans(n_clusters=best_num_cluster, metric="softdtw", max_iter=50) model1.fit(df.values[...,np.newaxis]) # we need the output to be visualized model2 = TimeSeriesKMeans(n_clusters=best_num_cluster, metric="softdtw", max_iter=50) model2.fit(df.values[...,np.newaxis]) # we need the output to be visualized print(normalized_mutual_info_score(model1.labels_, model2.labels_)) # scale the results between 0 (no mutual information) and 1 (perfect correlation) ###### Plotting the model ###### model.fit(df.values[...,np.newaxis]) plt.figure() sz = df.shape[1] ylim =df.values.max() for yi in range(best_num_cluster): plt.subplot(best_num_cluster, best_num_cluster , yi + 1) #for xx in X_train[y_pred == yi]: #plt.plot(xx.ravel(), "k-", alpha=.2) plt.plot(model.cluster_centers_[yi].ravel(), "r-") plt.xlim(0, sz) plt.ylim(0, ylim) plt.text(0.55, 0.85,'Cluster %d' % (yi + 1), transform=plt.gca().transAxes) if yi == 1: plt.title("DTW $k$-means") return model def countrymatch(df:pd.DataFrame): """ Function that prints the countries in the correponding clusters Args: pd.DataFrame: Columns are (country(index), year_week, value) """ import numpy as np from tslearn.clustering import silhouette_score import matplotlib.pyplot as plt mapping_country_cluster = {} mapping_cluster_country = {} for i in range(df.shape[0]): mapping_country_cluster[df.index[i]] = model.labels_[i] if model.labels_[i] in mapping_cluster_country: mapping_cluster_country[model.labels_[i]].append(df.index[i]) else: mapping_cluster_country[model.labels_[i]] = [df.index[i]] print(mapping_cluster_country) return mapping_cluster_country dataframe = load_data('agerangenotificationeu').drop(columns='source').dropna() print(dataframe) #print(dataframe["age_group"].unique()) age_groups=['<15yr' '15-24yr' '25-49yr' '50-64yr' '65-79yr' '80+yr'] # Want to have dataframe in the form of [year_week, '<15yr' '15-24yr' '25-49yr' '50-64yr' '65-79yr' '80+yr'] lessthan15yr=dataframe[dataframe["age_group"]=='<15yr'].drop(columns=['age_group']).rename(columns={'new_cases': '<15yr','rate_14_day_per_100k': '<15yr_rates'}) fifteento24yr=dataframe[dataframe["age_group"]=='15-24yr'].drop(columns=['age_group']).rename(columns={'new_cases': '15-24yr','rate_14_day_per_100k': '15-24yr_rates'}) twentyfiveto49yr=dataframe[dataframe["age_group"]=='25-49yr'].drop(columns=['age_group']).rename(columns={'new_cases': '25-49yr','rate_14_day_per_100k': '25-49yr_rates'}) fiftyto64yr=dataframe[dataframe["age_group"]=='50-64yr'].drop(columns=['age_group']).rename(columns={'new_cases': '50-64yr','rate_14_day_per_100k': '50-64yr_rates'}) sixtyfiveto79yr=dataframe[dataframe["age_group"]=='65-79yr'].drop(columns=['age_group']).rename(columns={'new_cases': '65-79yr','rate_14_day_per_100k': '65-79yr_rates'}) eightyplus=dataframe[dataframe["age_group"]=='80+yr'].drop(columns=['age_group']).rename(columns={'new_cases': '80+yr','rate_14_day_per_100k': '80+yr_rates'}) data_frames = [lessthan15yr, fifteento24yr, twentyfiveto49yr,fiftyto64yr, sixtyfiveto79yr,eightyplus ] df_merged = reduce(lambda left,right: pd.merge(left,right,on=['year_week',"country","country_code"], how='outer'), data_frames) print(df_merged) ############################################################################################################################################################ #Visualizing data df_mergedlessthan15=df_merged[['year_week','country','<15yr']] df_mergedlessthan15=df_mergedlessthan15.dropna() df_mergedlessthan15 = df_mergedlessthan15.pivot(index='year_week', columns='country', values='<15yr') df_mergedlessthan15.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('lessthan15yrscases') plt.show() df_merged1524yr=df_merged[['year_week','country','15-24yr']] df_merged1524yr=df_merged1524yr.dropna() df_merged1524yr = df_merged1524yr.pivot(index='year_week', columns='country', values='15-24yr') df_merged1524yr.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('15-24yrscases') plt.show() df_merged2549yr=df_merged[['year_week','country','25-49yr']] df_merged2549yr=df_merged2549yr.dropna() df_merged2549yr = df_merged2549yr.pivot(index='year_week', columns='country', values='25-49yr') df_merged2549yr.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('25-49yrscases') plt.show() df_merged5064=df_merged[['year_week','country','50-64yr']] df_merged5064=df_merged5064.dropna() df_merged5064 = df_merged5064.pivot(index='year_week', columns='country', values='50-64yr') df_merged5064.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('50-64yrscases') plt.show() df_merged6579=df_merged[['year_week','country','65-79yr']] df_merged6579=df_merged6579.dropna() df_merged6579 = df_merged6579.pivot(index='year_week', columns='country', values='65-79yr') df_merged6579.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('65-79yrscases') plt.show() df_merged80yr=df_merged[['year_week','country','80+yr']] df_merged80yr=df_merged80yr.dropna() df_merged80yr = df_merged80yr.pivot(index='year_week', columns='country', values='80+yr') df_merged80yr.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('80+yrscases') plt.show() # Heatmap representing the number of cases per age group df=df_merged[['year_week','country','<15yr', '15-24yr', '25-49yr' ,'50-64yr' ,'65-79yr', '80+yr']] print(df) df=df.dropna() #df=df_merged[['<15yr', '15-24yr', '25-49yr' ,'50-64yr' ,'65-79yr', '80+yr']] #df=sns.heatmap(df,linewidths=.5,cmap="YlGnBu") df1 = df.pivot(index='year_week', columns='country', values=['<15yr', '15-24yr', '25-49yr' ]) df1=sns.heatmap(df1,linewidths=.5,cmap="twilight_r") plt.savefig('agesheatmap1') plt.show() df2 = df.pivot(index='year_week', columns='country', values=['50-64yr' ,'65-79yr', '80+yr']) df2=sns.heatmap(df2,linewidths=.5,cmap="twilight_r") plt.savefig('agesheatmap2') plt.show() ############################################################################################################################################################ df_mergedlessthan15=df_merged[['year_week','country','<15yr_rates']] df_mergedlessthan15=df_mergedlessthan15.dropna() df_mergedlessthan15 = df_mergedlessthan15.pivot(index='country', columns='year_week', values='<15yr_rates') df_mergedlessthan15 = df_mergedlessthan15.fillna(0) df_mergedlessthan15.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('<15yr_rates') plt.show() best_num_cluster=elbowmethod(df_mergedlessthan15) model_lessthan15=construct_model(best_num_cluster,df_mergedlessthan15) countrym=countrymatch(df_mergedlessthan15) df_merged1524yr=df_merged[['year_week','country','15-24yr_rates']] df_merged1524yr=df_merged1524yr.dropna() df_merged1524yr = df_merged1524yr.pivot(index='country', columns='year_week', values='15-24yr_rates') df_merged1524yr = df_merged1524yr.fillna(0) df_merged1524yr.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('15-24yr_rates') plt.show() best_num_cluster=elbowmethod(df_merged1524yr) model1524yr=construct_model(best_num_cluster,df_merged1524yr) countrym=countrymatch(df_merged1524yr) df_merged2549yr=df_merged[['year_week','country','25-49yr_rates']] df_merged2549yr=df_merged2549yr.dropna() df_merged2549yr = df_merged2549yr.pivot(index='country', columns='year_week', values='25-49yr_rates') df_merged2549yr = df_merged2549yr.fillna(0) df_merged2549yr.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('25-49yr_rates') plt.show() best_num_cluster=elbowmethod(df_merged2549yr) model2549yr=construct_model(best_num_cluster,df_merged2549yr) countrym=countrymatch(df_merged2549yr) df_merged5064=df_merged[['year_week','country','50-64yr_rates']] df_merged5064=df_merged5064.dropna() df_merged5064 = df_merged5064.pivot(index='country', columns='year_week', values='50-64yr_rates') df_merged5064 = df_merged5064.fillna(0) df_merged5064.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('50-64yr_rates') plt.show() best_num_cluster=elbowmethod(df_merged5064) model5064=construct_model(best_num_cluster,df_merged5064) countrym=countrymatch(df_merged5064) df_merged6579=df_merged[['year_week','country','65-79yr_rates']] df_merged6579=df_merged6579.dropna() df_merged6579 = df_merged6579.pivot(index='country', columns='year_week', values='65-79yr_rates') df_merged6579 = df_merged6579.fillna(0) df_merged6579.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('65-79yr_rates') plt.show() best_num_cluster=elbowmethod(df_merged6579) model6579=construct_model(best_num_cluster,df_merged6579) countrym=countrymatch(df_merged6579) df_merged80yr=df_merged[['year_week','country','80+yr_rates']] df_merged80yr=df_merged80yr.dropna() df_merged80yr = df_merged80yr.pivot(index='country', columns='year_week', values='80+yr_rates') df_merged80yr = df_merged80yr.fillna(0) df_merged80yr.plot() plt.legend(loc=0, prop={'size': 2}) plt.savefig('80+yr_rates') plt.show() best_num_cluster=elbowmethod(df_merged80yr) model80yr=construct_model(best_num_cluster,df_merged80yr) countrym=countrymatch(df_merged80yr) #################### Lucy noob is stuck on plotting the corr heatmap. #################### # I want df to be in following shape # <15yr... 15-24yr ... 80+yr <15yr... 15-24yr ... 80+yr # US ... US Australia Australia #year_week