Requirement already satisfied: networkx==2.6.3 in /root/venv/lib/python3.7/site-packages (2.6.3)
WARNING: You are using pip version 20.1.1; however, version 21.3.1 is available.
You should consider upgrading via the '/root/venv/bin/python -m pip install --upgrade pip' command.

data = pd.read_parquet('algostrata_isin.parquet') tickers = data.columns.values data_name = pd.read_parquet('algostrata_name.parquet') names = data_name.columns.values

Define time slots for training and testing (FH = First Half, SH = Second Half)

start_FH="2013-01-01" end_FH="2016-12-31" start_SH="2017-01-01" end_SH = "2020-12-31"

df = pd.DataFrame()

dat = __init__(data)

statistics_df_FH, weeklyData_FH = __get_stat(data, start_FH, end_FH) statistics_df_SH, weeklyData_SH = __get_stat(data, start_SH, end_SH) statistics_df, weeklyData = __get_stat(data, start_FH, end_SH)

sharpe_ratio_FH = statistics_df_FH['Sharpe Ratio']

Q1: Sharpe Ratio of the Assets

sharpe_ratio_FH

sharpe_ratio_FH.hist(bins=50);

Strategy 1

Q2: Pick up the top 4-5 assets with the highest Sharpe Ratios for the training set

DecreasingOrd = sharpe_ratio_FH.sort_values(ascending=False) TopFive = DecreasingOrd[0:5].index TopFiveAssetdata = weeklyData_FH[TopFive] TopFiveAssetdata

Q3: Calculate the covariance matrix for these assets.

TopFiveAssetdata.cov()

Q4: Make an equal weight portfolio of these assets and test their performance on your testing data now.

weekly_cov = TopFiveAssetdata.cov() annual_cov = weekly_cov*52 annual_cov = annual_cov.values weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov)) var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data std_equal_weight = math.sqrt(var) print("Std Portfolio (Training Data): ", std_equal_weight) ann_return_portfolio = weights_equal_weights @ (statistics_df_FH.T[TopFiveAssetdata.columns].iloc[0]).values print("Annual Return Portfolio (Training Data): ", ann_return_portfolio) print("Sharpe Ratio (Training Data): ", ann_return_portfolio/std_equal_weight)

Std Portfolio (Training Data):  0.03851472544817102
Annual Return Portfolio (Training Data):  0.1081562868372679
Sharpe Ratio (Training Data):  2.8081801331496705

subset_test = weeklyData_SH[TopFiveAssetdata.columns] weekly_cov = subset_test.cov() annual_cov = weekly_cov*52 annual_cov = annual_cov.values weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov)) var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data std_equal_weight = math.sqrt(var) print("Std Portfolio (Test Data): ", std_equal_weight) ann_return_portfolio = weights_equal_weights @ (statistics_df_SH.T[TopFiveAssetdata.columns].iloc[0]).values print("Annual Return Portfolio (Test Data): ", ann_return_portfolio) print("Sharpe Ratio (Test Data): ", ann_return_portfolio/std_equal_weight)

Std Portfolio (Test Data):  0.05800265411156607
Annual Return Portfolio (Test Data):  0.02994088701938935
Sharpe Ratio (Test Data):  0.5161985684620415

Strategy 2

Q5: Pick up the top 200 assets with the highest Sharpe Ratios for the training set.

DecreasingOrd = sharpe_ratio_FH.sort_values(ascending=False) Top200 = DecreasingOrd[0:200].index Top200Assetdata = weeklyData_FH[Top200] Top200Assetdata

Q6: Run the Minimum Spanning Tree method on this set a number of times until you have only about 4-5 assets left.

# Import MSP from Investmentfunnel import pandas as pd import numpy as np import networkx as nx from sklearn.decomposition import PCA def MinimumSpanningTree(dataset): corr = dataset.corr(method="spearman") # calculate the correlation distance_corr = (2*(1-corr))**0.5 # calculate the distance mask = np.triu(np.ones_like(corr, dtype=np.bool)) # get only the upper half of the matrix distance_corr = distance_corr*mask # use the correlation matrix to create links links = distance_corr.stack().reset_index(level=1) links.columns = ["var2","value"] links = links.reset_index() links = links.replace(0, np.nan) # drop 0 values from the matrix links = links.dropna(how='any', axis=0) links.columns = ["var1", "var2", "value"] # rename the columns links_filtered = links.loc[(links["var1"] != links["var2"])] # filter out self-correlations # Create the graph G = nx.Graph() for i in range(len(corr)): # add nodes G.add_node(corr.index[i]) tuples = list(links_filtered.itertuples(index=False, name=None)) # add edges with weight G.add_weighted_edges_from(tuples) # Create a MST from the full graph mst = nx.minimum_spanning_tree(G) # Save the nodes with degree one degrees = [val for (node, val) in mst.degree()] df = pd.DataFrame(degrees, corr.index) df.columns = ["degree"] subset = df[df["degree"] == 1].index.tolist() # Create a new dataframe with only the assets from the subset subset_df = dataset.loc[:, dataset.columns.isin(subset)] # Calculate the average correlation of the subset corr_subset = subset_df.corr(method="spearman") corr_avg=corr_subset.mean().mean() # Calculate the PDI for the subset pca = PCA() pca.fit(corr_subset) value = 0 for i in range(1, corr_subset.shape[1]): value = value + i*pca.explained_variance_ratio_[i-1] PDI = 2*value - 1 return subset, subset_df, corr_avg, PDI

subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(Top200Assetdata) assets_left = subset_mst_df.shape[1] i = 1 print("Assets left after run 1 : ", assets_left) while assets_left > 4: subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(subset_mst_df) assets_left = subset_mst_df.shape[1] print("Assets left after run", i+1,": ", assets_left) i += 1 print("Number of runs: ", i) print("Assets in the end: ") print(subset_mst_df.columns)

Assets left after run 1 :  105
Assets left after run 2 :  55
Assets left after run 3 :  29
Assets left after run 4 :  12
Assets left after run 5 :  7
Assets left after run 6 :  2
Number of runs:  6
Assets in the end: 
Index(['DK0060158590', 'IE00B2QWCY14'], dtype='object')

After running the MST for 6 times with the condition "assets left > 4", only 2 assets are left. Hence, we re-run it with a condition of "assets left > 7" to stop the code after the fifth run and work with the remaining 7 assets (see output above).

subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(Top200Assetdata) assets_left = subset_mst_df.shape[1] i = 1 print("Assets left after run 1 : ", assets_left) while assets_left > 7: subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(subset_mst_df) assets_left = subset_mst_df.shape[1] print("Assets left after run", i+1,": ", assets_left) i += 1 print("Number of runs: ", i) print("Assets in the end: ") print(subset_mst_df.columns)

Assets left after run 1 :  105
Assets left after run 2 :  55
Assets left after run 3 :  29
Assets left after run 4 :  12
Assets left after run 5 :  7
Number of runs:  5
Assets in the end: 
Index(['DK0016272602', 'DK0060158590', 'DK0010106111', 'IE00B2QWCY14',
       'DK0060188902', 'DK0060498509', 'LU1028171848'],
      dtype='object')

Q7: Calculate the covariance matrix for these assets now and compare with the covariance matrix from strategy 1.

Covariance matrix STRATEGY 1:

TopFiveAssetdata.cov()

Covariance matrix STRATEGY 2:

weekly_cov = subset_mst_df.cov() weekly_cov

OBSERVATIONS:

Q8: Make an equal weight portfolio of these assets and test their performance on your testing data now.

weekly_cov = subset_mst_df.cov() annual_cov = weekly_cov*52 annual_cov = annual_cov.values weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov)) var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data std_equal_weight = math.sqrt(var) print("Std Portfolio (Training Data): ", std_equal_weight) ann_return_portfolio = weights_equal_weights @ (statistics_df_FH.T[subset_mst_df.columns].iloc[0]).values print("Annual Return Portfolio (Training Data): ", ann_return_portfolio) print("Sharpe Ratio (Training Data): ", ann_return_portfolio/std_equal_weight)

Std Portfolio (Training Data):  0.08124555846807419
Annual Return Portfolio (Training Data):  0.15687144976855813
Sharpe Ratio (Training Data):  1.930831084510318

subset_test = weeklyData_SH[subset_mst_df.columns] weekly_cov_test = subset_test.cov() annual_cov_test = weekly_cov_test*52 annual_cov_test = annual_cov_test.values var_test = weights_equal_weights.T @ annual_cov_test @ weights_equal_weights #var portfolio for test data std_test = math.sqrt(var_test) print("Std Portfolio (Test Data): ", std_test) ann_return_portfolio_test = weights_equal_weights @ (statistics_df_SH.T[subset_mst_df.columns].iloc[0]).values print("Annual Return Portfolio (Test Data): ", ann_return_portfolio_test) print("Sharpe Ratio (Test Data): ", ann_return_portfolio_test/std_test)

Std Portfolio (Test Data):  0.10308362222274055
Annual Return Portfolio (Test Data):  0.08006564056056129
Sharpe Ratio (Test Data):  0.7767057349571732

Q9: Calculate average return, std for the two strategies and compare.

STRATEGY 2: Std Portfolio: 0.10308362222274055 Annual Return Portfolio: 0.08006564056056129

STRATEGY 1: Std Portfolio: 0.05800265411156607 Annual Return Portfolio: 0.02994088701938935

Q10: Calculate and compare the Sharpe ratio for the two strategies.

STRATEGY 2: Sharpe Ratio: 0.7767057349571732

STRATEGY 1: Sharpe Ratio: 0.5161985684620415

Q11: Calculate and compare maximum drawdown for the two strategies.

Strategy 3

Q12: Run the Hierarchical Clustering method on this set to make three clusters.

#get 200 assets with the highest sharpe ratio DecreasingOrd = sharpe_ratio_FH.sort_values(ascending=False) Top200 = DecreasingOrd[0:200].index Top200Assetdata = weeklyData_FH[Top200]

#Import Clustering Algorithm from Investmentfunnel from scipy.spatial.distance import squareform from scipy.cluster.hierarchy import dendrogram, fcluster, complete # FUNCTION TO CREATE DENDOGRAM def fancy_dendrogram(*args, **kwargs): max_d = kwargs.pop('max_d', None) if max_d and 'color_threshold' not in kwargs: kwargs['color_threshold'] = max_d annotate_above = kwargs.pop('annotate_above', 0) ddata = dendrogram(*args, **kwargs) if not kwargs.get('no_plot', False): plt.title('Hierarchical Clustering Dendrogram (truncated)') plt.xlabel('sample index or (cluster size)') plt.ylabel('distance') for i, d, c in zip(ddata['icoord'], ddata['dcoord'], ddata['color_list']): x = 0.5 * sum(i[1:3]) y = d[1] if y > annotate_above: plt.plot(x, y, 'o', c=c) plt.annotate("%.3g" % y, (x, y), xytext=(0, -5), textcoords='offset points', va='top', ha='center') if max_d: plt.axhline(y=max_d, c='k') return ddata # FUNCTION TO CLUSTER DATA def Cluster(data, nClusters, dendogram): corr = data.corr(method="spearman") # calculate the correlation distance_corr = 1-corr # distance based on correlation # Person corr distance matrix con_distance_corr = squareform(distance_corr) # condence the distance matrix to be able to fit the hierarcal clustering complete_corr = complete(con_distance_corr) # apply hierarchical clustering using the single distance measure if dendogram == True: # draw the dendogram plt.figure(figsize=(25, 10)) fancy_dendrogram( complete_corr, leaf_rotation=90., # rotates the x axis labels leaf_font_size=8., color_threshold = 0.7,# font size for the x axis labels labels = distance_corr.index, # max_d=0.35, annotate_above=10 ) plt.title('Hierarchical Clustering Dendrogram: Complete Linkage, Spearman Correlation Distance Mearsure', fontsize = 16) plt.xlabel('Assets', fontsize = 16) plt.ylabel('Distance', fontsize = 16) plt.show() # And now we want to save the clustering into a dataframe. # Create the dataframe cluster_df = pd.DataFrame(index=distance_corr.index) # Save the Complete_Corr clustering into the dataframe with 8 clusters cluster_df["Complete_Corr"] = fcluster(complete_corr, nClusters, criterion="maxclust") # Column for plotting for index in cluster_df.index: cluster_df.loc[index,"Cluster"]="Cluster "+str(cluster_df.loc[index, "Complete_Corr"]) return cluster_df # METHOD TO PICK ASSETS FROM A CLUSTER BASED ON PERFORMANCE CRITERIA def pickCluster(data, stat, ML, nAssets): test = pd.concat([stat, ML], axis=1) # For each cluster find the asset with the highest Sharpe ratio ids = [] for clus in test["Cluster"].unique(): # number of elements in each cluster sizeMax = len(test[test["Cluster"]== str(clus)]) # Get indexes if nAssets <= sizeMax: ids.extend(test[test["Cluster"]== str(clus)].nlargest(nAssets, ["Sharpe Ratio"]).index) else: ids.extend(test[test["Cluster"]== str(clus)].nlargest(sizeMax, ["Sharpe Ratio"]).index) print("In "+str(clus)+" was picked only", sizeMax,"Assets") # Get returns result = data[ids] return ids, result

#Run Clustering to get 3 clusters cluster_df = Cluster(Top200Assetdata, 3, True)

#show distribution of clusters cluster_df["Cluster"].hist()

Q13: Pick the asset with the highest Sharpe ratio from the two smaller clusters.

cluster_df["Sharpe Ratio"] = DecreasingOrd cluster_df[cluster_df["Cluster"]!=("Cluster 1")].nlargest(1, columns='Sharpe Ratio')

Q14: Run the Minimum Spanning Three method on the biggest cluster a number of times until you are left with 3-4 assets only.

#get data from the biggest cluster biggest_cluster_ind = cluster_df[cluster_df["Cluster"]==("Cluster 1")].index biggest_cluster_data = weeklyData_FH[biggest_cluster_ind] #dataframe with weekly data from biggest cluster (119 assets)

# Import MSP from Investmentfunnel, you should use this for Strategy 2!!! import pandas as pd import numpy as np import networkx as nx from sklearn.decomposition import PCA def MinimumSpanningTree(dataset): corr = dataset.corr(method="spearman") # calculate the correlation distance_corr = (2*(1-corr))**0.5 # calculate the distance mask = np.triu(np.ones_like(corr, dtype=np.bool)) # get only the upper half of the matrix distance_corr = distance_corr*mask # use the correlation matrix to create links links = distance_corr.stack().reset_index(level=1) links.columns = ["var2","value"] links = links.reset_index() links = links.replace(0, np.nan) # drop 0 values from the matrix links = links.dropna(how='any', axis=0) links.columns = ["var1", "var2", "value"] # rename the columns links_filtered = links.loc[(links["var1"] != links["var2"])] # filter out self-correlations # Create the graph G = nx.Graph() for i in range(len(corr)): # add nodes G.add_node(corr.index[i]) tuples = list(links_filtered.itertuples(index=False, name=None)) # add edges with weight G.add_weighted_edges_from(tuples) # Create a MST from the full graph mst = nx.minimum_spanning_tree(G) # Save the nodes with degree one degrees = [val for (node, val) in mst.degree()] df = pd.DataFrame(degrees, corr.index) df.columns = ["degree"] subset = df[df["degree"] == 1].index.tolist() # Create a new dataframe with only the assets from the subset subset_df = dataset.loc[:, dataset.columns.isin(subset)] # Calculate the average correlation of the subset corr_subset = subset_df.corr(method="spearman") corr_avg=corr_subset.mean().mean() # Calculate the PDI for the subset pca = PCA() pca.fit(corr_subset) value = 0 for i in range(1, corr_subset.shape[1]): value = value + i*pca.explained_variance_ratio_[i-1] PDI = 2*value - 1 return subset, subset_df, corr_avg, PDI

#Code to run MST method until there are only 4 assets left (good to reuse!) subset, subset_df_1, corr_avg, PDI = MinimumSpanningTree(biggest_cluster_data) #just change the input here assets_left = subset_df_1.shape[1] i = 1 print("Assets left after run 1 : ", assets_left) while assets_left > 4: subset, subset_df_1, corr_avg, PDI = MinimumSpanningTree(subset_df_1) assets_left = subset_df_1.shape[1] print("Assets left after run", i+1,": ", assets_left) i += 1 print("Number of runs: ", i) print("Assets in the end: ") print(subset_df_1.columns)

Assets left after run 1 :  58
Assets left after run 2 :  33
Assets left after run 3 :  18
Assets left after run 4 :  11
Assets left after run 5 :  7
Assets left after run 6 :  5
Assets left after run 7 :  4
Number of runs:  7
Assets in the end: 
Index(['DK0016272602', 'DK0060497378', 'DK0060786994', 'LU1028171848'], dtype='object')

Q15: Calculate the covariance matrix for these assets now and compare with the covariance matrix from strategy 1 and 2.

Cov Matrix Strategy 3

weekly_cov_sub_2 = subset_df_1.cov() weekly_cov_sub_2*1000

Cov Matrix Strategy 2

weekly_cov*1000

Cov Matrix Strategy 1

TopFiveAssetdata.cov()*1000

Q16: Make an equal weight portfolio of these assets and test their performance on your testing data now.

# Find return and std for training data weekly_cov = subset_df_1.cov() annual_cov = weekly_cov*52 annual_cov = annual_cov.values weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov)) var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data std_equal_weight = math.sqrt(var) print("Std Portfolio (Training Data): ", std_equal_weight) ann_return_portfolio = weights_equal_weights @ (statistics_df_FH.T[subset_df_1.columns].iloc[0]).values print("Annual Return Portfolio (Training Data): ", ann_return_portfolio) print("Sharpe Ratio (Training Data): ", ann_return_portfolio/std_equal_weight)

Std Portfolio (Training Data):  0.08987856973738308
Annual Return Portfolio (Training Data):  0.1863151926734649
Sharpe Ratio (Training Data):  2.0729657049267782

#Evaluate on test data subset_test = weeklyData_SH[subset_df_1.columns] weekly_cov_test = subset_test.cov() annual_cov_test = weekly_cov_test*52 annual_cov_test = annual_cov_test.values var_test = weights_equal_weights.T @ annual_cov_test @ weights_equal_weights #var portfolio for test data std_test = math.sqrt(var_test) print("Std Portfolio (Test Data): ", std_test) ann_return_portfolio_test = weights_equal_weights @ (statistics_df_SH.T[subset_df_1.columns].iloc[0]).values print("Annual Return Portfolio (Test Data): ", ann_return_portfolio_test) print("Sharpe Ratio (Test Data): ", ann_return_portfolio_test/std_test)

Std Portfolio (Test Data):  0.11456401518379115
Annual Return Portfolio (Test Data):  0.09144362888168733
Sharpe Ratio (Test Data):  0.798188058745911

Q17: Calculate average return, std for the two strategies and compare

STRATEGY 2: Std Portfolio: 0.10308362222274055 Annual Return Portfolio: 0.08006564056056129

STRATEGY 3: Std Portfolio: 0.11456401518379115 Annual Return Portfolio: 0.09144362888168733

Q18: Calculate and compare the Sharpe ratio for the two strategies

STRATEGY 3: Sharpe Ratio: 0.798188058745911

STRATEGY 2: Sharpe Ratio: 0.7767057349571732

Q19: Calculate and compare maximum drawdown for the two strategies

EXTRA QUESTIONS

Q20, 24, 25: Draw histograms of returns for each of the assets, Skewness, Kurtosis (Strategy 2)

# Histograms for all assets # Reference https://www.geeksforgeeks.org/how-to-plot-normal-distribution-over-histogram-in-python/ # mu = mean, std = standard deviation Strategy2_Top7 = ['DK0016272602', 'DK0060158590', 'DK0010106111', 'IE00B2QWCY14', 'DK0060188902', 'DK0060498509', 'LU1028171848'] for n, col in enumerate(weeklyData[Strategy2_Top7]): # Normal distribution mu, std = norm.fit(weeklyData.iloc[0:,n]) # Skewness skeww = skew(weeklyData.iloc[0:,n]) # Kurtosis kurt = kurtosis(weeklyData.iloc[0:,n]) # Plot histogram plt.hist(weeklyData.iloc[0:,n]) # Plot the PDF. xmin, xmax = plt.xlim() x = np.linspace(xmin, xmax, 200) p = norm.pdf(x, mu, std) plt.plot(x, p, 'k', linewidth=2) title = "Fit Values: {:.4f} and {:.4f}. Skew: {:.2f}, Kurtosis: {:.2f}".format(mu, std, skeww, kurt) plt.title(title) plt.show()

Q21: Draw density function of returns for each of the assets (Strategy 2)

import plotly.express as px import plotly.graph_objects as go import plotly.io as pio import matplotlib.pyplot as plt import math from scipy.stats import norm from scipy.stats import skew from scipy.stats import kurtosis from scipy import stats import seaborn as sns import statsmodels.api as sm import pylab

fig_dims = (12, 8) fig, ax = plt.subplots(figsize=fig_dims) # Iterate through the top 7 assets for asset, col in enumerate(Strategy2_Top7): sns.kdeplot(weeklyData[col], shade = True) plt.axis([-0.02, 0.02, 0, 900]) plt.legend(labels=Strategy2_Top7) plt.title("Density function of returns")

Q22: Draw QQ-Plots of returns for each of the assets (Strategy 2)

for asset, col in enumerate(weeklyData[Strategy2_Top7]): sm.qqplot(weeklyData[col], line='s') plt.title("Asset: " + col) pylab.show()

Q23: Perform a Jarque-Bera test for normality of return for each of the assets (Strategy 2)

JarqueBeraResults = [] for asset, col in enumerate(weeklyData[Strategy2_Top7]): temp = stats.jarque_bera(weeklyData[col]) temp JarqueBeraResults = np.append(JarqueBeraResults, temp) # stats.jarque_bera will return two values: # (1) the test statistic and (2) the p-value for the hypothesis test JarqueBeraResults

For assets from strategy 3:

Strategy3_Top4 = ['DK0016272602', 'DK0060497378', 'DK0060786994', 'LU1028171848'] for n, col in enumerate(weeklyData[Strategy3_Top4]): # Normal distribution mu, std = norm.fit(weeklyData.iloc[0:,n]) # Skewness skeww = skew(weeklyData.iloc[0:,n]) # Kurtosis kurt = kurtosis(weeklyData.iloc[0:,n]) # Plot histogram plt.hist(weeklyData.iloc[0:,n]) # Plot the PDF. xmin, xmax = plt.xlim() x = np.linspace(xmin, xmax, 200) p = norm.pdf(x, mu, std) plt.plot(x, p, 'k', linewidth=2) title = "Fit Values: {:.4f} and {:.4f}. Skew: {:.2f}, Kurtosis: {:.2f}".format(mu, std, skeww, kurt) plt.title(title) plt.show()

#fig_dims = (12, 8) fig, ax = plt.subplots(figsize=fig_dims) # Iterate through the top 4 assets for asset, col in enumerate(Strategy3_Top4): sns.kdeplot(weeklyData[col], shade = True) plt.axis([-0.05, 0.05, 0, 100]) plt.legend(labels=Strategy3_Top4) plt.title("Density function of returns")

for asset, col in enumerate(weeklyData[Strategy3_Top4]): sm.qqplot(weeklyData[col], line='s') plt.title("Asset: " + col) pylab.show()

JarqueBeraResults = [] for asset, col in enumerate(weeklyData[Strategy3_Top4]): temp = stats.jarque_bera(weeklyData[col]) temp JarqueBeraResults = np.append(JarqueBeraResults, temp) # stats.jarque_bera will return two values: # (1) the test statistic and (2) the p-value for the hypothesis test JarqueBeraResults