data = pd.read_parquet('algostrata_isin.parquet')
tickers = data.columns.values
data_name = pd.read_parquet('algostrata_name.parquet')
names = data_name.columns.values
Define time slots for training and testing (FH = First Half, SH = Second Half)
start_FH="2013-01-01"
end_FH="2016-12-31"
start_SH="2017-01-01"
end_SH = "2020-12-31"
df = pd.DataFrame()
dat = __init__(data)
statistics_df_FH, weeklyData_FH = __get_stat(data, start_FH, end_FH)
statistics_df_SH, weeklyData_SH = __get_stat(data, start_SH, end_SH)
statistics_df, weeklyData = __get_stat(data, start_FH, end_SH)
sharpe_ratio_FH = statistics_df_FH['Sharpe Ratio']
Q1: Sharpe Ratio of the Assets
sharpe_ratio_FH
sharpe_ratio_FH.hist(bins=50);
Strategy 1
Q2: Pick up the top 4-5 assets with the highest Sharpe Ratios for the training set
DecreasingOrd = sharpe_ratio_FH.sort_values(ascending=False)
TopFive = DecreasingOrd[0:5].index
TopFiveAssetdata = weeklyData_FH[TopFive]
TopFiveAssetdata
Q3: Calculate the covariance matrix for these assets.
TopFiveAssetdata.cov()
Q4: Make an equal weight portfolio of these assets and test their performance on your testing data now.
weekly_cov = TopFiveAssetdata.cov()
annual_cov = weekly_cov*52
annual_cov = annual_cov.values
weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov))
var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data
std_equal_weight = math.sqrt(var)
print("Std Portfolio (Training Data): ", std_equal_weight)
ann_return_portfolio = weights_equal_weights @ (statistics_df_FH.T[TopFiveAssetdata.columns].iloc[0]).values
print("Annual Return Portfolio (Training Data): ", ann_return_portfolio)
print("Sharpe Ratio (Training Data): ", ann_return_portfolio/std_equal_weight)
subset_test = weeklyData_SH[TopFiveAssetdata.columns]
weekly_cov = subset_test.cov()
annual_cov = weekly_cov*52
annual_cov = annual_cov.values
weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov))
var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data
std_equal_weight = math.sqrt(var)
print("Std Portfolio (Test Data): ", std_equal_weight)
ann_return_portfolio = weights_equal_weights @ (statistics_df_SH.T[TopFiveAssetdata.columns].iloc[0]).values
print("Annual Return Portfolio (Test Data): ", ann_return_portfolio)
print("Sharpe Ratio (Test Data): ", ann_return_portfolio/std_equal_weight)
Strategy 2
Q5: Pick up the top 200 assets with the highest Sharpe Ratios for the training set.
DecreasingOrd = sharpe_ratio_FH.sort_values(ascending=False)
Top200 = DecreasingOrd[0:200].index
Top200Assetdata = weeklyData_FH[Top200]
Top200Assetdata
Q6: Run the Minimum Spanning Tree method on this set a number of times until you have only about 4-5 assets left.
# Import MSP from Investmentfunnel
import pandas as pd
import numpy as np
import networkx as nx
from sklearn.decomposition import PCA
def MinimumSpanningTree(dataset):
corr = dataset.corr(method="spearman") # calculate the correlation
distance_corr = (2*(1-corr))**0.5 # calculate the distance
mask = np.triu(np.ones_like(corr, dtype=np.bool)) # get only the upper half of the matrix
distance_corr = distance_corr*mask
# use the correlation matrix to create links
links = distance_corr.stack().reset_index(level=1)
links.columns = ["var2","value"]
links = links.reset_index()
links = links.replace(0, np.nan) # drop 0 values from the matrix
links = links.dropna(how='any', axis=0)
links.columns = ["var1", "var2", "value"] # rename the columns
links_filtered = links.loc[(links["var1"] != links["var2"])] # filter out self-correlations
# Create the graph
G = nx.Graph()
for i in range(len(corr)): # add nodes
G.add_node(corr.index[i])
tuples = list(links_filtered.itertuples(index=False, name=None)) # add edges with weight
G.add_weighted_edges_from(tuples)
# Create a MST from the full graph
mst = nx.minimum_spanning_tree(G)
# Save the nodes with degree one
degrees = [val for (node, val) in mst.degree()]
df = pd.DataFrame(degrees, corr.index)
df.columns = ["degree"]
subset = df[df["degree"] == 1].index.tolist()
# Create a new dataframe with only the assets from the subset
subset_df = dataset.loc[:, dataset.columns.isin(subset)]
# Calculate the average correlation of the subset
corr_subset = subset_df.corr(method="spearman")
corr_avg=corr_subset.mean().mean()
# Calculate the PDI for the subset
pca = PCA()
pca.fit(corr_subset)
value = 0
for i in range(1, corr_subset.shape[1]):
value = value + i*pca.explained_variance_ratio_[i-1]
PDI = 2*value - 1
return subset, subset_df, corr_avg, PDI
subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(Top200Assetdata)
assets_left = subset_mst_df.shape[1]
i = 1
print("Assets left after run 1 : ", assets_left)
while assets_left > 4:
subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(subset_mst_df)
assets_left = subset_mst_df.shape[1]
print("Assets left after run", i+1,": ", assets_left)
i += 1
print("Number of runs: ", i)
print("Assets in the end: ")
print(subset_mst_df.columns)
After running the MST for 6 times with the condition "assets left > 4", only 2 assets are left. Hence, we re-run it with a condition of "assets left > 7" to stop the code after the fifth run and work with the remaining 7 assets (see output above).
subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(Top200Assetdata)
assets_left = subset_mst_df.shape[1]
i = 1
print("Assets left after run 1 : ", assets_left)
while assets_left > 7:
subset, subset_mst_df, corr_avg, PDI = MinimumSpanningTree(subset_mst_df)
assets_left = subset_mst_df.shape[1]
print("Assets left after run", i+1,": ", assets_left)
i += 1
print("Number of runs: ", i)
print("Assets in the end: ")
print(subset_mst_df.columns)
Q7: Calculate the covariance matrix for these assets now and compare with the covariance matrix from strategy 1.
Covariance matrix STRATEGY 1:
TopFiveAssetdata.cov()
Covariance matrix STRATEGY 2:
weekly_cov = subset_mst_df.cov()
weekly_cov
OBSERVATIONS:
Q8: Make an equal weight portfolio of these assets and test their performance on your testing data now.
weekly_cov = subset_mst_df.cov()
annual_cov = weekly_cov*52
annual_cov = annual_cov.values
weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov))
var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data
std_equal_weight = math.sqrt(var)
print("Std Portfolio (Training Data): ", std_equal_weight)
ann_return_portfolio = weights_equal_weights @ (statistics_df_FH.T[subset_mst_df.columns].iloc[0]).values
print("Annual Return Portfolio (Training Data): ", ann_return_portfolio)
print("Sharpe Ratio (Training Data): ", ann_return_portfolio/std_equal_weight)
subset_test = weeklyData_SH[subset_mst_df.columns]
weekly_cov_test = subset_test.cov()
annual_cov_test = weekly_cov_test*52
annual_cov_test = annual_cov_test.values
var_test = weights_equal_weights.T @ annual_cov_test @ weights_equal_weights #var portfolio for test data
std_test = math.sqrt(var_test)
print("Std Portfolio (Test Data): ", std_test)
ann_return_portfolio_test = weights_equal_weights @ (statistics_df_SH.T[subset_mst_df.columns].iloc[0]).values
print("Annual Return Portfolio (Test Data): ", ann_return_portfolio_test)
print("Sharpe Ratio (Test Data): ", ann_return_portfolio_test/std_test)
Q9: Calculate average return, std for the two strategies and compare.
STRATEGY 2: Std Portfolio: 0.10308362222274055 Annual Return Portfolio: 0.08006564056056129
STRATEGY 1: Std Portfolio: 0.05800265411156607 Annual Return Portfolio: 0.02994088701938935
Q10: Calculate and compare the Sharpe ratio for the two strategies.
STRATEGY 2: Sharpe Ratio: 0.7767057349571732
STRATEGY 1: Sharpe Ratio: 0.5161985684620415
Q11: Calculate and compare maximum drawdown for the two strategies.
Strategy 3
Q12: Run the Hierarchical Clustering method on this set to make three clusters.
#get 200 assets with the highest sharpe ratio
DecreasingOrd = sharpe_ratio_FH.sort_values(ascending=False)
Top200 = DecreasingOrd[0:200].index
Top200Assetdata = weeklyData_FH[Top200]
#Import Clustering Algorithm from Investmentfunnel
from scipy.spatial.distance import squareform
from scipy.cluster.hierarchy import dendrogram, fcluster, complete
# FUNCTION TO CREATE DENDOGRAM
def fancy_dendrogram(*args, **kwargs):
max_d = kwargs.pop('max_d', None)
if max_d and 'color_threshold' not in kwargs:
kwargs['color_threshold'] = max_d
annotate_above = kwargs.pop('annotate_above', 0)
ddata = dendrogram(*args, **kwargs)
if not kwargs.get('no_plot', False):
plt.title('Hierarchical Clustering Dendrogram (truncated)')
plt.xlabel('sample index or (cluster size)')
plt.ylabel('distance')
for i, d, c in zip(ddata['icoord'], ddata['dcoord'], ddata['color_list']):
x = 0.5 * sum(i[1:3])
y = d[1]
if y > annotate_above:
plt.plot(x, y, 'o', c=c)
plt.annotate("%.3g" % y, (x, y), xytext=(0, -5),
textcoords='offset points',
va='top', ha='center')
if max_d:
plt.axhline(y=max_d, c='k')
return ddata
# FUNCTION TO CLUSTER DATA
def Cluster(data, nClusters, dendogram):
corr = data.corr(method="spearman") # calculate the correlation
distance_corr = 1-corr # distance based on correlation
# Person corr distance matrix
con_distance_corr = squareform(distance_corr) # condence the distance matrix to be able to fit the hierarcal clustering
complete_corr = complete(con_distance_corr) # apply hierarchical clustering using the single distance measure
if dendogram == True:
# draw the dendogram
plt.figure(figsize=(25, 10))
fancy_dendrogram(
complete_corr,
leaf_rotation=90., # rotates the x axis labels
leaf_font_size=8.,
color_threshold = 0.7,# font size for the x axis labels
labels = distance_corr.index,
# max_d=0.35,
annotate_above=10
)
plt.title('Hierarchical Clustering Dendrogram: Complete Linkage, Spearman Correlation Distance Mearsure', fontsize = 16)
plt.xlabel('Assets', fontsize = 16)
plt.ylabel('Distance', fontsize = 16)
plt.show()
# And now we want to save the clustering into a dataframe.
# Create the dataframe
cluster_df = pd.DataFrame(index=distance_corr.index)
# Save the Complete_Corr clustering into the dataframe with 8 clusters
cluster_df["Complete_Corr"] = fcluster(complete_corr, nClusters, criterion="maxclust")
# Column for plotting
for index in cluster_df.index:
cluster_df.loc[index,"Cluster"]="Cluster "+str(cluster_df.loc[index, "Complete_Corr"])
return cluster_df
# METHOD TO PICK ASSETS FROM A CLUSTER BASED ON PERFORMANCE CRITERIA
def pickCluster(data, stat, ML, nAssets):
test = pd.concat([stat, ML], axis=1)
# For each cluster find the asset with the highest Sharpe ratio
ids = []
for clus in test["Cluster"].unique():
# number of elements in each cluster
sizeMax = len(test[test["Cluster"]== str(clus)])
# Get indexes
if nAssets <= sizeMax:
ids.extend(test[test["Cluster"]== str(clus)].nlargest(nAssets,
["Sharpe Ratio"]).index)
else:
ids.extend(test[test["Cluster"]== str(clus)].nlargest(sizeMax,
["Sharpe Ratio"]).index)
print("In "+str(clus)+" was picked only", sizeMax,"Assets")
# Get returns
result = data[ids]
return ids, result
#Run Clustering to get 3 clusters
cluster_df = Cluster(Top200Assetdata, 3, True)
#show distribution of clusters
cluster_df["Cluster"].hist()
Q13: Pick the asset with the highest Sharpe ratio from the two smaller clusters.
cluster_df["Sharpe Ratio"] = DecreasingOrd
cluster_df[cluster_df["Cluster"]!=("Cluster 1")].nlargest(1, columns='Sharpe Ratio')
Q14: Run the Minimum Spanning Three method on the biggest cluster a number of times until you are left with 3-4 assets only.
#get data from the biggest cluster
biggest_cluster_ind = cluster_df[cluster_df["Cluster"]==("Cluster 1")].index
biggest_cluster_data = weeklyData_FH[biggest_cluster_ind] #dataframe with weekly data from biggest cluster (119 assets)
# Import MSP from Investmentfunnel, you should use this for Strategy 2!!!
import pandas as pd
import numpy as np
import networkx as nx
from sklearn.decomposition import PCA
def MinimumSpanningTree(dataset):
corr = dataset.corr(method="spearman") # calculate the correlation
distance_corr = (2*(1-corr))**0.5 # calculate the distance
mask = np.triu(np.ones_like(corr, dtype=np.bool)) # get only the upper half of the matrix
distance_corr = distance_corr*mask
# use the correlation matrix to create links
links = distance_corr.stack().reset_index(level=1)
links.columns = ["var2","value"]
links = links.reset_index()
links = links.replace(0, np.nan) # drop 0 values from the matrix
links = links.dropna(how='any', axis=0)
links.columns = ["var1", "var2", "value"] # rename the columns
links_filtered = links.loc[(links["var1"] != links["var2"])] # filter out self-correlations
# Create the graph
G = nx.Graph()
for i in range(len(corr)): # add nodes
G.add_node(corr.index[i])
tuples = list(links_filtered.itertuples(index=False, name=None)) # add edges with weight
G.add_weighted_edges_from(tuples)
# Create a MST from the full graph
mst = nx.minimum_spanning_tree(G)
# Save the nodes with degree one
degrees = [val for (node, val) in mst.degree()]
df = pd.DataFrame(degrees, corr.index)
df.columns = ["degree"]
subset = df[df["degree"] == 1].index.tolist()
# Create a new dataframe with only the assets from the subset
subset_df = dataset.loc[:, dataset.columns.isin(subset)]
# Calculate the average correlation of the subset
corr_subset = subset_df.corr(method="spearman")
corr_avg=corr_subset.mean().mean()
# Calculate the PDI for the subset
pca = PCA()
pca.fit(corr_subset)
value = 0
for i in range(1, corr_subset.shape[1]):
value = value + i*pca.explained_variance_ratio_[i-1]
PDI = 2*value - 1
return subset, subset_df, corr_avg, PDI
#Code to run MST method until there are only 4 assets left (good to reuse!)
subset, subset_df_1, corr_avg, PDI = MinimumSpanningTree(biggest_cluster_data) #just change the input here
assets_left = subset_df_1.shape[1]
i = 1
print("Assets left after run 1 : ", assets_left)
while assets_left > 4:
subset, subset_df_1, corr_avg, PDI = MinimumSpanningTree(subset_df_1)
assets_left = subset_df_1.shape[1]
print("Assets left after run", i+1,": ", assets_left)
i += 1
print("Number of runs: ", i)
print("Assets in the end: ")
print(subset_df_1.columns)
Q15: Calculate the covariance matrix for these assets now and compare with the covariance matrix from strategy 1 and 2.
Cov Matrix Strategy 3
weekly_cov_sub_2 = subset_df_1.cov()
weekly_cov_sub_2*1000
Cov Matrix Strategy 2
weekly_cov*1000
Cov Matrix Strategy 1
TopFiveAssetdata.cov()*1000
Q16: Make an equal weight portfolio of these assets and test their performance on your testing data now.
# Find return and std for training data
weekly_cov = subset_df_1.cov()
annual_cov = weekly_cov*52
annual_cov = annual_cov.values
weights_equal_weights = (1/len(annual_cov))*np.ones(len(annual_cov))
var = weights_equal_weights.T @ annual_cov @ weights_equal_weights #var portfolio for training data
std_equal_weight = math.sqrt(var)
print("Std Portfolio (Training Data): ", std_equal_weight)
ann_return_portfolio = weights_equal_weights @ (statistics_df_FH.T[subset_df_1.columns].iloc[0]).values
print("Annual Return Portfolio (Training Data): ", ann_return_portfolio)
print("Sharpe Ratio (Training Data): ", ann_return_portfolio/std_equal_weight)
#Evaluate on test data
subset_test = weeklyData_SH[subset_df_1.columns]
weekly_cov_test = subset_test.cov()
annual_cov_test = weekly_cov_test*52
annual_cov_test = annual_cov_test.values
var_test = weights_equal_weights.T @ annual_cov_test @ weights_equal_weights #var portfolio for test data
std_test = math.sqrt(var_test)
print("Std Portfolio (Test Data): ", std_test)
ann_return_portfolio_test = weights_equal_weights @ (statistics_df_SH.T[subset_df_1.columns].iloc[0]).values
print("Annual Return Portfolio (Test Data): ", ann_return_portfolio_test)
print("Sharpe Ratio (Test Data): ", ann_return_portfolio_test/std_test)
Q17: Calculate average return, std for the two strategies and compare
STRATEGY 2: Std Portfolio: 0.10308362222274055 Annual Return Portfolio: 0.08006564056056129
STRATEGY 3: Std Portfolio: 0.11456401518379115 Annual Return Portfolio: 0.09144362888168733
Q18: Calculate and compare the Sharpe ratio for the two strategies
STRATEGY 3: Sharpe Ratio: 0.798188058745911
STRATEGY 2: Sharpe Ratio: 0.7767057349571732
Q19: Calculate and compare maximum drawdown for the two strategies
EXTRA QUESTIONS
Q20, 24, 25: Draw histograms of returns for each of the assets, Skewness, Kurtosis (Strategy 2)
# Histograms for all assets
# Reference https://www.geeksforgeeks.org/how-to-plot-normal-distribution-over-histogram-in-python/
# mu = mean, std = standard deviation
Strategy2_Top7 = ['DK0016272602', 'DK0060158590', 'DK0010106111', 'IE00B2QWCY14',
'DK0060188902', 'DK0060498509', 'LU1028171848']
for n, col in enumerate(weeklyData[Strategy2_Top7]):
# Normal distribution
mu, std = norm.fit(weeklyData.iloc[0:,n])
# Skewness
skeww = skew(weeklyData.iloc[0:,n])
# Kurtosis
kurt = kurtosis(weeklyData.iloc[0:,n])
# Plot histogram
plt.hist(weeklyData.iloc[0:,n])
# Plot the PDF.
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 200)
p = norm.pdf(x, mu, std)
plt.plot(x, p, 'k', linewidth=2)
title = "Fit Values: {:.4f} and {:.4f}. Skew: {:.2f}, Kurtosis: {:.2f}".format(mu, std, skeww, kurt)
plt.title(title)
plt.show()
Q21: Draw density function of returns for each of the assets (Strategy 2)
import plotly.express as px
import plotly.graph_objects as go
import plotly.io as pio
import matplotlib.pyplot as plt
import math
from scipy.stats import norm
from scipy.stats import skew
from scipy.stats import kurtosis
from scipy import stats
import seaborn as sns
import statsmodels.api as sm
import pylab
fig_dims = (12, 8)
fig, ax = plt.subplots(figsize=fig_dims)
# Iterate through the top 7 assets
for asset, col in enumerate(Strategy2_Top7):
sns.kdeplot(weeklyData[col], shade = True)
plt.axis([-0.02, 0.02, 0, 900])
plt.legend(labels=Strategy2_Top7)
plt.title("Density function of returns")
Q22: Draw QQ-Plots of returns for each of the assets (Strategy 2)
for asset, col in enumerate(weeklyData[Strategy2_Top7]):
sm.qqplot(weeklyData[col], line='s')
plt.title("Asset: " + col)
pylab.show()
Q23: Perform a Jarque-Bera test for normality of return for each of the assets (Strategy 2)
JarqueBeraResults = []
for asset, col in enumerate(weeklyData[Strategy2_Top7]):
temp = stats.jarque_bera(weeklyData[col])
temp
JarqueBeraResults = np.append(JarqueBeraResults, temp)
# stats.jarque_bera will return two values:
# (1) the test statistic and (2) the p-value for the hypothesis test
JarqueBeraResults
For assets from strategy 3:
Strategy3_Top4 = ['DK0016272602', 'DK0060497378', 'DK0060786994', 'LU1028171848']
for n, col in enumerate(weeklyData[Strategy3_Top4]):
# Normal distribution
mu, std = norm.fit(weeklyData.iloc[0:,n])
# Skewness
skeww = skew(weeklyData.iloc[0:,n])
# Kurtosis
kurt = kurtosis(weeklyData.iloc[0:,n])
# Plot histogram
plt.hist(weeklyData.iloc[0:,n])
# Plot the PDF.
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 200)
p = norm.pdf(x, mu, std)
plt.plot(x, p, 'k', linewidth=2)
title = "Fit Values: {:.4f} and {:.4f}. Skew: {:.2f}, Kurtosis: {:.2f}".format(mu, std, skeww, kurt)
plt.title(title)
plt.show()
#fig_dims = (12, 8)
fig, ax = plt.subplots(figsize=fig_dims)
# Iterate through the top 4 assets
for asset, col in enumerate(Strategy3_Top4):
sns.kdeplot(weeklyData[col], shade = True)
plt.axis([-0.05, 0.05, 0, 100])
plt.legend(labels=Strategy3_Top4)
plt.title("Density function of returns")
for asset, col in enumerate(weeklyData[Strategy3_Top4]):
sm.qqplot(weeklyData[col], line='s')
plt.title("Asset: " + col)
pylab.show()
JarqueBeraResults = []
for asset, col in enumerate(weeklyData[Strategy3_Top4]):
temp = stats.jarque_bera(weeklyData[col])
temp
JarqueBeraResults = np.append(JarqueBeraResults, temp)
# stats.jarque_bera will return two values:
# (1) the test statistic and (2) the p-value for the hypothesis test
JarqueBeraResults