from warnings import filterwarnings
filterwarnings('ignore')
import numpy as np
import matplotlib.pyplot as plt
import yfinance as yf
import pandas as pd
import pandas_datareader as pdr
import datetime
data = pd.read_csv('./ind_nifty100list.csv')
#remove entries without values
data.head()
Company Nameobject
Industryobject
0
ACC Ltd.
Construction Materials
1
Adani Enterprises Ltd.
Metals & Mining
2
Adani Green Energy Ltd.
Power
3
Adani Ports and Special Economic Zone Ltd.
Services
4
Adani Transmission Ltd.
Power
#dict from dataframe
companies_dict={}
for i in range(len(data)):
companies_dict[data.iloc[i]['Company Name']]=data.iloc[i]['Symbol']+'.NS'
companies = sorted(companies_dict.items(), key=lambda x: x[1])
#dataframe of name and symbols
companies_df = pd.DataFrame(companies, columns=['Company Name', 'Symbol'])
companies_df.head()
Company Nameobject
Symbolobject
0
ACC Ltd.
ACC.NS
1
Adani Enterprises Ltd.
ADANIENT.NS
2
Adani Green Energy Ltd.
ADANIGREEN.NS
3
Adani Ports and Special Economic Zone Ltd.
ADANIPORTS.NS
4
Adani Transmission Ltd.
ADANITRANS.NS
start_date = '2019-08-01'
end_date = '2020-02-01'
panel_data = pdr.get_data_yahoo(list(companies_dict.values()),
start=start_date,
end=end_date)
panel_data.head()
Adj Close ACC.NSfloat64
Adj Close ADANIENT.NSfloat64
2019-08-01T00:00:00.000000
1463.82312
124.4563446
2019-08-02T00:00:00.000000
1465.900024
127.0853653
2019-08-05T00:00:00.000000
1463.629883
123.7122879
2019-08-06T00:00:00.000000
1496.426514
127.1845703
2019-08-07T00:00:00.000000
1504.154907
124.6051636
stock_close = panel_data.loc[:, 'Close']
stock_open = panel_data.loc[:, 'Open']
Symbols
ACC.NS 1515.300049
ADANIENT.NS 125.449997
ADANIGREEN.NS 45.599998
ADANIPORTS.NS 371.100006
ADANITRANS.NS 203.500000
...
ULTRACEMCO.NS 4231.700195
MCDOWELL-N.NS 575.500000
VEDL.NS 145.449997
WIPRO.NS 270.399994
ZYDUSLIFE.NS 226.300003
Name: 2019-08-01 00:00:00, Length: 95, dtype: float64
stock_close = np.array(stock_close).T
stock_open = np.array(stock_open).T
row, col = stock_close.shape
movements = np.zeros([row, col])
for i in range(0, row):
movements[i, :] = (stock_close[i, :] - stock_open[i, :])/stock_open[i, :]
plt.figure(figsize=(20, 5))
ax1 = plt.subplot(221)
plt.plot(movements[0, :])
plt.title(companies[0][0])
plt.subplot(222, sharey=ax1)
plt.plot(movements[1, :])
plt.title(companies[1][0])
plt.show()
from sklearn.preprocessing import Normalizer
normalise = Normalizer().fit_transform(movements)
plt.figure(figsize=(20, 5))
ax1 = plt.subplot(221)
plt.plot(movements[0, :])
plt.title(companies[0][0])
plt.subplot(222, sharey=ax1)
plt.plot(movements[1, :])
plt.title(companies[1][0])
plt.show()
# machine learning libraries
from sklearn.pipeline import make_pipeline
from sklearn.cluster import KMeans
# define Normalizer
normalizer = Normalizer()
new = normalizer.fit_transform(movements)
# K-Means
inertia = []
for i in range(2,90):
kmeans = KMeans(n_clusters=i, random_state=0, max_iter=1000)
pipeline = make_pipeline(normalizer, kmeans)
pipeline.fit(movements)
inertia.append(kmeans.inertia_)
inertia = np.array(inertia)
# plot of inertia
plt.figure(figsize=(20, 5))
plt.plot(np.arange(2,30), -1*np.diff(inertia,1)[:28])
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()
kmeans = KMeans(n_clusters=12, random_state=0, max_iter=1000)
# pipeline chaining Normalizer and K-Means
pipeline = make_pipeline(normalizer, kmeans)
pipeline.fit(movements)
# predict cluster labels
labels = pipeline.predict(movements)
# dataframe aligning labels and companies
df = pd.DataFrame({'Company': list(companies), 'Cluster': labels})
# df sorted by cluster labels
df_sorted = df.sort_values(by='Cluster')
df_sorted
Companyobject
('Bandhan Bank Ltd.', 'BANDHANBNK.NS')1.1%
('Cipla Ltd.', 'CIPLA.NS')1.1%
93 others97.9%
Clusterint32
0 - 11
13
('Bandhan Bank Ltd.', 'BANDHANBNK.NS')
0
22
('Cipla Ltd.', 'CIPLA.NS')
0
21
('Cholamandalam Investment and Finance Company Ltd.', 'CHOLAFIN.NS')
0
20
('Britannia Industries Ltd.', 'BRITANNIA.NS')
0
35
('HCL Technologies Ltd.', 'HCLTECH.NS')
0
36
('Housing Development Finance Corporation Ltd.', 'HDFC.NS')
0
66
('NMDC Ltd.', 'NMDC.NS')
0
42
('Hindustan Unilever Ltd.', 'HINDUNILVR.NS')
0
76
('Steel Authority of India Ltd.', 'SAIL.NS')
0
12
('Bajaj Finance Ltd.', 'BAJFINANCE.NS')
0
# PCA
from sklearn.decomposition import PCA
# visualize the results
reduced_data = PCA(n_components=2).fit_transform(new)
# run kmeans on reduced data
kmeans = KMeans(n_clusters=12)
kmeans.fit(reduced_data)
labels = kmeans.predict(reduced_data)
# create DataFrame aligning labels & companies
df = pd.DataFrame({'labels': labels, 'companies': companies})
# Display df sorted by cluster labels
df.sort_values('labels')
labelsint32
0 - 11
companiesobject
('Grasim Industries Ltd.', 'GRASIM.NS')1.1%
('Adani Enterprises Ltd.', 'ADANIENT.NS')1.1%
93 others97.9%
33
0
('Grasim Industries Ltd.', 'GRASIM.NS')
1
0
('Adani Enterprises Ltd.', 'ADANIENT.NS')
92
0
('Vedanta Ltd.', 'VEDL.NS')
26
0
("Divi's Laboratories Ltd.", 'DIVISLAB.NS')
24
0
('Colgate Palmolive (India) Ltd.', 'COLPAL.NS')
53
0
('Jubilant Foodworks Ltd.', 'JUBLFOOD.NS')
84
0
('Tata Motors Ltd.', 'TATAMOTORS.NS')
64
0
('Info Edge (India) Ltd.', 'NAUKRI.NS')
67
0
('NTPC Ltd.', 'NTPC.NS')
31
0
('GAIL (India) Ltd.', 'GAIL.NS')
# Define step size of mesh
h = 0.01
# plot the decision boundary
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
# Obtain abels for each point in the mesh using our trained model
Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
Z = Z.reshape(xx.shape)
# define colorplot
cmap = plt.cm.Paired
# plot figure
plt.clf()
plt.figure(figsize=(10, 10))
plt.imshow(Z,
interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
cmap=cmap,
aspect='auto',
origin='lower')
plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.', markersize=5)
# plot the centroid of each cluster as a white X
centroids = kmeans.cluster_centers_
plt.scatter(centroids[:, 0],
centroids[:, 1],
marker='x',
s=169,
linewidth=3,
color='w',
zorder=10)
plt.title('K-Means Clustering on Stock Market Movements (PCA-Reduced Data)')
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.show()
start_date = '2020-02-01'
end_date = '2020-09-01'
panel_data = pdr.get_data_yahoo(list(companies_dict.values()),
start=start_date,
end=end_date)
panel_data.head()
Adj Close ACC.NSfloat64
Adj Close ADANIENT.NSfloat64
2020-02-03T00:00:00.000000
1407.06897
219.1503143
2020-02-04T00:00:00.000000
1436.72583
223.6642761
2020-02-05T00:00:00.000000
1419.868774
234.3787384
2020-02-06T00:00:00.000000
1439.092773
234.6267548
2020-02-07T00:00:00.000000
1451.651001
237.6029968
stock_close = panel_data.loc[:, 'Close']
stock_open = panel_data.loc[:, 'Open']
print(stock_close.iloc[0])
Symbols
ACC.NS 1456.550049
ADANIENT.NS 220.899994
ADANIGREEN.NS 196.149994
ADANIPORTS.NS 367.350006
ADANITRANS.NS 332.049988
...
ULTRACEMCO.NS 4370.649902
MCDOWELL-N.NS 621.700012
VEDL.NS 134.149994
WIPRO.NS 237.399994
ZYDUSLIFE.NS 260.399994
Name: 2020-02-03 00:00:00, Length: 95, dtype: float64
stock_close = np.array(stock_close).T
stock_open = np.array(stock_open).T
row, col = stock_close.shape
movements = np.zeros([row, col])
for i in range(0, row):
movements[i, :] = (stock_close[i, :] - stock_open[i, :])/stock_open[i, :]
print('Company', '\t', 'Movement')
for i in range(0, len(companies)):
print(companies[i][0], '\t\t', sum(movements[i, :]), '\n')
Company Movement
ACC Ltd. -0.06448828484892541
Adani Enterprises Ltd. 0.4037813182277168
Adani Green Energy Ltd. 0.8800030862103004
Adani Ports and Special Economic Zone Ltd. 0.07924550501613442
Adani Transmission Ltd. -0.26580308110554185
Ambuja Cements Ltd. 0.055473388689200966
Apollo Hospitals Enterprise Ltd. -0.11758214869418836
Asian Paints Ltd. -0.018871049666451544
Axis Bank Ltd. 0.22151711444850816
Bajaj Auto Ltd. -0.5769591639133744
Bajaj Finserv Ltd. -0.15814295242615586
Bajaj Holdings & Investment Ltd. -0.25192717371181284
Bajaj Finance Ltd. -0.5112328113420949
Bandhan Bank Ltd. -0.8296548232595921
Bank of Baroda -0.6322434432608022
Berger Paints India Ltd. -0.5771827929403721
Bharti Airtel Ltd. -0.05237431374230748
Biocon Ltd. -0.10509141353719367
Bosch Ltd. -0.0707985580132039
Bharat Petroleum Corporation Ltd. 0.05419759595372875
Britannia Industries Ltd. -0.26963417354205876
Cholamandalam Investment and Finance Company Ltd. -0.043642419448363896
Cipla Ltd. -0.521864435682198
Coal India Ltd. -0.21716042164636803
Colgate Palmolive (India) Ltd. -0.2842981555062249
Dabur India Ltd. -0.18763890498182098
Divi's Laboratories Ltd. -0.28903130944546546
DLF Ltd. -0.1942247922575091
Avenue Supermarts Ltd. 0.24862076863962695
Dr. Reddy's Laboratories Ltd. 0.12369639877305107
Eicher Motors Ltd. -0.16084905882746925
GAIL (India) Ltd. -0.5043220282982964
Godrej Consumer Products Ltd. -0.3476449710651231
Grasim Industries Ltd. -0.07564624140441069
Havells India Ltd. 0.16185968934234157
HCL Technologies Ltd. 0.022407539566903216
Housing Development Finance Corporation Ltd. -0.3145334953559769
HDFC Asset Management Company Ltd. -7.061556760914556e-05
HDFC Bank Ltd. 0.035519893827917
HDFC Life Insurance Company Ltd. 0.16045233201970185
Hero MotoCorp Ltd. -0.10409619865102729
Hindalco Industries Ltd. -0.14618324032856506
Hindustan Unilever Ltd. -0.2592946359236916
ICICI Bank Ltd. -0.32661681367768797
ICICI Lombard General Insurance Company Ltd. 0.07046218814478643
ICICI Prudential Life Insurance Company Ltd. 0.08079902944708903
InterGlobe Aviation Ltd. -0.5167926675751247
IndusInd Bank Ltd. -0.4339856794638171
Indus Towers Ltd. -0.48544947038656977
Infosys Ltd. -0.7860024913827969
Indian Oil Corporation Ltd. 0.26642816966276206
ITC Ltd. 0.23622694579220482
JSW Steel Ltd. 0.8145280744324728
Jubilant Foodworks Ltd. 0.2786895813349867
Kotak Mahindra Bank Ltd. 0.8019281175026826
Larsen & Toubro Ltd. -0.18018637413434455
Larsen & Toubro Infotech Ltd. 0.43433588281793284
Lupin Ltd. -0.6039598905267752
Mahindra & Mahindra Ltd. 0.08551371279039627
Marico Ltd. 0.2904067337504171
Maruti Suzuki India Ltd. 0.009688005676742412
United Spirits Ltd. 0.12221830374024979
MindTree Ltd. 0.6498594822919123
Muthoot Finance Ltd. 0.4886465146887329
Info Edge (India) Ltd. -0.14435791377237864
Nestle India Ltd. -0.3124090692682812
NMDC Ltd. -0.1611149987508502
NTPC Ltd. -0.20512475052017826
Oil & Natural Gas Corporation Ltd. 0.24464199645279575
Piramal Enterprises Ltd. -0.2995024375960463
Procter & Gamble Hygiene & Health Care Ltd. -0.08135196265205191
Pidilite Industries Ltd. 0.07926295592135557
PI Industries Ltd. -0.22026020467281973
Punjab National Bank -0.70223229723874
Power Grid Corporation of India Ltd. 0.11264845506814554
Reliance Industries Ltd. -0.1853963822600271
Steel Authority of India Ltd. 0.058087557293847326
SBI Life Insurance Company Ltd. 0.004373640699424627
State Bank of India -0.3704296393571581
Shree Cement Ltd. -0.5356537778247719
Siemens Ltd. -0.1691448846519206
SRF Ltd. -0.21998651550119966
Sun Pharmaceutical Industries Ltd. -0.013001265821466237
Tata Consumer Products Ltd. 0.12381984830075389
Tata Motors Ltd. -0.18549826768330677
Tata Steel Ltd. 0.12004436025871938
Tata Consultancy Services Ltd. -0.045292959236934514
Tech Mahindra Ltd. 0.05497597310226058
Titan Company Ltd. 0.06301293885457973
Torrent Pharmaceuticals Ltd. -0.44690460559073814
UltraTech Cement Ltd. -0.1721341255815417
UPL Ltd. -0.2873289002257844
Vedanta Ltd. -0.14360853848211755
Wipro Ltd. 0.007645404450356723
Zydus Lifesciences Ltd. -0.24409576090149143
plt.figure(figsize=(20, 5))
ax1 = plt.subplot(221)
plt.plot(movements[0, :])
plt.title(companies[0][0])
plt.subplot(222, sharey=ax1)
plt.plot(movements[1, :])
plt.title(companies[1][0])
plt.show()
normalise = Normalizer().fit_transform(movements)
plt.figure(figsize=(20, 5))
ax1 = plt.subplot(221)
plt.plot(movements[0, :])
plt.title(companies[0][0])
plt.subplot(222, sharey=ax1)
plt.plot(movements[1, :])
plt.title(companies[1][0])
plt.show()
# define Normalizer
normalizer = Normalizer()
new = normalizer.fit_transform(movements)
# K-Means
inertia = []
for i in range(2,90):
kmeans = KMeans(n_clusters=i, random_state=0, max_iter=1000)
pipeline = make_pipeline(normalizer, kmeans)
pipeline.fit(movements)
inertia.append(kmeans.inertia_)
inertia = np.array(inertia)
np.diff(inertia,1)
# plot inertia
plt.figure(figsize=(20, 5))
plt.plot(np.arange(2,30), -1*np.diff(inertia,1)[:28])
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()
kmeans = KMeans(n_clusters=10, random_state=0, max_iter=1000)
# pipeline chaining Normalizer and K-Means
pipeline = make_pipeline(normalizer, kmeans)
pipeline.fit(movements)
print(kmeans.inertia_)
45.83706733669879
# predict cluster labels
labels = pipeline.predict(movements)
# dataframe aligning labels and companies
df = pd.DataFrame({'Company': list(companies), 'Cluster': labels})
# df sorted by cluster labels
df_sorted = df.sort_values(by='Cluster')
df_sorted
Companyobject
('Adani Green Energy Ltd.', 'ADANIGREEN.NS')1.1%
('Adani Transmission Ltd.', 'ADANITRANS.NS')1.1%
93 others97.9%
Clusterint32
0 - 9
68
('Oil & Natural Gas Corporation Ltd.', 'ONGC.NS')
9
16
('Bharti Airtel Ltd.', 'BHARTIARTL.NS')
9
83
('Tata Consumer Products Ltd.', 'TATACONSUM.NS')
9
62
('MindTree Ltd.', 'MINDTREE.NS')
9
54
('Kotak Mahindra Bank Ltd.', 'KOTAKBANK.NS')
9
# visualize the results
reduced_data = PCA(n_components=2).fit_transform(new)
# run kmeans on reduced data
kmeans = KMeans(n_clusters=10)
kmeans.fit(reduced_data)
labels = kmeans.predict(reduced_data)
# create DataFrame aligning labels & companies
df = pd.DataFrame({'labels': labels, 'companies': companies})
# Display df sorted by cluster labels
df.sort_values('labels')
labelsint32
0 - 9
companiesobject
('DLF Ltd.', 'DLF.NS')1.1%
("Dr. Reddy's Laboratories Ltd.", 'DRREDDY.NS')1.1%
93 others97.9%
47
8
('IndusInd Bank Ltd.', 'INDUSINDBK.NS')
66
9
('NMDC Ltd.', 'NMDC.NS')
93
9
('Wipro Ltd.', 'WIPRO.NS')
41
9
('Hindalco Industries Ltd.', 'HINDALCO.NS')
21
9
('Cholamandalam Investment and Finance Company Ltd.', 'CHOLAFIN.NS')
# Define step size of mesh
h = 0.01
# plot the decision boundary
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
# Obtain abels for each point in the mesh using our trained model
Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
Z = Z.reshape(xx.shape)
# define colorplot
cmap = plt.cm.Paired
# plot figure
plt.clf()
plt.figure(figsize=(10, 10))
plt.imshow(Z,
interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
cmap=cmap,
aspect='auto',
origin='lower')
plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.', markersize=5)
# plot the centroid of each cluster as a white X
centroids = kmeans.cluster_centers_
plt.scatter(centroids[:, 0],
centroids[:, 1],
marker='x',
s=169,
linewidth=3,
color='w',
zorder=10)
plt.title('K-Means Clustering on Stock Market Movements (PCA-Reduced Data)')
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.show()