# openjdk install
!apt-get update
!apt-get install -y unzip openjdk-8-jre-headless xvfb libxi6 libgconf-2-4
# Install Chrome.
!apt-get -y install wget
!wget http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-beta/google-chrome-beta_91.0.4472.27-1_amd64.deb
!apt -y install ./google-chrome-beta_91.0.4472.27-1_amd64.deb
# Install ChromeDriver.
!wget -N https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_linux64.zip -P ~/
!unzip ~/chromedriver_linux64.zip -d ~/
!rm ~/chromedriver_linux64.zip
!mv -f ~/chromedriver /usr/local/bin/chromedriver
!chown root:root /usr/local/bin/chromedriver
!chmod 0755 /usr/local/bin/chromedriver
# Install Selenium
!pip install selenium==3.141.0
!pip install chromedriver-py==91.0.4472.19
!pip install beautifulsoup4
!pip install ipdb
!pip install pystan==2.19.1.1
!pip install prophet
!pip install anvil-uplink
!pip install cvxpy
!pip install cvxopt
!pip install PyPortfolioOpt
from chromedriver_py import binary_path
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import os
import shutil
import pandas as pd
from bs4 import BeautifulSoup
import requests
import time
import glob
import matplotlib.pyplot as plt
import numpy as np
import sys
import plotly.express as px
import plotly.graph_objs as go
from prophet import Prophet
from prophet.plot import plot_plotly, plot_components_plotly
import plotly.tools as tls
import json
filepath = 'data'
dest = 'data'
url_path = 'urls.csv'
mse_a = pd.read_csv(url_path)
mse_a.head()
def get_data():
for url in mse_a['URL']:
chrome_options = webdriver.ChromeOptions()
prefs = {"download.default_directory" : 'data',
"download.prompt_for_download": False,
"download.directory_upgrade": True}
chrome_options.add_experimental_option("prefs",prefs)
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--window-size=1280x1696')
chrome_options.add_argument('--user-data-dir=/tmp/user-data')
chrome_options.add_argument('--hide-scrollbars')
chrome_options.add_argument('--enable-logging')
chrome_options.add_argument('--log-level=0')
chrome_options.add_argument('--v=99')
chrome_options.add_argument('--single-process')
chrome_options.add_argument('--data-path=/tmp/data-path')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--homedir=/tmp')
chrome_options.add_argument('--disk-cache-dir=/tmp/cache-dir')
chrome_options.add_argument('user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36')
driver = webdriver.Chrome(executable_path = binary_path,options=chrome_options)
try:
page_response = requests.get(url, timeout=15)
soup = BeautifulSoup(page_response.content)
symbol = soup.find(id="trade_chart").h1.string.split('(')[-1].split(')')[0]
driver.get(url)
down_arrow = driver.find_element_by_xpath('//*[@id="chartdiv"]/div/div[2]/ul/li/a')
save_as = driver.find_element_by_xpath('//*[@id="chartdiv"]/div/div[2]/ul/li/ul/li[2]/a')
button = driver.find_element_by_xpath('//*[@id="chartdiv"]/div/div[2]/ul/li/ul/li[2]/ul/li[1]/a')
ActionChains(driver).move_to_element(down_arrow).move_to_element(save_as).click(button).perform()
time.sleep(10)
driver.close()
list_of_files = glob.glob(filepath + '/*') # * means all if need specific format then *.csv
latest_file = max(list_of_files, key=os.path.getctime)
os.rename(latest_file, dest + '/' + symbol + '.csv')
time.sleep(10)
except:
print('error on: ' + url)
get_data()
## Momentum Strategy
def buy_sell(data):
sigPriceBuy = []
sigPriceSell = []
flag = -1
for i in range(len(data)):
if data['SMA30'][i] > data['SMA100'][i]:
if flag != 1:
sigPriceBuy.append(data['value'][i])
sigPriceSell.append(np.nan)
flag = 1
else:
sigPriceBuy.append(np.nan)
sigPriceSell.append(np.nan)
elif data['SMA30'][i] < data['SMA100'][i]:
if flag != 0:
sigPriceBuy.append(np.nan)
sigPriceSell.append(data['value'][i])
flag = 0
else:
sigPriceBuy.append(np.nan)
sigPriceSell.append(np.nan)
else:
sigPriceBuy.append(np.nan)
sigPriceSell.append(np.nan)
return (sigPriceBuy, sigPriceSell)
import anvil.server
import anvil.mpl_util
anvil.server.connect("GVJYCJ6HX42BEZG6FR2FQBEN-UB3GQ5B2IA2KQ32O")
def load_data(symbol):
df= pd.read_csv('data/'+symbol+'.csv')
df['SMA30'] = df['value'].rolling(window = 30).mean()
df['SMA100'] = df['value'].rolling(window = 100).mean()
bs = buy_sell(df)
df['Buy_Signal_Price'] = bs[0]
df['Sell_Signal_Price'] = bs[1]
array = df.to_numpy()
date = array[:,0]
value = array[:,1]
SMA30 = array[:,3]
SMA100 = array[:,4]
buy_signal = array[:,5]
sell_signal = array[:,6]
return date, value, SMA30, SMA100, buy_signal, sell_signal
from pypfopt.efficient_frontier import EfficientFrontier
from pypfopt import risk_models
from pypfopt import expected_returns
from pypfopt.discrete_allocation import DiscreteAllocation, get_latest_prices
from functools import reduce
def optimize(symbols,investment):
df_merged = pd.DataFrame(columns = ['date'])
df_merged = df_merged.set_index('date')
for item in symbols:
temp_df = pd.read_csv('data/'+item+'.csv')
temp_df = temp_df.set_index('date')
temp_df = temp_df.drop(columns = {'volume'}).rename(columns = {'value': item})
df_merged = pd.merge(df_merged, temp_df, how='outer', on='date')
#df_merged = df_merged.dropna()
#portfolio:
mu = expected_returns.mean_historical_return(df_merged)
S = risk_models.sample_cov(df_merged)
ef = EfficientFrontier(mu,S)
ef.add_constraint(lambda w: sum(w) == 1)
weights = ef.max_sharpe()
var = ef.portfolio_performance(verbose=True)
#Allocation
latest_prices = get_latest_prices(df_merged)
da = DiscreteAllocation(weights,latest_prices,total_portfolio_value = investment)
allocation,leftover = da.lp_portfolio()
return var,weights, allocation, leftover
a = dict(optimize(['APU','LEND'],20000)[1])
a
b = optimize(['APU','LEND'],20000)[2]
b
def mergeDict(dict1, dict2):
''' Merge dictionaries and keep values of common keys in list'''
dict3 = {**dict1, **dict2}
for key, value in dict3.items():
if key in dict1 and key in dict2:
dict3[key] = {value , dict1[key]}
return dict3
c = mergeDict(a,b)
def prophet_data(symbol):
data = pd.read_csv('data/'+symbol+'.csv')
data['date'] = pd.to_datetime(data['date'])
data = data.drop(columns = ['volume']).rename(columns={"date": "ds", "value": "y"})
data = data[data['ds'].dt.dayofweek < 5]
m = Prophet(interval_width=0.95, weekly_seasonality= False)
m.add_seasonality(name = 'monthly', period=30.5, fourier_order=5)
m.fit(data)
future = m.make_future_dataframe(periods=365)
forecast = m.predict(future)
#the main one
fig = plot_plotly(m,forecast,xlabel = 'Огноо',ylabel = 'Хувьцааны үнэ (₮)', trend =True )
fig1 = fig.to_json()
fig_data = json.loads(fig1)['data']
fig_layout = json.loads(fig1)['layout']
#the seasonality
sea = m.plot_components(forecast).savefig(os.devnull)
return fig1,fig_data, fig_layout, anvil.mpl_util.plot_image()