import scipy import sklearn import numpy as np import pandas as pd from datetime import datetime, timedelta import matplotlib.pyplot as plt import os

from backtester.dataSource.csv_data_source import CsvDataSource

# Load the data cachedFolderName = '/data/' dataSetId = 'trainingData1' startDate = '2017/01/06' endDate = '2017/02/09' instrumentIds = ['MQK'] downloadUrl = 'https://github.com/Auquan/auquan-historical-data/raw/master/qq2Data' ds = CsvDataSource(cachedFolderName='historicalData/', dataSetId=dataSetId, instrumentIds=instrumentIds, downloadUrl = downloadUrl, timeKey = 'datetime', timeStringFormat = '%Y-%m-%d %H:%M:%S', startDateStr=startDate, endDateStr=endDate, liveUpdates=False, pad=True)

ds.getInstrumentIds()

ds.getBookDataFeatures()

#Loading our data def loadData(ds, id): data = ds._bookDataByInstrument[id].getBookData() data['Stock Price'] = data['stockTopBidPrice'] +\ data['stockTopAskPrice'] / 2.0 data['Future Price'] = data['futureTopBidPrice'] +\ data['futureTopAskPrice'] / 2.0 data['Y(Target)'] = data['basis'].shift(-5) del data['benchmark_score'] del data['FairValue'] return data data = loadData(ds,instrumentIds[0])

print(data.head(6))

# Training Data dataSetId = 'trainingData1' ds_training = CsvDataSource(cachedFolderName='historicalData/', dataSetId=dataSetId, instrumentIds=instrumentIds, downloadUrl = downloadUrl, timeKey = 'datetime', timeStringFormat = '%Y-%m-%d %H:%M:%S', liveUpdates=False, pad=True) training_data = loadData(ds_training, instrumentIds[0]) # Validation Data dataSetId = 'trainingData2' ds_validation = CsvDataSource(cachedFolderName='historicalData/', dataSetId=dataSetId, instrumentIds=instrumentIds, downloadUrl = downloadUrl, timeKey = 'datetime', timeStringFormat = '%Y-%m-%d %H:%M:%S', liveUpdates=False, pad=True) validation_data = loadData(ds_validation, instrumentIds[0]) # Test Data dataSetId = 'trainingData3' ds_test = CsvDataSource(cachedFolderName='historicalData/', dataSetId=dataSetId, instrumentIds=instrumentIds, downloadUrl = downloadUrl, timeKey = 'datetime', timeStringFormat = '%Y-%m-%d %H:%M:%S', liveUpdates=False, pad=True) out_of_sample_test_data = loadData(ds_test, instrumentIds[0])

def prepareData(data, period): data['Y(Target)'] = data['basis'].rolling(period).mean().shift(-period) if 'FairValue' in data.columns: del data['FairValue'] data.dropna(inplace=True) period = 5 prepareData(training_data, period) prepareData(validation_data, period) prepareData(out_of_sample_test_data, period)

def difference(dataDf, period): return dataDf.sub(dataDf.shift(period), fill_value=0) def ewm(dataDf, halflife): return dataDf.ewm(halflife=halflife,ignore_na=False,min_periods=0,adjust=True).mean() def rsi(data, period): data_upside = data.sub(data.shift(1), fill_value=0) data_downside = data_upside.copy() data_downside[data_upside > 0] = 0 data_upside[data_upside < 0] = 0 avg_upside = data_upside.rolling(period).mean() avg_downside = - data_downside.rolling(period).mean() rsi = 100 - (100 * avg_downside / (avg_downside + avg_upside)) rsi[avg_downside == 0] = 100 rsi[(avg_downside == 0) & (avg_upside == 0)] = 0 return rsi

def create_features(data): basis_X = pd.DataFrame(index = data.index, columns = [ 'emabasis3','emabasis5','emabasis2','emabasis7','emabasis10',\ 'emabasis4','rsi15','rsi10','rsi5','mom1','mom10',\ 'mom3','mom5']) basis_X['mom1'] = difference(data['basis'],2) basis_X['mom3'] = difference(data['basis'],4) basis_X['mom5'] = difference(data['basis'],6) basis_X['mom10'] = difference(data['basis'],11) basis_X['rsi15'] = rsi(data['basis'],15) basis_X['rsi10'] = rsi(data['basis'],10) basis_X['rsi5'] = rsi(data['basis'],5) basis_X['emabasis2'] = ewm(data['basis'],2) basis_X['emabasis3'] = ewm(data['basis'],3) basis_X['emabasis4'] = ewm(data['basis'],4) basis_X['emabasis5'] = ewm(data['basis'],5) basis_X['emabasis7'] = ewm(data['basis'],7) basis_X['emabasis10'] = ewm(data['basis'],10) basis_X['basis'] = data['basis'] basis_X['vwapbasis'] = data['stockVWAP']-data['futureVWAP'] basis_X['swidth'] = data['stockTopAskPrice']-data['stockTopBidPrice'] basis_X['fwidth'] = data['futureTopAskPrice']-data['futureTopBidPrice'] basis_X['btopask'] = data['stockTopAskPrice']-data['futureTopAskPrice'] basis_X['btopbid'] =data['stockTopBidPrice']-data['futureTopBidPrice'] basis_X['bavgask'] = data['stockAverageAskPrice']-data['futureAverageAskPrice'] basis_X['bavgbid'] = data['stockAverageBidPrice']-data['futureAverageBidPrice'] basis_X['bnextask'] = data['stockNextAskPrice']-data['futureNextAskPrice'] basis_X['bnextbid'] = data['stockNextBidPrice']-data['futureNextBidPrice'] basis_X['topaskvolratio'] = data['stockTopAskVol']/data['futureTopAskVol'] basis_X['topbidvolratio'] = data['stockTopBidVol']/data['futureTopBidVol'] basis_X['totalaskvolratio'] = data['stockTotalAskVol']/data['futureTotalAskVol'] basis_X['totalbidvolratio'] = data['stockTotalBidVol']/data['futureTotalBidVol'] basis_X['nextbidvolratio'] = data['stockNextBidVol']/data['futureNextBidVol'] basis_X['nextaskvolratio'] = data['stockNextAskVol']-data['futureNextAskVol'] basis_X['emabasisdi4'] = basis_X['emabasis7'] - basis_X['emabasis5'] + basis_X['emabasis2'] basis_X['emabasisdi7'] = basis_X['emabasis7'] - basis_X['emabasis5']+ basis_X['emabasis3'] basis_X['emabasisdi1'] = basis_X['emabasis10'] - basis_X['emabasis5'] + basis_X['emabasis3'] basis_X['emabasisdi3'] = basis_X['emabasis10'] - basis_X['emabasis3']+ basis_X['emabasis5'] basis_X['emabasisdi5'] = basis_X['emabasis7']- basis_X['emabasis5'] + data['basis'] basis_X['emabasisdi'] = basis_X['emabasis5'] - basis_X['emabasis3'] + data['basis'] basis_X['emabasisdi6'] = basis_X['emabasis7'] - basis_X['emabasis3']+ data['basis'] basis_X['emabasisdi2'] = basis_X['emabasis10'] - basis_X['emabasis5']+ data['basis'] basis_X['emabasisdi3'] = basis_X['emabasis10'] - basis_X['emabasis3']+ basis_X['emabasis5'] basis_X = basis_X.fillna(0) basis_y = data['Y(Target)'] basis_y.dropna(inplace=True) print("Any null data in y: %s, X: %s"%(basis_y.isnull().values.any(), basis_X.isnull().values.any())) print("Length y: %s, X: %s"%(len(basis_y.index), len(basis_X.index))) return basis_X, basis_y

basis_X_train, basis_y_train = create_features(training_data) basis_X_test, basis_y_test = create_features(validation_data)

from sklearn import linear_model from sklearn.metrics import mean_squared_error, r2_score def linear_regression(basis_X_train, basis_y_train, basis_X_test,basis_y_test): regr = linear_model.LinearRegression() # Train the model using the training sets regr.fit(basis_X_train, basis_y_train) # Make predictions using the testing set basis_y_pred = regr.predict(basis_X_test) # The coefficients print('Coefficients: \n', regr.coef_) # The mean squared error print("Mean squared error: %.2f" % mean_squared_error(basis_y_test, basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test, basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test, color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show() return regr, basis_y_pred

_, basis_y_pred = linear_regression(basis_X_train, basis_y_train, basis_X_test,basis_y_test)

def normalize(basis_X, basis_y, period): basis_X_norm = (basis_X - basis_X.rolling(period).mean())/basis_X.rolling(period).std() basis_X_norm.dropna(inplace=True) basis_y_norm = (basis_y - basis_X['basis'].rolling(period).mean())/basis_X['basis'].rolling(period).std() basis_y_norm = basis_y_norm[basis_X_norm.index] return basis_X_norm, basis_y_norm

norm_period = 375 basis_X_norm_test, basis_y_norm_test = normalize(basis_X_test,basis_y_test, norm_period) basis_X_norm_train, basis_y_norm_train = normalize(basis_X_train, basis_y_train, norm_period) regr_norm, basis_y_pred = linear_regression(basis_X_norm_train, basis_y_norm_train, basis_X_norm_test, basis_y_norm_test)

basis_y_pred = basis_y_pred * basis_X_test['basis'].rolling(period).std()[basis_y_norm_test.index] + basis_X_test['basis'].rolling(period).mean()[basis_y_norm_test.index] print("Mean squared error: %.2f" % mean_squared_error(basis_y_test[basis_y_norm_test.index], basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test[basis_y_norm_test.index], basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test[basis_y_norm_test.index], color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show()

for i in range(len(basis_X_train.columns)): print('%.4f, %s'%(regr_norm.coef_[i], basis_X_train.columns[i]))

import seaborn c = basis_X_train.corr() plt.figure(figsize=(10,10)) seaborn.heatmap(c, cmap='RdYlGn_r', mask = (np.abs(c) <= 0.8)) plt.show()

def create_features_again(data): basis_X = pd.DataFrame(index = data.index, columns = []) basis_X['mom10'] = difference(data['basis'],11) basis_X['emabasis2'] = ewm(data['basis'],2) basis_X['emabasis5'] = ewm(data['basis'],5) basis_X['emabasis10'] = ewm(data['basis'],10) basis_X['basis'] = data['basis'] basis_X['totalaskvolratio'] = (data['stockTotalAskVol']-data['futureTotalAskVol'])/100000 basis_X['totalbidvolratio'] = (data['stockTotalBidVol']-data['futureTotalBidVol'])/100000 basis_X = basis_X.fillna(0) basis_y = data['Y(Target)'] basis_y.dropna(inplace=True) print("Any null data in y: %s, X: %s"%(basis_y.isnull().values.any(), basis_X.isnull().values.any())) print("Length y: %s, X: %s"%(len(basis_y.index), len(basis_X.index))) return basis_X, basis_y

basis_X_test, basis_y_test = create_features_again(validation_data) basis_X_train, basis_y_train = create_features_again(training_data) norm_period = 375 basis_X_norm_test, basis_y_norm_test = normalize(basis_X_test,basis_y_test, norm_period) basis_X_norm_train, basis_y_norm_train = normalize(basis_X_train, basis_y_train, norm_period) regr_norm, basis_y_pred = linear_regression(basis_X_norm_train, basis_y_norm_train, basis_X_norm_test, basis_y_norm_test)

basis_y_pred = basis_y_pred * basis_X_test['basis'].rolling(period).std()[basis_y_norm_test.index] + basis_X_test['basis'].rolling(period).mean()[basis_y_norm_test.index] print("Mean squared error: %.2f" % mean_squared_error(basis_y_test[basis_y_norm_test.index], basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test[basis_y_norm_test.index], basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test[basis_y_norm_test.index], color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show() for i in range(len(basis_X_train.columns)): print(regr_norm.coef_[i], basis_X_train.columns[i])

_, basis_y_pred = linear_regression(basis_X_train, basis_y_train, basis_X_test,basis_y_test) basis_y_regr = basis_y_pred.copy()

from sklearn import neighbors n_neighbors = 5 model = neighbors.KNeighborsRegressor(n_neighbors, weights='distance') model.fit(basis_X_train, basis_y_train) basis_y_pred = model.predict(basis_X_test) # The mean squared error print("Mean squared error: %.2f" % mean_squared_error(basis_y_test, basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test, basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test, color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show() basis_y_knn = basis_y_pred.copy()

from sklearn.svm import SVR model = SVR(kernel='rbf', C=1e3, gamma=0.1) model.fit(basis_X_train, basis_y_train) basis_y_pred = model.predict(basis_X_test) # The mean squared error print("Mean squared error: %.2f" % mean_squared_error(basis_y_test, basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test, basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test, color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show() basis_y_svr = basis_y_pred.copy()

from sklearn import ensemble model=ensemble.ExtraTreesRegressor() model.fit(basis_X_train, basis_y_train) basis_y_pred = model.predict(basis_X_test) # The mean squared error print("Mean squared error: %.2f" % mean_squared_error(basis_y_test, basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test, basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test, color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show() basis_y_trees = basis_y_pred.copy()

basis_y_pred_ensemble = (basis_y_trees + basis_y_svr + +basis_y_knn + basis_y_regr)/4

# The mean squared error print("Mean squared error: %.2f" % mean_squared_error(basis_y_test, basis_y_pred)) # Explained variance score: 1 is perfect prediction print('Variance score: %.2f' % r2_score(basis_y_test, basis_y_pred)) # Plot outputs plt.scatter(basis_y_pred, basis_y_test, color='black') plt.plot(basis_y_test, basis_y_test, color='blue', linewidth=3) plt.xlabel('Y(actual)') plt.ylabel('Y(Predicted)') plt.show()

from backtester.trading_system import TradingSystem from backtester.trading_system_parameters import TradingSystemParameters from backtester.features.feature import Feature from backtester.executionSystem.simple_execution_system_fairvalue import SimpleExecutionSystemWithFairValue from backtester.orderPlacer.backtesting_order_placer import BacktestingOrderPlacer from backtester.version import updateCheck from backtester.constants import * from backtester.timeRule.us_time_rule import USTimeRule from backtester.logger import *

class MyTradingParams(TradingSystemParameters): def __init__(self): super(MyTradingParams, self).__init__() self.count = 0 self.params = {} self.start = '2017/01/06' self.end = '2017/01/10' self.instrumentIds = ['MQK'] ''' Returns an instance of class DataParser. Source of data for instruments ''' def getDataParser(self): dataSetId = 'trainingData3' downloadUrl = 'https://github.com/Auquan/auquan-historical-data/raw/master/qq2Data' z= CsvDataSource(cachedFolderName='historicalData/', dataSetId=dataSetId, instrumentIds=self.instrumentIds, downloadUrl = downloadUrl, timeKey = '', timeStringFormat = '%Y-%m-%d %H:%M:%S', startDateStr=self.start, endDateStr=self.end, liveUpdates=True, pad=True) return z def getTimeRuleForUpdates(self): return USTimeRule(startDate = self.start, endDate = self.end, startTime='9:30', endTime='15:30', frequency='M', sample='1') ''' Return starting capital ''' def getStartingCapital(self): return 10000 ''' This is a way to use any custom features you might have made. ''' def getCustomFeatures(self): return {'prediction': TrainingPredictionFeature} def getInstrumentFeatureConfigDicts(self): predictionDict = {'featureKey': 'prediction', 'featureId': 'prediction', 'params': {}} # ADD RELEVANT FEATURES HERE expma5dic = {'featureKey': 'emabasis5', 'featureId': 'exponential_moving_average', 'params': {'period': 5, 'featureName': 'basis'}} expma10dic = {'featureKey': 'emabasis10', 'featureId': 'exponential_moving_average', 'params': {'period': 10, 'featureName': 'basis'}} expma2dic = {'featureKey': 'emabasis2', 'featureId': 'exponential_moving_average', 'params': {'period': 2, 'featureName': 'basis'}} mom10dic = {'featureKey': 'mom10', 'featureId': 'difference', 'params': {'period': 10, 'featureName': 'basis'}} scoreDict = {'featureKey': 'score', 'featureId': 'score_fv', 'params': {'predictionKey': 'prediction', 'price': 'basis'}} return {INSTRUMENT_TYPE_STOCK: [expma5dic,expma2dic,expma10dic,mom10dic, predictionDict, scoreDict]} ''' Returns an array of market feature config dictionaries ''' def getMarketFeatureConfigDicts(self): # ADD RELEVANT FEATURES HERE scoreDict = {'featureKey': 'score', 'featureId': 'score_fv', 'params': {'featureName': self.getPriceFeatureKey(), 'instrument_score_feature': 'score'}} return [scoreDict] ''' A function that returns your predicted value based on your heuristics. ''' def getPrediction(self, time, updateNum, instrumentManager): predictions = pd.Series(0.0, index = self.instrumentIds) # holder for all the instrument features lbInstF = instrumentManager.getLookbackInstrumentFeatures() ### TODO : FILL THIS FUNCTION TO RETURN A BUY (1) or SELL (0) prediction for each stock ### USE TEMPLATE BELOW AS EXAMPLE # dataframe for a historical instrument feature (mom10 in this case). The index is the timestamps # of upto lookback data points. The columns of this dataframe are the stock symbols/instrumentIds. # Get the last row of the dataframe, the most recent datapoint mom10 = lbInstF.getFeatureDf('mom10').iloc[-1] emabasis2 = lbInstF.getFeatureDf('emabasis2').iloc[-1] emabasis5 = lbInstF.getFeatureDf('emabasis5').iloc[-1] emabasis10 = lbInstF.getFeatureDf('emabasis10').iloc[-1] basis = lbInstF.getFeatureDf('basis').iloc[-1] totalaskvol = (lbInstF.getFeatureDf('stockTotalAskVol').iloc[-1] - lbInstF.getFeatureDf('futureTotalAskVol').iloc[-1])/100000.0 totalbidvol = (lbInstF.getFeatureDf('stockTotalBidVol').iloc[-1] - lbInstF.getFeatureDf('futureTotalBidVol').iloc[-1])/100000.0 coeff = [ 0.03249183, 0.49675487, -0.22289464, 0.2025182, 0.5080227, -0.21557005, 0.17128488] predictions['MQK'] = coeff[0] * mom10['MQK'] + coeff[1] * emabasis2['MQK'] +\ coeff[2] * emabasis5['MQK'] + coeff[3] * emabasis10['MQK'] +\ coeff[4] * basis['MQK'] + coeff[5] * totalaskvol['MQK']+\ coeff[6] * totalbidvol['MQK'] predictions.fillna(emabasis5,inplace=True) print('Current basis: %.3f, predicted basis: %.3f'%(basis['MQK'], predictions['MQK'])) if updateNum>1: print('Current position: %.0f'%lbInstF.getFeatureDf('position').iloc[-1]['MQK']) return predictions ''' Here we convert prediction to intended positions for different instruments. ''' def getExecutionSystem(self): return SimpleExecutionSystemWithFairValue(enter_threshold_deviation=0.5, exit_threshold_deviation=0.2, longLimit=250, shortLimit=250, capitalUsageLimit=0.05, enterlotSize=10, exitlotSize=10, limitType='L', price=self.getPriceFeatureKey()) ''' For Backtesting, we use the BacktestingOrderPlacer, which places the order which we want, and automatically confirms it too. ''' def getOrderPlacer(self): return BacktestingOrderPlacer() ''' Returns the amount of lookback data you want for your calculations. ''' def getLookbackSize(self): return 90 def getPriceFeatureKey(self): return 'basis' def getMetricsToLogRealtime(self): # Everything will be logged if left as is return { 'market': None, 'instruments': None } class TrainingPredictionFeature(Feature): @classmethod def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager): t = MyTradingParams() return t.getPrediction(time, updateNum, instrumentManager)

if updateCheck(): print('Your version of the auquan toolbox package is old. Please update by running the following command:') print('pip install -U auquan_toolbox') else: tsParams = MyTradingParams() # import pdb;pdb.set_trace() tradingSystem = TradingSystem(tsParams) results = tradingSystem.startTrading(onlyAnalyze=False, shouldPlot=False, makeInstrumentCsvs=False) # Set onlyAnalyze to True to quickly generate csv files with all the features # Set onlyAnalyze to False to run a full backtest # Set makeInstrumentCsvs to False to not make instrument specific csvs in runLogs. # This improves the performance BY A LOT

results