from scipy.io import loadmat import pandas as pd import numpy as np import matplotlib.pyplot as plt

oneDPoly = loadmat('1D_poly.mat') x_train = oneDPoly['x_train'][0] y_train = np.array([y[0] for y in oneDPoly['y_train']]) y_fresh = np.array([y[0] for y in oneDPoly['y_fresh']])

def main(): n = len(x_train) #random estimate of max n used errors = np.zeros(n-1) for d in range(n -1): # Formulating the x data D = d + 1 for i in range(D + 1): if i == 0: Xf = np.array([1] * len(x_train)) #first column of 1's else: Xf = np.vstack([np.power(x_train, i), Xf]) #succeeding columns of xn**D Xf = Xf.T #convert to column # Using least squares to train and create the weights #w = lstsq(Xf, y_train) w = np.linalg.solve(Xf.T @ Xf, Xf.T @ y_train) # predicting y using weights (trained on those y's as well though) y_pred = Xf @ w # measuring training error errors[d] = (np.linalg.norm(y_train - y_pred)**2) / n plt.plot(range(1,n-1),errors[:n-2]) plt.xlabel('Polynomial Degree') plt.ylabel('Training Error') plt.show() if __name__ == "__main__": main()

def main(): n = len(x_train) #random estimate of max n used errors_train = np.zeros(n-1) errors_fresh = np.zeros(n-1) for d in range(n -1): # Formulating the x data D = d + 1 for i in range(D + 1): if i == 0: Xf = np.array([1] * len(x_train)) #first column of 1's else: Xf = np.vstack([np.power(x_train, i), Xf]) #succeeding columns of xn**D Xf = Xf.T #convert to column # Using least squares to train and create the weights #w = lstsq(Xf, y_train) w = np.linalg.solve(Xf.T @ Xf, Xf.T @ y_train) # predicting y using weights (trained on those y's as well though) y_pred = Xf @ w # measuring training error errors_train[d] = (np.linalg.norm(y_train - y_pred)**2) / n # measuring training error errors_fresh[d] = (np.linalg.norm(y_fresh - y_pred)**2) / n plt.figure() plt.ylim([0, 6]) plt.plot(errors_train[:n-2], label='train') plt.plot(errors_fresh[:n-2], label='fresh') plt.legend() plt.show() #Zoomed in plot plt.figure() plt.ylim([0, 6]) plt.xlim([2, 6]) plt.plot(errors_train, label='train') plt.plot(errors_fresh, label='fresh') plt.legend() plt.show() if __name__ == "__main__": main()

data = loadmat('polynomial_regression_samples.mat', squeeze_me=True) x_attributes = data['x'] y_flyingtime = data['y'] folds = 4 Dcount = 5 Lambda = 0.1 def assemble_feature(x, D): ''' x should be an Nx5 dimensional numpy array, where N is the number of data points D is the maximum degree of the multivariate polynomial ''' n_feature = x.shape[1] Q = [(np.ones(x.shape[0]), 0, 0)] i = 0 while Q[i][1] < D: cx, degree, last_index = Q[i] for j in range(last_index, n_feature): Q.append((cx * x[:, j], degree + 1, j)) i += 1 return np.column_stack([q[0] for q in Q]) def fit(D, Lambda): N_train = int((len(x_attributes)*(folds-1))/folds) N_valid = int(len(x_attributes)-N_train) Errors_train, Errors_valid = np.zeros(folds), np.zeros(folds) for fold in range(4): valid_x = currattribute_x[fold * N_valid:(fold+1)*N_valid] valid_y = y_flyingtime[fold * N_valid:(fold+1)*N_valid] train_x = np.delete(currattribute_x, list(range(fold * N_valid, (fold+1)*N_valid)), axis=0) train_y = np.delete(y_flyingtime, list(range(fold * N_valid, (fold+1)*N_valid))) w = np.linalg.solve(train_x.T @ train_x + Lambda * np.eye(train_x.shape[1]), train_x.T @ train_y) Errors_train[fold] = np.mean((train_y - train_x @ w)**2) Errors_valid[fold] = np.mean((valid_y - valid_x @ w)**2) return np.mean(Errors_train), np.mean(Errors_valid) def main(): errors_train, errors_valid = np.zeros(Dcount), np.zeros(Dcount) for d in range(Dcount): global currattribute_x currattribute_x = assemble_feature(x_attributes, d+1) errors_train[d], errors_valid[d] = fit(d+1, Lambda) print('Avg train error:', errors_train) print('Avg valid error:', errors_valid) if __name__ == "__main__": main()

data = loadmat('polynomial_regression_samples.mat', squeeze_me=True) x_attributes = data['x'] y_flyingtime = data['y'] folds = 4 Dcount = 5 Lambda = [0, 0.05, 0.1, 0.15, 0.2] currattribute_x = 0 def main(): errors_train, errors_valid = np.zeros((Dcount, len(Lambda))), np.zeros((Dcount, len(Lambda))) for d in range(Dcount): global currattribute_x currattribute_x = assemble_feature(x_attributes, d+1) for i in range(len(Lambda)): errors_train[d, i], errors_valid[d, i] = fit(d+1, Lambda[i]) print('Avg train error:', errors_train) print('Avg valid error:', errors_valid) D, i = np.unravel_index(errors_valid.argmin(), errors_valid.shape) print("D =", D+1, ", lambda =", Lambda[i]) if __name__ == "__main__": main()