import numpy as np import torch import matplotlib.pyplot as plt from IPython.display import clear_output import sklearn.datasets as datasets

fig, ax = plt.subplots() # This is the simplest way to create a plot with Pyplot. The 'fig' object represents the entire figure you will # create, and the 'ax' object represents the axes you will plot on. By passing a value or tuple to subplots(), # you can create multiple axes or a grid of axes that all belong to the same figure. Here, we just create # one set of axes by leaving the argument blank. #Let's generate some data to plot: x = np.linspace(0,10,100) y = (x-3)**3 #The plot function plots the data on the axes as a graph ax.plot(x,y) #We also ought to adorn our axes with the appropriate titles and legends: ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_title('A Simple Function') ax.legend(['The Simple Function']) # Legends take a list of labels, in case you've plotted more than one curve on the axes #Though it's not necessary here, we can also adjust the axis limits: #ax.set_xlim(-10,10) #ax.set_ylim(-0,100) #When the plot is ready, use fig.show() fig.show() #If necessary, you can easily save a figure you've generated with plt.savefig()

# You can also plot on 3D axes with Pyplot fig = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax = fig.add_subplot(projection = '3d') # To plot a surface, create a 'meshgrid' of points in the x,y plane: x = np.linspace(0,10,100) y = np.linspace(0,10,100) X,Y = np.meshgrid(x,y) # You can then evaluate a 2D function on the meshgrid coordinates, then plot the surface Z = np.sin(X)+np.cos(Y) ax.plot_surface(X,Y,Z, cmap = 'coolwarm') # Trying out different colormaps is fun! # We can also add axis labels and such: ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') ax.set_title('My Lumpy Function') # For surfaces, it isn't trivial to set up a legend, but you can do so easily for scatter plots and such ax.set_zlim(-1.1,1.1) ax.view_init(20,30) # This sets the perspective angle from which we view the graph

#Defining a PyTorch Tensor x = torch.tensor([1.0, 2.0, 3.0]) print(x) print(x.device) print(x.type()) #If we had a CUDA capable GPU, we could define our tensors on a GPU # x = torcbh.tensor([1.0, 2.0, 3.0], device="cuda:0")

#PyTorch recycles a lot of the syntax from NumPy so we can carry out our familiar NumPy operations on PyTorch tensors x = torch.tensor([1.0, 2.0, 3.0]) x += 5 print(x) print(x.size()) x = torch.tensor(([1, 2], [3, 4])) print(x.T) y = torch.tensor(([1, 3], [5, 6])) print(x @ y) #What does this do? #Matrix multiplication print(x * y) #How is this different from above? #Elementwise multiplication x = torch.ones((6)) x.unsqueeze(0) print(x) x = x.unsqueeze(0) print(x) print(x.shape)

x = torch.linspace(-1,1,100, requires_grad=True) y = x**2 plt.plot(x.detach().numpy(), y.detach().numpy(),label = '$y = x^2$') external_grad = torch.ones(100) y.backward(gradient=external_grad) plt.plot(x.detach().numpy(), x.grad.detach().numpy(),label = '$y = 2x$') plt.legend() plt.xlabel('x') plt.ylabel('y') plt.title('Gradient Test')

# Generate some linearly separable data np.random.seed(3) # Set a fixed RNG seed so everyone's data looks the same n_data = 100 # How many data points we want in each cluster a_data = np.random.multivariate_normal([1.0,1.0], 0.2*np.eye(2), [n_data]) # Generate the data b_data = np.random.multivariate_normal([-1.0,-1.0], 0.2*np.eye(2), [n_data]) # Plot the data (using Pyplot's scatter function) fig, ax = plt.subplots() ax.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax.scatter(b_data[:,0], b_data[:,1], color = 'red') ax.set_title("Linearly Separable Data") ax.set_xlabel('x') ax.set_ylabel('y') fig.show()

#Convert your data into inputs and outputs. Make a = -1 and b = 1 for classification and make that z xy = np.concatenate([a_data,b_data],axis=0) z = np.concatenate([np.zeros(len(a_data)),np.ones(len(b_data))]) # print(xy.shape,z.shape) #Solve for w using the normal equation w = np.linalg.inv(xy.T @ xy) @ xy.T @ z print('Parameters = '+str(w)) # print(w.shape,xy.shape) #Plot onto the linspace x_pred = np.linspace(0,10,100) y_pred = np.linspace(0,10,100) X,Y = np.meshgrid(x_pred,y_pred) Z = X*w[0] + Y*w[1] fig = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax = fig.add_subplot(projection = '3d') ax.plot_surface(X,Y,Z, cmap = 'coolwarm') #Project onto 2D (i..e want to look at where Z = 0 = X*w[0] + Y*w[1]) fig2 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax2 = fig2.add_subplot() x_test = np.linspace(-1,1,100) ax2.plot(x_test,-(x_test*w[0])/w[1]) ax2.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax2.scatter(b_data[:,0], b_data[:,1], color = 'red') #Find the prediction accuracy z_pred = xy @ w < 0 #predict whether it thinks it's less than zero correct = z_pred == (z<0.5) acc = np.count_nonzero(correct)/len(xy) print('Prediction accuracy = '+str(int(100*(acc)))+'%')

class LinearRegression(torch.nn.Module): def __init__(self): super().__init__() self.w = torch.nn.Parameter(torch.rand(2)) def forward(self,x): y = x @ self.w return y def get_theta(self): return self.w #Create the module and the optimizer lr = LinearRegression() optimizer = torch.optim.SGD(lr.parameters(), lr=1e-4) #Establish the tensors xy = torch.tensor(xy).float().requires_grad_(True) z = torch.tensor(z).float().requires_grad_(True) #Training step steps = 10000 all_loss = [] #list to store the loss for s in range(steps): predictions = lr.forward(xy) loss = torch.nn.functional.mse_loss(predictions,z) all_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() #Plot the training loss fig1 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax1 = fig1.add_subplot() ax1.plot(np.arange(steps),all_loss) ax1.set_xlabel('Steps') ax1.set_ylabel('Total loss') ax1.set_title('Training loss') #Return the training parameters w = lr.get_theta().detach().numpy() print('Final parameters are = '+str(w)) #Plot onto the linspace x_pred = np.linspace(0,10,100) y_pred = np.linspace(0,10,100) X,Y = np.meshgrid(x_pred,y_pred) Z = X*w[0] + Y*w[1] fig2 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax2 = fig2.add_subplot(projection = '3d') ax2.plot_surface(X,Y,Z, cmap = 'coolwarm') #Project onto 2D (i..e want to look at where Z = 0 = X*w[0] + Y*w[1]) fig3 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax3 = fig3.add_subplot() x_test = np.linspace(-1,1,100) ax3.plot(x_test,-(x_test*w[0])/w[1]) ax3.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax3.scatter(b_data[:,0], b_data[:,1], color = 'red') #Find the prediction accuracy z_pred = xy.detach().numpy() @ w < 0 #predict whether it thinks it's less than zero correct = z_pred == (z.detach().numpy()<0.5) acc = np.count_nonzero(correct)/len(xy) print('Prediction accuracy = '+str(int(100*(acc)))+'%')

# Generate Two-Moons data np.random.seed(4) # Set a fixed RNG seed so everyone's data looks the same n_data = 100 # How many data points we want in each cluster data, targets = datasets.make_moons((n_data,n_data), shuffle = False, noise = 0.05) # Create the data b_data = data[:n_data] # Label the data a_data = data[n_data:] fig, ax = plt.subplots() # Plot the data ax.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax.scatter(b_data[:,0], b_data[:,1], color = 'red') ax.set_title("Data that is NOT Linearly Separable") ax.set_xlabel('x') ax.set_ylabel('y') fig.show()

# Apply the normal equations here #Convert your data into inputs and outputs. Make a = -1 and b = 1 for classification and make that z xy = np.concatenate([a_data,b_data],axis=0) z = np.concatenate([np.zeros(len(a_data)),np.ones(len(b_data))]) # print(xy.shape,z.shape) #Solve for w using the normal equation w = np.linalg.inv(xy.T @ xy) @ xy.T @ z print('Parameters = '+str(w)) # print(w.shape,xy.shape) #Plot onto the linspace x_pred = np.linspace(0,10,100) y_pred = np.linspace(0,10,100) X,Y = np.meshgrid(x_pred,y_pred) Z = X*w[0] + Y*w[1] fig = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax = fig.add_subplot(projection = '3d') ax.plot_surface(X,Y,Z, cmap = 'coolwarm') #Project onto 2D (i..e want to look at where Z = 0 = X*w[0] + Y*w[1]) fig2 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax2 = fig2.add_subplot() x_test = np.linspace(-1,2,100) ax2.plot(x_test,-(x_test*w[0])/w[1]) ax2.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax2.scatter(b_data[:,0], b_data[:,1], color = 'red') #Find the prediction accuracy z_pred = xy @ w < 0 #predict whether it thinks it's less than zero correct = z_pred == (z<0.5) acc = np.count_nonzero(correct)/len(xy) print('Prediction accuracy = '+str(int(100*(acc)))+'%') #Classification accuracy takes a steep hit!

#Create the module and the optimizer lr = LinearRegression() optimizer = torch.optim.SGD(lr.parameters(), lr=1e-4) #Establish the tensors xy = torch.tensor(xy).float().requires_grad_(True) z = torch.tensor(z).float().requires_grad_(True) #Training step steps = 10000 all_loss = [] #list to store the loss for s in range(steps): predictions = lr.forward(xy) loss = torch.nn.functional.mse_loss(predictions,z) all_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() #Plot the training loss fig1 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax1 = fig1.add_subplot() ax1.plot(np.arange(steps),all_loss) ax1.set_xlabel('Steps') ax1.set_ylabel('Total loss') ax1.set_title('Training loss') #Return the training parameters w = lr.get_theta().detach().numpy() print('Final parameters are = '+str(w)) #Plot onto the linspace x_pred = np.linspace(0,10,100) y_pred = np.linspace(0,10,100) X,Y = np.meshgrid(x_pred,y_pred) Z = X*w[0] + Y*w[1] fig2 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax2 = fig2.add_subplot(projection = '3d') ax2.plot_surface(X,Y,Z, cmap = 'coolwarm') #Project onto 2D (i..e want to look at where Z = 0 = X*w[0] + Y*w[1]) fig3 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax3 = fig3.add_subplot() x_test = np.linspace(-1,1,100) ax3.plot(x_test,-(x_test*w[0])/w[1]) ax3.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax3.scatter(b_data[:,0], b_data[:,1], color = 'red') #Find the prediction accuracy z_pred = xy.detach().numpy() @ w < 0 #predict whether it thinks it's less than zero correct = z_pred == (z.detach().numpy()<0.5) acc = np.count_nonzero(correct)/len(xy) print('Prediction accuracy = '+str(int(100*(acc)))+'%')

class LogisticRegression(torch.nn.Module): def __init__(self): super().__init__() self.w = torch.nn.Parameter(torch.rand(2)) self.b = torch.nn.Parameter(torch.rand(1)) self.a1 = torch.nn.Parameter(torch.rand(1)) self.a2 = torch.nn.Parameter(torch.rand(1)) self.a3 = torch.nn.Parameter(torch.rand(1)) self.a4 = torch.nn.Parameter(torch.rand(1)) self.a5 = torch.nn.Parameter(torch.rand(1)) self.a6 = torch.nn.Parameter(torch.rand(1)) def forward(self,x): #Make a basis function of the form a1x^5 + a2x^4 +a3x^3 + a4x^2 + a5x + a6 x = (self.a1*(x**5))+(self.a2*(x**4))+(self.a2*(3**3))+(self.a4*(x**2))+(self.a5*(x))+self.a6 y = (x @ self.w)+(self.b) y = torch.sigmoid(y) return y def get_params(self): return [self.w, self.b, self.a1, self.a2, self.a3, self.a4, self.a5, self.a6] #Create the module and the optimizer lr = LogisticRegression() optimizer = torch.optim.SGD(lr.parameters(), lr=1e-4) #Establish the tensors xy = torch.tensor(xy).float().requires_grad_(True) z = torch.tensor(z).float().requires_grad_(True) #Training step steps = 1000000 all_loss = [] #list to store the loss for s in range(steps): predictions = lr.forward(xy) loss = torch.nn.functional.mse_loss(predictions,z) all_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() #Plot the training loss fig1 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax1 = fig1.add_subplot() ax1.plot(np.arange(steps),all_loss) ax1.set_xlabel('Steps') ax1.set_ylabel('Total loss') ax1.set_title('Training loss') #Return the training parameters params = lr.get_params() params = [x.detach().numpy() for x in params] print('Final parameters are:'+str([str(x) for x in params])) w = params[0] b = params[1] a1 = params[2] a2 = params[3] a3 = params[4] a4 = params[5] a5 = params[6] a6 = params[7]

w = params[0] b = params[1] a1 = params[2] a2 = params[3] a3 = params[4] a4 = params[5] #Plot onto the linspace x_pred = np.linspace(-2,2,100) y_pred = np.linspace(-2,2,100) X,Y = np.meshgrid(x_pred,y_pred) X2 = (a1*(X**4))+(a2*(X**3))+(a3*(X**2))+(a4*(X))+a5 Y2 = (a1*(Y**4))+(a2*(Y**3))+(a3*(Y**2))+(a4*(Y))+a5 Z = X2*w[0] + Y2*w[1] + b fig2 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax2 = fig2.add_subplot(projection = '3d') ax2.plot_surface(X,Y,Z, cmap = 'coolwarm') #Project onto 2D (i..e want to look at where Z = 0.5 = exp(X)*w[0] + exp(Y)*w[1]+b) fig3 = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax3 = fig3.add_subplot() err = 0.01 xt = np.linspace(-2,2,100) yt = np.linspace(-2,2,100) x_slice = [] y_slice = [] for i in range(len(xt)): for j in range(len(yt)): x2 = (a1*(xt[i]**5))+(a2*(xt[i]**4))+(a3*(xt[i]**3))+(a4*(xt[i]**2))+(a5*(xt[i]))+a6 y2 = (a1*(yt[j]**5))+(a2*(yt[j]**4))+(a3*(yt[j]**3))+(a4*(yt[j]**2))+(a5*(yt[j]))+a6 z = x2*w[0] + y2*w[1] + b if (abs(z-0.5)) < err: x_slice += [xt[i]] y_slice += [yt[j]] x_slice = np.array(x_slice) y_slice = np.array(y_slice) print(x_slice.shape,y_slice.shape) # print(x_slice,y_slice) ax3.plot(x_slice,y_slice) # plt.xlim(-2,2) ax3.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax3.scatter(b_data[:,0], b_data[:,1], color = 'red') # #Find the prediction accuracy # z_pred = (xy.detach().numpy() @ w) + b < 0.5 #predict whether it thinks it's less than zero # correct = z_pred == (z.detach().numpy()<0.5) # acc = np.count_nonzero(correct)/len(xy) # print('Prediction accuracy = '+str(int(100*(acc)))+'%') print("couldn't get this one to work :(")