import numpy as np import torch import matplotlib.pyplot as plt from IPython.display import clear_output import sklearn.datasets as datasets

fig, ax = plt.subplots() # This is the simplest way to create a plot with Pyplot. The 'fig' object represents the entire figure you will # create, and the 'ax' object represents the axes you will plot on. By passing a value or tuple to subplots(), # you can create multiple axes or a grid of axes that all belong to the same figure. Here, we just create # one set of axes by leaving the argument blank. #Let's generate some data to plot: x = np.linspace(0,10,100) y = (x-3)**3 #The plot function plots the data on the axes as a graph ax.plot(x,y) #We also ought to adorn our axes with the appropriate titles and legends: ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_title('A Simple Function') ax.legend(['The Simple Function']) # Legends take a list of labels, in case you've plotted more than one curve on the axes #Though it's not necessary here, we can also adjust the axis limits: #ax.set_xlim(-10,10) #ax.set_ylim(-0,100) #When the plot is ready, use fig.show() fig.show() #If necessary, you can easily save a figure you've generated with plt.savefig()

# You can also plot on 3D axes with Pyplot fig = plt.figure() # We can't pass the projection argument directly to subplot(), so we generate the axes in two steps ax = fig.add_subplot(projection = '3d') # To plot a surface, create a 'meshgrid' of points in the x,y plane: x = np.linspace(0,10,100) y = np.linspace(0,10,100) X,Y = np.meshgrid(x,y) # You can then evaluate a 2D function on the meshgrid coordinates, then plot the surface Z = np.sin(X)+np.cos(Y) ax.plot_surface(X,Y,Z, cmap = 'coolwarm') # Trying out different colormaps is fun! # We can also add axis labels and such: ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') ax.set_title('My Lumpy Function') # For surfaces, it isn't trivial to set up a legend, but you can do so easily for scatter plots and such ax.set_zlim(-1.1,1.1) ax.view_init(20,30) # This sets the perspective angle from which we view the graph

#Defining a PyTorch Tensor x = torch.tensor([1.0, 2.0, 3.0]) print(x) print(x.device) print(x.type()) #If we had a CUDA capable GPU, we could define our tensors on a GPU # x = torcbh.tensor([1.0, 2.0, 3.0], device="cuda:0")

#PyTorch recycles a lot of the syntax from NumPy so we can carry out our familiar NumPy operations on PyTorch tensors x = torch.tensor([1.0, 2.0, 3.0]) x += 5 print(x) print(x.size()) x = torch.tensor(([1, 2], [3, 4])) print(x.T) y = torch.tensor(([1, 3], [5, 6])) print(x @ y) #What does this do? print(x * y) #How is this different from above? x = torch.ones((6)) x.unsqueeze(0) print(x) x = x.unsqueeze(0) print(x) print(x.shape)

fig = plt.figure() x = torch.linspace(-1,1, 100,requires_grad=True) y = x**2 y.backward(torch.ones(len(x))) plt.plot(x.detach().numpy(), y.detach().numpy(), label="x^2") plt.plot(x.detach().numpy(), x.grad.detach().numpy(),label='gradient') plt.plot(x.detach().numpy(), 2*x.detach().numpy(), label='true gradient') plt.legend()

# Generate some linearly separable data np.random.seed(3) # Set a fixed RNG seed so everyone's data looks the same n_data = 100 # How many data points we want in each cluster a_data = np.random.multivariate_normal([1.0,1.0], 0.2*np.eye(2), [n_data]) # Generate the data b_data = np.random.multivariate_normal([-1.0,-1.0], 0.2*np.eye(2), [n_data]) # Plot the data (using Pyplot's scatter function) fig, ax = plt.subplots() ax.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax.scatter(b_data[:,0], b_data[:,1], color = 'red') ax.set_title("Linearly Separable Data") ax.set_xlabel('x') ax.set_ylabel('y') fig.show()

# add a column to data to include a bias term a_data = np.concatenate([a_data, np.ones(len(a_data)).reshape(-1,1)],axis=1) b_data = np.concatenate([b_data, np.ones(len(b_data)).reshape(-1,1)],axis=1) x = np.concatenate([a_data, b_data]) y_a = np.array([1]*len(a_data)) y_b = np.array([0]*len(b_data)) y = np.concatenate([y_a, y_b])

# make the x numpy a tensor x = torch.tensor(x) # calculate weights! w = (torch.inverse(x.T @ x) @ x.T) @ y

w

def line(x_i, w): return (.5 - w[0]*x_i - w[2])/w[1] def make_plots(w, a_data, b_data): fig = plt.figure(figsize=(12,12)) ax = fig.add_subplot(projection='3d') # plot data points xx, yy = np.meshgrid(range(-3, 3), range(-3,3)) z = w[0]*xx + w[1]*yy + w[2] ax.scatter3D(a_data[:,0], a_data[:,1], y_a.reshape(-1,1),label='class_A', color='red') ax.scatter3D(b_data[:,0], b_data[:,1], y_b.reshape(-1,1), label='class_B', color='blue') ax.plot_surface(xx, yy, z.detach().numpy(), alpha=.5, cmap="coolwarm") ax.view_init(10,30) # This sets the perspective angle from which we view the graph} plt.show() fig = plt.figure() ax = fig.add_subplot() ax.scatter(a_data[:,0], a_data[:,1]) ax.scatter(b_data[:,0], b_data[:,1]) ax.contour(xx, yy, z, levels=[0.5]) plt.show() make_plots(w, a_data, b_data)

class LinearClassifier(torch.nn.Module): def __init__(self): super().__init__() self.params = torch.tensor([1.,1.,1.],requires_grad=True) def forward(self, x): output = x @ self.params return output

LC = LinearClassifier() y = torch.tensor(y) optimizer = torch.optim.SGD([LC.params], lr=.01, momentum=.9) all_loss = [] for step in range(1000): preds = LC(x.float()) loss = ((preds - y)**2).mean() all_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step()

plt.plot(list(range(1000)), all_loss) plt.ylabel("Loss") plt.xlabel("Step number")

print(LC.params)

final_predictions = x @ w.T

results = (final_predictions > .5) == y n_correct = len(results[results==True]) n_wrong = len(results[results==False]) print('accuracy: ', n_correct/(n_correct+n_wrong))

# Generate Two-Moons data np.random.seed(4) # Set a fixed RNG seed so everyone's data looks the same n_data = 100 # How many data points we want in each cluster data, targets = datasets.make_moons((n_data,n_data), shuffle = False, noise = 0.05) # Create the data b_data = data[:n_data] # Label the data a_data = data[n_data:] a_data = np.concatenate((a_data, np.ones(len(a_data)).reshape(-1,1)), axis=1) b_data = np.concatenate((b_data, np.ones(len(b_data)).reshape(-1,1)), axis=1) fig, ax = plt.subplots() # Plot the data ax.scatter(a_data[:,0], a_data[:,1], color = 'blue') ax.scatter(b_data[:,0], b_data[:,1], color = 'red') ax.set_title("Data that is NOT Linearly Separable") ax.set_xlabel('x') ax.set_ylabel('y') fig.show()

## Apply the normal equations here x = np.concatenate([a_data, b_data]) y_a = np.array([1]*len(a_data)) y_b = np.array([0]*len(b_data)) y = np.concatenate([y_a, y_b]) # make the x numpy a tensor x = torch.tensor(x) # calculate weights! w = (torch.inverse(x.T @ x) @ x.T) @ y make_plots(w, a_data,b_data)

class LogisticClassifier(torch.nn.Module): def __init__(self): super().__init__() self.params = torch.tensor([1.,1.,1.,1., 1.,1., 1.,1., 1.],requires_grad=True) def forward(self, x): basis_input = self.params[0]*(x[:, 0]**3) + self.params[1]*(x[:,0]**2) + \ self.params[2]*x[:, 0] + self.params[3]*(x[:,1]**3) + \ self.params[4]*(x[:, 1]**2) + self.params[5]*(x[:,1]) + self.params[6] + \ self.params[7]*(x[:, 0]**4) + self.params[8]*(x[:,1]**4) output = 1/(1 + torch.exp(basis_input)) return output

# Apply gradient descent here LC = LogisticClassifier() y = torch.tensor(y) optimizer = torch.optim.SGD([LC.params], lr=.01, momentum=.9) loss_func = torch.nn.BCELoss() all_loss = [] for step in range(50000): preds = LC(x.float()) #loss = ((preds - y)**2).mean() loss = loss_func(preds, y.float()) all_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() # make_plots(LC.params.detach(), a_data, b_data)

w = LC.params.detach().numpy()

w

final_predictions = LC(x.float()) results = (final_predictions > .5) == y n_correct = len(results[results==True]) n_wrong = len(results[results==False]) print('accuracy: ', n_correct/(n_correct+n_wrong))

def make_plots_logistic(w, a_data, b_data): fig = plt.figure(figsize=(12,12)) ax = fig.add_subplot(projection='3d') # plot data points xx, yy = np.meshgrid(range(-3, 3), range(-3,3)) z = w[0]*(xx**3) + w[1]*(xx**2) + w[2]*(xx) + w[3]*(yy**3) + w[4]*(yy**2) +\ w[5]*(yy) + w[6] ax.scatter3D(a_data[:,0], a_data[:,1], y_a.reshape(-1,1),label='class_A', color='red') ax.scatter3D(b_data[:,0], b_data[:,1], y_b.reshape(-1,1), label='class_B', color='blue') ax.plot_surface(xx, yy, z.detach().numpy().T, alpha=.5, cmap="coolwarm") ax.view_init(20,30) # This sets the perspective angle from which we view the graph} plt.show() fig = plt.figure() ax = fig.add_subplot() ax.scatter(a_data[:,0], a_data[:,1]) ax.scatter(b_data[:,0], b_data[:,1]) ax.contour(xx, yy, z, levels=[0.5]) plt.show()

make_plots_logistic(LC.params.detach(), a_data, b_data)