Neural Networks -- Multi-layer Perceptrons

import struct import numpy as np import matplotlib.pyplot as plt

def slp(x, w, b): return np.sign(np.dot(x, w) + b) def slp_and(x): w = np.array([ 1, 1 ], dtype=np.float32) # What should these be? b = -0.5 # What should this be? return slp(x, w, b) def slp_or(x): w = np.array([ 1, 1 ], dtype=np.float32) # What should these be? b = 0.5 # What should this be? return slp(x, w, b) def mlp_xor(x): w1 = np.array([ -1, 1 ], dtype=np.float32) # What should these be? b1 = -0.5 # What should this be? y1 = slp(x, w1, b1) w2 = np.array([ 1, -1 ], dtype=np.float32) # What should these be? b2 = -0.5 # What should this be? y2 = slp(x, w2, b2) w3 = np.array([ 1, 1 ], dtype=np.float32) # What should these be? b3 = 0.5 # What should this be? y3 = slp(np.stack([y1, y2], -1), w3, b3) # y3 = slp_or([slp_and([x[0], -x[1]]), slp_and([-x[0], x[1]])]) return y3 x = np.array([ [-1.0, -1.0], [-1.0, 1.0], [ 1.0, -1.0], [ 1.0, 1.0], ]) print("AND") print(np.concatenate([x, slp_and(x).reshape(-1,1)], -1)) print("OR") print(np.concatenate([x, slp_or(x).reshape(-1,1)], -1)) print("XOR") print(np.concatenate([x, mlp_xor(x).reshape(-1,1)], -1))

###################################################################### # nnet_demo.py - Neural Network demo # Writen for ENGR 027 - Computer Vision # Matt Zucker 2020-2021 ###################################################################### IMAGE_SIZE = 28 INSET_SIZE = 20 INSET_MARGIN = (IMAGE_SIZE - INSET_SIZE)//2 NUM_INPUT = IMAGE_SIZE*IMAGE_SIZE NUM_HIDDEN = 300 NUM_OUTPUT = 10 PIXEL_SCALE = 16 PEN_SIZE = 28 WINDOW_SIZE = IMAGE_SIZE*PIXEL_SCALE WINDOW_INSET_SIZE = INSET_SIZE*PIXEL_SCALE WINDOW_INSET_MARGIN = INSET_MARGIN*PIXEL_SCALE SCALE_MIN = 0.5 / PIXEL_SCALE SCALE_MAX = 2.0 / PIXEL_SCALE WINDOW_TITLE = 'MNIST Demo' ###################################################################### # preprocess image array of dtype uint8 and shape (n, 28, 28) # into "sphered" array of dtype float32 and shape (n, 784) # https://en.wikipedia.org/wiki/Whitening_transformation def preprocess(imgs): assert len(imgs.shape) == 3 assert imgs.shape[1:] == (IMAGE_SIZE, IMAGE_SIZE) v = imgs.reshape(-1, NUM_INPUT).astype(np.float32) v -= v.mean(axis=1).reshape(-1, 1) # subtract row-wise mean s = v.std(axis=1).reshape(-1, 1) v /= np.where(s, s, 1) # divide off row-wise stddev but avoid divide by zero return v ###################################################################### # predict classes for batch of data def predict(weights, x): # x.shape == (num_samples, num_features) for w in weights: # w.shape == (num_layer_inputs+1, num_layer_outputs) # all columns but last correspond to weight terms layer_weights = w[:-1] # last column corresponds to bias terms layer_bias = w[-1] # matrix multiplication plus bias xw = np.dot(x, layer_weights) + layer_bias # pass thru nonlinear tanh activation function x = np.tanh(xw) # return final classification = max over all class outputs # per-sample -- shape is just (num_samples,) return x.argmax(axis=1) ###################################################################### # class for running this demo # load weights with np.load('/work/data/mnist_weights_quantized.npz') as npzfile: weights = [] for key in npzfile.files: w = npzfile[key] # weights are stored in the file as signed 8-bit integers # where +/- 127 corresponds to +/- 0.5 w = w.astype(np.float32) * 0.5 / 127 weights.append(w) # load test data with np.load('/work/data/sample_data.npz') as npzfile: labels = npzfile['labels'] images = npzfile['images'] # shuffle so user sees different thing each program run idx = np.arange(len(labels)) np.random.shuffle(idx) sample_labels = labels[idx] sample_images = images[idx] # preprocess into (n-by-784) array of floats sample_vectors = preprocess(sample_images) # make predictions on test set and print error rate sample_predictions = predict(weights, sample_vectors) is_incorrect = (sample_predictions != sample_labels) incorrect_idx = np.where(is_incorrect)[0] error_rate = is_incorrect.mean() print('error rate on sample dataset: {:.2f}% ({} out of {})'.format(error_rate*100, len(incorrect_idx), len(sample_labels))) rows, cols = 3, 7 fig, ax = plt.subplots(rows, cols, figsize=(3*cols, 3.2*rows)) show_idx = 0 for i in range(rows): for j in range(cols-1): ax[i,j].imshow(sample_images[show_idx]) ax[i,j].set_title("Real={}, Predicted={}".format(sample_labels[show_idx], sample_predictions[show_idx])) ax[i,j].axis('off') show_idx += 1 # Errors for i in range(rows): ax[i,-1].imshow(sample_images[incorrect_idx[i]]) ax[i,-1].set_title("Real={}, Predicted={}".format(sample_labels[incorrect_idx[i]], sample_predictions[incorrect_idx[i]]))

with np.load('/work/data/mnist_weights_quantized.npz') as npzfile: weights = npzfile[npzfile.files[0]] assert weights.shape == (785, 300) and weights.dtype == np.int8 weights = weights[:-1, :].transpose() weights = weights.reshape(300, 28, 28).astype(np.float32) rows = 20 cols = 15 size = 28 margin = 2 height = (size + margin)*rows + margin width = (size + margin)*cols + margin output = np.zeros((height, width), dtype=np.uint8) row = 0 col = 0 for wimage in weights: wmin = wimage.min() wmax = wimage.max() display = (wimage - wmin) / (wmax - wmin) display = (display*255).astype(np.uint8) y = row * (size+margin) + margin x = col * (size+margin) + margin output[y:y+size, x:x+size] = display col += 1 if col >= cols: col = 0 row += 1 h, w = output.shape h2, w2 = h/w, 1 sz = 25 plt.figure(figsize=(sz*h2, sz*h2)) plt.imshow(output) plt.axis('off')