Assignment 7: Neural Networks using Keras and Tensorflow 

# imports from __future__ import print_function from tensorflow import keras from tensorflow.keras.datasets import mnist from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Dropout, Flatten from tensorflow.keras.layers import Conv2D, MaxPooling2D from tensorflow.keras import backend as K import tensorflow as tf from matplotlib import pyplot as plt

# Hyper-parameters data-loading and formatting batch_size = 128 num_classes = 10 epochs = 10 img_rows, img_cols = 28, 28 (x_train, lbl_train), (x_test, lbl_test) = mnist.load_data() if K.image_data_format() == "channels_first": x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1)

x_train = x_train.astype("float32") x_test = x_test.astype("float32") x_train /= 255 x_test /= 255 print("x_train shape:", x_train.shape) print("x_test shape:", x_test.shape)

y_train = keras.utils.to_categorical(lbl_train, num_classes) y_test = keras.utils.to_categorical(lbl_test, num_classes) print('y_train shape, one-hot encoded:', y_train.shape) print('y_test shape, one-hot encoded:', y_test.shape)

## Define model ## model = Sequential()

model.add(Flatten(input_shape=(28,28)))

model.add(Dense(64, activation = 'relu')) model.add(Dense(64, activation = 'relu'))

model.add(Dense(num_classes, activation='softmax'))

model.compile( loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(lr=0.1), metrics=["accuracy"], )

model.summary()

fit_info = model.fit( x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test), # callbacks=[TqdmCallback()] ) loss, accuracy = model.evaluate(x_test, y_test, verbose=0) print("Test loss: {}, Test accuracy {}".format(loss, accuracy))

from tensorflow.keras import regularizers import numpy as np epochs = 40 replicas = 3 regularization_factors = np.linspace(0.000001, 0.001, 5) print('regularization_factors:', regularization_factors) accuracies = {} for l2_factor in regularization_factors: # regularizer, The L2 regularization penalty is computed as: loss = l2 * reduce_sum(square(x)) regularizer = regularizers.l2(l2_factor) print('Training models with regulizer factor:',l2_factor) accuracies[l2_factor] = [] for replica in range(replicas): # configure model model = Sequential() model.add(Flatten(input_shape=(28, 28))) model.add(Dense(500, activation='relu', kernel_regularizer=regularizer)) model.add(Dense(300, activation='relu')) model.add(Dense(num_classes, activation='softmax')) model.compile( loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(learning_rate=0.1), metrics=["accuracy"], ) fit = fit_info = model.fit( x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_data=(x_test, y_test), # callbacks=[TqdmCallback()] ) # collect accuracy of each replica for each l2 factor accuracies[l2_factor].append(np.max(fit.history.get('val_accuracy')))

# get mean validation accuracy and standard deviation from model training validation_accuracies = np.array([np.mean(accs) for accs in accuracies.values()]) standard_deviations = np.array([np.std(accs) for accs in accuracies.values()]) # plot mean validation accuracy with standard deviation as a function of the regularization factor plt.plot(regularization_factors, validation_accuracies, '--o', label='Mean validation accuracy') plt.fill_between( regularization_factors, validation_accuracies - standard_deviations, validation_accuracies + standard_deviations, color="lightgray", label="Standard deviation" ) plt.ylabel('Mean validation accuracy') plt.xlabel('Regularization factor') plt.legend(loc='best');

model = Sequential() model.add(Conv2D(32,kernel_size=(3,3), activation='relu')) model.add(MaxPooling2D(pool_size=(2,2),strides=2)) model.add(Conv2D(64,kernel_size=(3,3), activation='relu')) model.add(MaxPooling2D(pool_size=(2,2),strides=2)) model.add(Flatten(input_shape=(28, 28))) model.add(Dense(64, activation='relu', kernel_regularizer=regularizer)) model.add(Dense(64, activation='relu')) model.add(Dense(num_classes, activation='softmax')) model.compile( loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(learning_rate=0.1), metrics=["accuracy"], ) fit = model.fit( x_train, y_train, batch_size=batch_size, epochs=40, verbose=0, validation_data=(x_test, y_test), # callbacks=[TqdmCallback()] ) loss, accuracy = model.evaluate(x_test, y_test, verbose=0) print("Test loss: {}, Test accuracy {}".format(loss, accuracy))

import numpy as np def salt_and_pepper(input, noise_level=0.5): """ This applies salt and pepper noise to the input tensor - randomly setting bits to 1 or 0. Parameters ---------- input : tensor The tensor to apply salt and pepper noise to. noise_level : float The amount of salt and pepper noise to add. Returns ------- tensor Tensor with salt and pepper noise applied. """ # salt and pepper noise a = np.random.binomial(size=input.shape, n=1, p=(1 - noise_level)) b = np.random.binomial(size=input.shape, n=1, p=0.5) c = (a==0) * b return input * a + c #data preparation flattened_x_train = x_train.reshape(-1,784) flattened_x_train_seasoned = salt_and_pepper(flattened_x_train, noise_level=0.4) flattened_x_test = x_test.reshape(-1,784) flattened_x_test_seasoneed = salt_and_pepper(flattened_x_test, noise_level=0.4)

latent_dim = 96 input_image = keras.Input(shape=(784,)) encoded = Dense(128, activation='relu')(input_image) encoded = Dense(latent_dim, activation='relu')(encoded) decoded = Dense(128, activation='relu')(encoded) decoded = Dense(784, activation='sigmoid')(decoded) autoencoder = keras.Model(input_image, decoded) encoder = keras.Model(input_image, encoded) encoded_input = keras.Input(shape=(latent_dim,)) decoder_layer = Sequential(autoencoder.layers[-2:]) decoder = keras.Model(encoded_input, decoder_layer(encoded_input)) autoencoder.compile(optimizer='adam', loss='binary_crossentropy')

fit_info_AE = autoencoder.fit( flattened_x_train_seasoned, flattened_x_train, epochs=40, batch_size=64, shuffle=True, validation_data=(flattened_x_test_seasoneed, flattened_x_test), )