# imports from __future__ import print_function import keras from keras import utils as np_utils import tensorflow from keras.datasets import mnist from keras.models import Sequential from keras.layers import Dense, Dropout, Flatten from keras.layers import Conv2D, MaxPooling2D from keras import backend as K import tensorflow as tf from matplotlib import pyplot as plt import numpy as np import pandas as pd

# data-loading and formatting img_rows, img_cols = 28, 28 (x_train_orig, y_train_orig), (x_test_orig, y_test_orig) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train_orig.reshape(x_train_orig.shape[0], 1, img_rows, img_cols) x_test = x_test_orig.reshape(x_test_orig.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: x_train = x_train_orig.reshape(x_train_orig.shape[0], img_rows, img_cols, 1) x_test = x_test_orig.reshape(x_test_orig.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1)

print(K.image_data_format()) print(x_train_orig.shape) print(x_train.shape) print(x_test_orig.shape) print(x_test.shape)

# normalizing the x_train & x_test data x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # one-hot enconding num_classes = 10 y_train = keras.utils.np_utils.to_categorical(y_train_orig, num_classes) y_test = keras.utils.np_utils.to_categorical(y_test_orig, num_classes)

y_train_orig[:5]

y_train[:5]

# Define model batch_size = 128 epochs = 10 model = Sequential() model.add(Flatten()) # input layer, flattened in dimension from 2D (28, 28) to 1D (784) model.add(Dense(64, activation = 'relu')) # first hidden layer with 64 neurons with activation function relu model.add(Dense(64, activation = 'relu')) # second hidden layer with 64 neurons with activation function relu model.add(Dense(num_classes, activation='softmax')) # the output layer model.compile(loss=keras.losses.categorical_crossentropy, optimizer=tensorflow.keras.optimizers.SGD(learning_rate = 0.1), metrics=['accuracy'],) fit_info = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss: {}, Test accuracy {}'.format(score[0], score[1]))

model.summary()

# training accuracy plt.plot(fit_info.history['accuracy']) plt.title('Training Accuracy') plt.xlabel('epoch') plt.ylabel('accuracy')

# validation accuracy plt.plot(fit_info.history['val_accuracy']) plt.title('Validation Accuracy') plt.xlabel('epoch') plt.ylabel('val_accuracy')

# Training for 40 epochs epochs = 40 # 5 L2 reg factor values from 0.000001 to 0.001 l2v = np.linspace(0.000001, 0.001, 5) score = [] # 5 regularizations for i in range(5): # 3 replicate networks for j in range(3): model = Sequential() # input layer model.add(Flatten()) # first layer with 500 hidden units and i-th regularization factor from the 5 values defined above model.add(Dense(500, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l2v[i]))) # second layer with 300 hidden units and the ith regularization factor from the 5 values defined above model.add(Dense(300, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l2v[i]))) # output layer model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=tensorflow.keras.optimizers.SGD(learning_rate = 0.1), metrics=['accuracy'],) fit_info = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) score.append(model.evaluate(x_test, y_test, verbose=0)) print(score)

pd.DataFrame(score, columns=['loss', 'accuracy']).to_csv('scores_Hinton.csv') df = pd.read_csv('scores_Hinton.csv') mean = [] std = [] # Variable to keep count of replicates per regularization reg_repl = 0 # 5 iterations representing the 5 regs for reg in range(5): # storing mean and standard deviation for the corresponding replicates per regularization mean.append(np.mean(df['accuracy'][reg_repl:reg_repl+2])) std.append(np.std(df['accuracy'][reg_repl:reg_repl+2])) # moving to the next set of replicates reg_repl += 3 plt.scatter(l2v, mean) plt.xlabel('Reg. factor') plt.ylabel('Val. accuracy') plt.title('Validation accuracy trainings with weight decay') plt.errorbar(l2v, mean, std, fmt = 'bo', markersize=10, capsize=5) max_accuracy = np.max(df['accuracy']) Hinton_margin = 0.9847 - max_accuracy print('Best score: ', max_accuracy) print('Difference between Hinton score and best score here: ', Hinton_margin)

batch_size = 128 epochs = 10 model = Sequential() # a convolutional layer model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # a max pooling layer to downsize the input model.add(MaxPooling2D(pool_size=(2, 2))) # a flatten layer to decrease the dimensions model.add(Flatten()) # a fully connected layer with 128 neurons model.add(Dense(128, activation='relu')) # the output layer model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(lr = 0.1), metrics=['accuracy'],) epochs = 10 # training and validating the model fit_info = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) pd.DataFrame(fit_info.history).to_csv("scores3_1.csv") model.save("model3_1")

df_scores_1 = pd.read_csv("scores3_1.csv") max_val_1 = df_scores_1['val_accuracy'].max().round(4)*100 print("Max Validation Accuracy of Model 1 is", max_val_1, "%")

batch_size = 128 epochs = 10 model = Sequential() # a convolutional layer with 32 filters and 3x3 kernel model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # a convolutional layer with 64 filters and 3x3 kernel model.add(Conv2D(64, (3, 3), activation='relu')) # a max pooling layer to downsize the input model.add(MaxPooling2D(pool_size=(2, 2))) # a flatten layer to decrease the dimensions model.add(Flatten()) # a fullly connected layer model.add(Dense(128, activation='relu')) # the output layer model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(lr = 0.1), metrics=['accuracy'],) epochs = 10 fit_info = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) pd.DataFrame(fit_info.history).to_csv("scores3_0.csv") #<--Saving the scores model.save("model3_0")

df_scores_2 = pd.read_csv("scores3_2.csv") max_val_2 = df_scores_2['val_accuracy'].max().round(4)*100 print("Max Validation Accuracy of Model 2 is", max_val_2, "%")

batch_size = 128 epochs = 10 model = Sequential() # a convolutional layer with 32 filters and 3x3 kernel model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # a convolutional layer with 64 filters and 3x3 kernel model.add(Conv2D(64, (3, 3), activation='relu')) # a max pooling layer model.add(MaxPooling2D(pool_size=(2, 2))) # a dropout layer for preventing overfitting model.add(Dropout(0.5)) # a flatten layer to decrease the dimensions model.add(Flatten()) # a fully connected layer with 128 neurons model.add(Dense(128, activation='relu')) # a dropout layer model.add(Dropout(0.5)) # the output layer, using softmax to get probabilities model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(lr = 0.1), metrics=['accuracy'],) epochs = 10 fit_info = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) pd.DataFrame(fit_info.history).to_csv("scores3_1.csv") #<--Saving the scores model.save("model3_1")

df_scores_3 = pd.read_csv("scores3_3.csv") max_val_3 = df_scores_3['val_accuracy'].max().round(4)*100 print("Max Validation Accuracy of Model 3 is", max_val_3, "%")

batch_size = 128 epochs = 10 model = Sequential() # a convolutional layer with 32 filters and 3x3 kernel model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # a convolutional layer with 64 filters and 3x3 kernel model.add(Conv2D(64, (3, 3), activation='relu')) #64 convolution filters used each of size 3x3 # a max pooling layer to downsize the input model.add(MaxPooling2D(pool_size=(2, 2))) # a dropout layer with dropout rate of 0.25 model.add(Dropout(0.25)) # a flatten layer to decrease the dimensions model.add(Flatten()) # a fully connected layer with 128 neurons model.add(Dense(128, activation='relu')) # a dropout layer with dropout rate of 0.5 model.add(Dropout(0.5)) # the output layer model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.SGD(lr = 0.1), metrics=['accuracy'],) epochs = 10 fit_info = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) pd.DataFrame(fit_info.history).to_csv("scores3_2.csv") #<--Saving the scores model.save("model3_2")

df_scores_4 = pd.read_csv("scores3_4.csv") max_val_4 = df_scores_4['val_accuracy'].max().round(4)*100 print("Max Validation Accuracy of Model 4 is", max_val_4, "%")

print("Summary of Max Valudations Accuracy") print("Max Validation Accuracy of Model 1 is", max_val_1, "%") print("Max Validation Accuracy of Model 2 is", max_val_2, "%") print("Max Validation Accuracy of Model 3 is", max_val_3, "%") print("Max Validation Accuracy of Model 4 is", max_val_4, "%")

from IPython.display import display, Image display(Image(filename='nn (1).png'))

def salt_and_pepper(input, noise_level=0.5): """ This applies salt and pepper noise to the input tensor - randomly setting bits to 1 or 0. Parameters ---------- input : tensor The tensor to apply salt and pepper noise to. noise_level : float The amount of salt and pepper noise to add. Returns ------- tensor Tensor with salt and pepper noise applied. """ # salt and pepper noise a = np.random.binomial(size=input.shape, n=1, p=(1 - noise_level)) b = np.random.binomial(size=input.shape, n=1, p=0.5) c = (a==0) * b return input * a + c # data preparation flattened_x_train = x_train.reshape(-1,784) flattened_x_train_seasoned = salt_and_pepper(flattened_x_train, noise_level=0.4) # create noised train data flattened_x_test = x_test.reshape(-1,784) flattened_x_test_seasoneed = salt_and_pepper(flattened_x_test, noise_level=0.4) # create noised test data

latent_dim = 96 input_image = keras.Input(shape=(784,)) encoded = Dense(128, activation='relu')(input_image) encoded = Dense(latent_dim, activation='relu')(encoded) decoded = Dense(128, activation='relu')(encoded) decoded = Dense(784, activation='sigmoid')(decoded) autoencoder = keras.Model(input_image, decoded) encoder_only = keras.Model(input_image, encoded) encoded_input = keras.Input(shape=(latent_dim,)) decoder_layer = Sequential(autoencoder.layers[-2:]) decoder = keras.Model(encoded_input, decoder_layer(encoded_input)) autoencoder.compile(optimizer='adam', loss='binary_crossentropy')

fit_info_AE = autoencoder.fit(flattened_x_train_seasoned, flattened_x_train, epochs=32, batch_size=64, shuffle=True, validation_data=(flattened_x_test_seasoneed, flattened_x_test))

autoencoder.save("autoencoder")

autoencoder = keras.models.load_model("autoencoder") noise = np.linspace(0,1,6).round(2) index = np.random.randint(len(flattened_x_test_seasoneed), size=4) for i in range(len(noise)): flattened_x_test_seasoneed = salt_and_pepper(flattened_x_test, noise[i]) denoised_images = autoencoder.predict(flattened_x_test_seasoneed) fig, axes = plt.subplots(3, 4, figsize=(10,5)) fig.suptitle('Noise:'+str(noise[i]), fontsize=12) for i in range(4): # Plot original images axes[0][i].set_title('original', fontsize=10) axes[0][i].imshow(x_test[index[i]].reshape(28,28), cmap='gray') axes[0][i].get_xaxis().set_visible(False) axes[0][i].get_yaxis().set_visible(False) # Plot seasoned images axes[1][i].set_title('seasoned', fontsize=10) axes[1][i].imshow(flattened_x_test_seasoneed[[index[i]]].reshape(28,28), cmap='gray') axes[1][i].get_xaxis().set_visible(False) axes[1][i].get_yaxis().set_visible(False) # Plot denoised images axes[2][i].set_title('denoised', fontsize=10) axes[2][i].imshow(denoised_images[index[i]].reshape(28,28), cmap='gray') axes[2][i].get_xaxis().set_visible(False) axes[2][i].get_yaxis().set_visible(False)

# load the autoencoder autoencoder = keras.models.load_model("autoencoder") # load the best performing CNN model best_model = keras.models.load_model("model3_4") noise = np.linspace(0, 1, 20) denoised_score = [] seasoned_score = [] for i in range(len(noise)): # noised data flattened_noised_images = salt_and_pepper(flattened_x_test, noise[i]) # denoised data flattened_denoised_images = autoencoder.predict(flattened_noised_images) decoded_images = flattened_denoised_images.reshape(len(flattened_denoised_images), 28, 28, 1) noised_images = flattened_noised_images.reshape(len(flattened_noised_images), 28, 28, 1) denoised_score.append(best_model.evaluate(decoded_images, y_test)[1]) seasoned_score.append(best_model.evaluate(noised_images, y_test)[1])

plt.figure(figsize=(5,4)) plt.scatter(noise, denoised_score, color='green', label='denoised') plt.scatter(noise, seasoned_score, color='orange', label='seasoned') plt.xlabel("Noise") plt.ylabel("True Positive Rate") plt.title("True Positive Rate for Denoised and Seasoned Images", fontsize=12) plt.legend()

# find the index for the images of digit 8 # we use these two images to create a new handwritten digit of 8 for i in range(100): if y_test_orig[i] == 8: print(i)

# create 10 interpolated images of imag1 and imag2 with different alpha values def interpolate_images(imag1, imag2): vectors = [] for i in np.linspace(0, 1, 10): v = (1 - i) * imag1 + i * imag2 vectors.append(v) return np.asarray(vectors) # two original images of digit 8 used to generate synthtic handwritten digit of 8 imag1 = x_test[61].reshape(-1,784) imag2 = x_test[84].reshape(-1,784) # encode original images encoded_imag1 = encoder_only.predict(imag1).flatten() encoded_imag2 = encoder_only.predict(imag2).flatten() # interpolate original images and create 10 interpolated images interpolated_imag = interpolate_images(encoded_imag1, encoded_imag2) interpolated_orig_imag = interpolate_images(imag1.flatten(), imag2.flatten()) # generate 10 new handwritten digit of 8 decoded_images = decoder.predict(interpolated_imag) # plot the interpolated images of the original images fig, axes = plt.subplots(1,len(interpolated_orig_imag), figsize=(8,1)) fig.suptitle("Interpolated Encoded Image") for i in range(len(interpolated_imag)): axes[i].imshow(interpolated_orig_imag[i].reshape(28,28), cmap='gray') axes[i].get_xaxis().set_visible(False) axes[i].get_yaxis().set_visible(False) # plot the generated images fig, axes = plt.subplots(1,len(interpolated_imag), figsize=(8,1)) fig.suptitle("Generated Image") for i in range(len(interpolated_imag)): axes[i].imshow(decoder.predict(interpolated_imag[i].reshape(1,latent_dim)).reshape(28,28), cmap='gray') axes[i].get_xaxis().set_visible(False) axes[i].get_yaxis().set_visible(False)