Assignment 7: Neural Networks using Keras and Tensorflow
# imports
from __future__ import print_function
from tensorflow import keras
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras import backend as K
import tensorflow as tf
from matplotlib import pyplot as plt
# Hyper-parameters data-loading and formatting
batch_size = 128
num_classes = 10
epochs = 10
img_rows, img_cols = 28, 28
(x_train, lbl_train), (x_test, lbl_test) = mnist.load_data()
if K.image_data_format() == "channels_first":
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")
x_train /= 255
x_test /= 255
print("x_train shape:", x_train.shape)
print("x_test shape:", x_test.shape)
x_train shape: (60000, 28, 28, 1)
x_test shape: (10000, 28, 28, 1)
y_train = keras.utils.to_categorical(lbl_train, num_classes)
y_test = keras.utils.to_categorical(lbl_test, num_classes)
print('y_train shape, one-hot encoded:', y_train.shape)
print('y_test shape, one-hot encoded:', y_test.shape)
y_train shape, one-hot encoded: (60000, 10)
y_test shape, one-hot encoded: (10000, 10)
## Define model ##
model = Sequential()
model.add(Flatten(input_shape=(28,28)))
model.add(Dense(64, activation = 'relu'))
model.add(Dense(64, activation = 'relu'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(
loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.SGD(lr=0.1),
metrics=["accuracy"],
)
model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
flatten (Flatten) (None, 784) 0
_________________________________________________________________
dense (Dense) (None, 64) 50240
_________________________________________________________________
dense_1 (Dense) (None, 64) 4160
_________________________________________________________________
dense_2 (Dense) (None, 10) 650
=================================================================
Total params: 55,050
Trainable params: 55,050
Non-trainable params: 0
_________________________________________________________________
fit_info = model.fit(
x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test),
# callbacks=[TqdmCallback()]
)
loss, accuracy = model.evaluate(x_test, y_test, verbose=0)
print("Test loss: {}, Test accuracy {}".format(loss, accuracy))
Epoch 1/10
469/469 [==============================] - 4s 7ms/step - loss: 0.8062 - accuracy: 0.7738 - val_loss: 0.2807 - val_accuracy: 0.9188
Epoch 2/10
469/469 [==============================] - 2s 3ms/step - loss: 0.2456 - accuracy: 0.9282 - val_loss: 0.1944 - val_accuracy: 0.9434
Epoch 3/10
469/469 [==============================] - 2s 4ms/step - loss: 0.1827 - accuracy: 0.9460 - val_loss: 0.1661 - val_accuracy: 0.9518
Epoch 4/10
469/469 [==============================] - 2s 4ms/step - loss: 0.1503 - accuracy: 0.9567 - val_loss: 0.1395 - val_accuracy: 0.9572
Epoch 5/10
469/469 [==============================] - 2s 4ms/step - loss: 0.1286 - accuracy: 0.9624 - val_loss: 0.1286 - val_accuracy: 0.9596
Epoch 6/10
469/469 [==============================] - 2s 4ms/step - loss: 0.1125 - accuracy: 0.9667 - val_loss: 0.1096 - val_accuracy: 0.9657
Epoch 7/10
469/469 [==============================] - 2s 3ms/step - loss: 0.0968 - accuracy: 0.9715 - val_loss: 0.1044 - val_accuracy: 0.9672
Epoch 8/10
469/469 [==============================] - 2s 3ms/step - loss: 0.0880 - accuracy: 0.9747 - val_loss: 0.1034 - val_accuracy: 0.9671
Epoch 9/10
469/469 [==============================] - 2s 4ms/step - loss: 0.0776 - accuracy: 0.9771 - val_loss: 0.1105 - val_accuracy: 0.9638
Epoch 10/10
469/469 [==============================] - 2s 3ms/step - loss: 0.0721 - accuracy: 0.9785 - val_loss: 0.0956 - val_accuracy: 0.9691
Test loss: 0.09558530151844025, Test accuracy 0.9690999984741211
from tensorflow.keras import regularizers
import numpy as np
epochs = 40
replicas = 3
regularization_factors = np.linspace(0.000001, 0.001, 5)
print('regularization_factors:', regularization_factors)
accuracies = {}
for l2_factor in regularization_factors:
# regularizer, The L2 regularization penalty is computed as: loss = l2 * reduce_sum(square(x))
regularizer = regularizers.l2(l2_factor)
print('Training models with regulizer factor:',l2_factor)
accuracies[l2_factor] = []
for replica in range(replicas):
# configure model
model = Sequential()
model.add(Flatten(input_shape=(28, 28)))
model.add(Dense(500, activation='relu', kernel_regularizer=regularizer))
model.add(Dense(300, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(
loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.SGD(learning_rate=0.1),
metrics=["accuracy"],
)
fit = fit_info = model.fit(
x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_data=(x_test, y_test),
# callbacks=[TqdmCallback()]
)
# collect accuracy of each replica for each l2 factor
accuracies[l2_factor].append(np.max(fit.history.get('val_accuracy')))
regularization_factors: [1.0000e-06 2.5075e-04 5.0050e-04 7.5025e-04 1.0000e-03]
Training models with regulizer factor: 1e-06
Training models with regulizer factor: 0.00025075000000000005
Training models with regulizer factor: 0.0005005000000000001
Training models with regulizer factor: 0.0007502500000000002
Training models with regulizer factor: 0.001
# get mean validation accuracy and standard deviation from model training
validation_accuracies = np.array([np.mean(accs) for accs in accuracies.values()])
standard_deviations = np.array([np.std(accs) for accs in accuracies.values()])
# plot mean validation accuracy with standard deviation as a function of the regularization factor
plt.plot(regularization_factors, validation_accuracies, '--o', label='Mean validation accuracy')
plt.fill_between(
regularization_factors,
validation_accuracies - standard_deviations,
validation_accuracies + standard_deviations,
color="lightgray",
label="Standard deviation"
)
plt.ylabel('Mean validation accuracy')
plt.xlabel('Regularization factor')
plt.legend(loc='best');
Question 3)
model = Sequential()
model.add(Conv2D(32,kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2),strides=2))
model.add(Conv2D(64,kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2),strides=2))
model.add(Flatten(input_shape=(28, 28)))
model.add(Dense(64, activation='relu', kernel_regularizer=regularizer))
model.add(Dense(64, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(
loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.SGD(learning_rate=0.1),
metrics=["accuracy"],
)
fit = model.fit(
x_train,
y_train,
batch_size=batch_size,
epochs=40,
verbose=0,
validation_data=(x_test, y_test),
# callbacks=[TqdmCallback()]
)
loss, accuracy = model.evaluate(x_test, y_test, verbose=0)
print("Test loss: {}, Test accuracy {}".format(loss, accuracy))
Test loss: 0.06852845847606659, Test accuracy 0.9850000143051147
import numpy as np
def salt_and_pepper(input, noise_level=0.5):
"""
This applies salt and pepper noise to the input tensor - randomly setting bits to 1 or 0.
Parameters
----------
input : tensor
The tensor to apply salt and pepper noise to.
noise_level : float
The amount of salt and pepper noise to add.
Returns
-------
tensor
Tensor with salt and pepper noise applied.
"""
# salt and pepper noise
a = np.random.binomial(size=input.shape, n=1, p=(1 - noise_level))
b = np.random.binomial(size=input.shape, n=1, p=0.5)
c = (a==0) * b
return input * a + c
#data preparation
flattened_x_train = x_train.reshape(-1,784)
flattened_x_train_seasoned = salt_and_pepper(flattened_x_train, noise_level=0.4)
flattened_x_test = x_test.reshape(-1,784)
flattened_x_test_seasoneed = salt_and_pepper(flattened_x_test, noise_level=0.4)
latent_dim = 96
input_image = keras.Input(shape=(784,))
encoded = Dense(128, activation='relu')(input_image)
encoded = Dense(latent_dim, activation='relu')(encoded)
decoded = Dense(128, activation='relu')(encoded)
decoded = Dense(784, activation='sigmoid')(decoded)
autoencoder = keras.Model(input_image, decoded)
encoder = keras.Model(input_image, encoded)
encoded_input = keras.Input(shape=(latent_dim,))
decoder_layer = Sequential(autoencoder.layers[-2:])
decoder = keras.Model(encoded_input, decoder_layer(encoded_input))
autoencoder.compile(optimizer='adam', loss='binary_crossentropy')
fit_info_AE = autoencoder.fit(
flattened_x_train_seasoned,
flattened_x_train,
epochs=40,
batch_size=64,
shuffle=True,
validation_data=(flattened_x_test_seasoneed, flattened_x_test),
)
Epoch 1/5
938/938 [==============================] - 8s 8ms/step - loss: 0.0040 - val_loss: 0.0041
Epoch 2/5
938/938 [==============================] - 7s 8ms/step - loss: 0.0040 - val_loss: 0.0041
Epoch 3/5
938/938 [==============================] - 7s 8ms/step - loss: 0.0040 - val_loss: 0.0041
Epoch 4/5
938/938 [==============================] - 8s 9ms/step - loss: 0.0040 - val_loss: 0.0041
Epoch 5/5
938/938 [==============================] - 8s 8ms/step - loss: 0.0040 - val_loss: 0.0040