ASL project

# Import Tensorflow 2.0 #%tensorflow_version 2.x --> only google collab import tensorflow as tf !pip install mitdeeplearning import mitdeeplearning as mdl import matplotlib.pyplot as plt import numpy as np import random from tqdm import tqdm import cv2 import os from sklearn.model_selection import GridSearchCV from keras.wrappers.scikit_learn import KerasClassifier import time import seaborn as sns import pandas as pd from sklearn.metrics import confusion_matrix from keras import layers from keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D from keras.models import Model, load_model from keras.initializers import glorot_uniform from keras.utils import plot_model from IPython.display import SVG from keras.utils.vis_utils import model_to_dot import keras.backend as K

train_dir = "data/asl_alphabet_train/asl_alphabet_train/" test_dir = "data/asl_alphabet_test/asl_alphabet_test/" IMG_SIZE = 64 labels_map = {'A':0,'B':1,'C': 2, 'D': 3, 'E':4,'F':5,'G':6, 'H': 7, 'I':8, 'J':9,'K':10,'L':11, 'M': 12, 'N': 13, 'O':14, 'P':15,'Q':16, 'R': 17, 'S': 18, 'T':19, 'U':20,'V':21, 'W': 22, 'X': 23, 'Y':24, 'Z':25, 'del': 26, 'nothing': 27,'space':28}

classes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'nothing', 'space', 'del'] plt.figure(figsize=(11, 11)) for i in range (0,29): plt.subplot(7,7,i+1) plt.xticks([]) plt.yticks([]) path = train_dir + "/{0}/{0}1.jpg".format(classes[i]) img = plt.imread(path) plt.imshow(img) plt.xlabel(classes[i])

#dict_characters=labels_map #df = pd.DataFrame() #df["labels"]=y_train #lab = df['labels'] #dist = lab.value_counts() #plt.figure(figsize=(12,8)) #sns.countplot(lab) #print(dict_characters)

def create_train_data(): x_train = [] y_train = [] for folder_name in os.listdir(train_dir): label = labels_map[folder_name] for image_filename in tqdm(os.listdir(train_dir + folder_name)): path = os.path.join(train_dir,folder_name,image_filename) img = cv2.resize(cv2.imread(path, cv2.IMREAD_COLOR),(IMG_SIZE, IMG_SIZE )) x_train.append(np.array(img)) y_train.append(np.array(label)) print("Done creating train data") return x_train, y_train

train_dir

files = [f for f in tqdm(os.listdir(train_dir + 'A'))] len(files) files[0]

def create_test_data(): x_test = [] y_test = [] for folder_name in os.listdir(test_dir): label = labels_map[folder_name] for image_filename in tqdm(os.listdir(test_dir + folder_name)): path = os.path.join(test_dir,folder_name,image_filename) img = cv2.resize(cv2.imread(path, cv2.IMREAD_COLOR),(IMG_SIZE, IMG_SIZE )) x_test.append(np.array(img)) y_test.append(np.array(label)) print("Done creating test data") return x_test,y_test

x_train, y_train= create_train_data()

x_test,y_test = create_test_data()

#num_features = 2500 #num_classes = 29 #x_train, x_test = np.array(x_train, np.float32), np.array(x_test, np.float32) #x_train, x_test = x_train.reshape([-1, num_features]), x_test.reshape([-1, num_features]) #x_train, x_test = x_train / 255., x_test / 255.

x_train = x_train[0:1000] y_train = y_train[0:1000]

y_train[0]

def convert_to_one_hot(Y, C): Y = np.eye(C)[Y.reshape(-1)].T return Y

y_train_hot = convert_to_one_hot(np.array(y_train), 29).T

y_test_hot = convert_to_one_hot(np.array(y_test), 29).T

def identity_block(X, f, filters, stage, block): # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value. We'll need this later to add back to the main path. X_shortcut = X # First component of main path X = Conv2D(filters = F1, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2a', kernel_initializer = glorot_uniform(seed=0))(X) X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X) X = Activation('relu')(X) # Second component of main path X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1,1), padding = 'same', name = conv_name_base + '2b', kernel_initializer = glorot_uniform(seed=0))(X) X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path X = Conv2D(filters = F3, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2c', kernel_initializer = glorot_uniform(seed=0))(X) X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X) # Final step: Add shortcut value to main path, and pass it through a RELU activation X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X

# tf.compat.v1 instead of tf as functions are outdated tf.compat.v1.reset_default_graph() with tf.compat.v1.Session() as test: A_prev = tf.compat.v1.placeholder("float", [3, 4, 4, 6]) X = np.random.randn(3, 4, 4, 6) A = identity_block(A_prev, f = 2, filters = [2, 4, 6], stage = 1, block = 'a') test.run(tf.compat.v1.global_variables_initializer()) out = test.run([A], feed_dict={A_prev: X, K.learning_phase(): 0}) print("out = ", out[0][1][1][0])

def convolutional_block(X, f, filters, stage, block, s = 2): # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value X_shortcut = X ##### MAIN PATH ##### # First component of main path X = Conv2D(F1, (1, 1), strides = (s,s), name = conv_name_base + '2a', kernel_initializer = glorot_uniform(seed=0))(X) X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X) X = Activation('relu')(X) # Second component of main path X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X) ##### SHORTCUT PATH #### X_shortcut = Conv2D(F3, (1, 1), strides = (s,s), name = conv_name_base + '1', kernel_initializer = glorot_uniform(seed=0))(X_shortcut) X_shortcut = BatchNormalization(axis = 3, name = bn_name_base + '1')(X_shortcut) # Final step: Add shortcut value to main path, and pass it through a RELU activation X = Add()([X, X_shortcut]) X = Activation('relu')(X) return X

tf.compat.v1.reset_default_graph() with tf.compat.v1.Session() as test: A_prev = tf.compat.v1.placeholder("float", [3, 4, 4, 6]) X = np.random.randn(3, 4, 4, 6) A = convolutional_block(A_prev, f = 2, filters = [2, 4, 6], stage = 1, block = 'a') test.run(tf.compat.v1.global_variables_initializer()) out = test.run([A], feed_dict={A_prev: X, K.learning_phase(): 0}) print("out = ",out[0][1][1][0])

def ResNet50(input_shape = (64, 64, 3), classes = 29): # Define the input as a tensor with shape input_shape X_input = Input(input_shape) # Zero-Padding X = ZeroPadding2D((3, 3))(X_input) # Stage 1 X = Conv2D(64, (7, 7), strides = (2, 2), name = 'conv1', kernel_initializer = glorot_uniform(seed=0))(X) X = BatchNormalization(axis = 3, name = 'bn_conv1')(X) X = Activation('relu')(X) X = MaxPooling2D((3, 3), strides=(2, 2))(X) # Stage 2 X = convolutional_block(X, f = 3, filters = [64, 64, 256], stage = 2, block='a', s = 1) X = identity_block(X, 3, [64, 64, 256], stage=2, block='b') X = identity_block(X, 3, [64, 64, 256], stage=2, block='c') # Stage 3 X = convolutional_block(X, f = 3, filters = [128, 128, 512], stage = 3, block='a', s = 2) X = identity_block(X, 3, [128, 128, 512], stage=3, block='b') X = identity_block(X, 3, [128, 128, 512], stage=3, block='c') X = identity_block(X, 3, [128, 128, 512], stage=3, block='d') # Stage 4 X = convolutional_block(X, f = 3, filters = [256, 256, 1024], stage = 4, block='a', s = 2) X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b') X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c') X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d') X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e') X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f') # Stage 5 X = convolutional_block(X, f = 3, filters = [512, 512, 2048], stage = 5, block='a', s = 2) X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b') X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c') # AVGPOOL. X = AveragePooling2D((2, 2), name='avg_pool')(X) # output layer X = Flatten()(X) X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer = glorot_uniform(seed=0))(X) # Create model model = Model(inputs = X_input, outputs = X, name='ResNet50') return model

ROWS = 64 COLS = 64 CHANNELS = 3 CLASSES = 29

# build the model's graph model = ResNet50(input_shape =(ROWS, COLS, CHANNELS), classes = CLASSES)

#Now we need to configure the learning process by compiling the model. model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

preds = model.evaluate(X_test, Y_test) print ("Loss = " + str(preds[0])) print ("Test Accuracy = " + str(preds[1]))

model.summary()

plot_model(model, to_file='model.png') SVG(model_to_dot(model).create(prog='dot', format='svg'))

%matplotlib inline import matplotlib.pyplot as plt def display_image(num): label = y_train[num] plt.title('Label: %d' % (label)) image = x_train[num].reshape([IMG_SIZE,IMG_SIZE]) plt.imshow(image, cmap=plt.get_cmap('gray_r')) plt.show() display_image(1000)

def build_fc_model(): fc_model = tf.keras.Sequential([ # First define a Flatten layer tf.keras.layers.Flatten(), # '''TODO: Define the activation function for the first fully connected (Dense) layer.''' tf.keras.layers.Dense(128, activation= 'relu'), # '''TODO: Define the second Dense layer to output the classification probabilities''' tf.keras.layers.Dense(29, activation= 'softmax') ]) return fc_model model_sgd = build_fc_model()

'''TODO: Experiment with different optimizers and learning rates. How do these affect the accuracy of the trained model? Which optimizers and/or learning rates yield the best performance?''' model_sgd.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=1e-1), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

# Define the batch size and the number of epochs to use during training BATCH_SIZE = 64 EPOCHS = 5

print(model_sgd.summary())

'''TODO: Use the evaluate method to test the model!''' test_loss, test_acc = model_sgd.evaluate(x= x_test, y=y_test) '''TODO: Use the evaluate method to test the model!''' train_loss, train_acc = model_sgd.evaluate(x= x_train, y=y_train) print('Test accuracy:', test_acc) print('Train accuracy:', train_acc)

def build_classifier(optimizer = 'adam', learning_rate=1e-1): classifier = build_fc_model() if optimizer == 'SGD': optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate) if optimizer == 'RMSprop': optimizer = tf.keras.optimizers.RMSprop(learning_rate=learning_rate) if optimizer == 'Adagrad': optimizer = tf.keras.optimizers.Adagrad(learning_rate=learning_rate) if optimizer == 'Adadelta': optimizer = tf.keras.optimizers.Adadelta(learning_rate=learning_rate) if optimizer == 'Adam': optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate) if optimizer == 'Adamax': optimizer = tf.keras.optimizers.Adamax(learning_rate=learning_rate) if optimizer == 'Nadam': optimizer = tf.keras.optimizers.Nadam(learning_rate=learning_rate) classifier.compile(#optimizer=tf.keras.optimizers.SGD(learning_rate=1e-1), optimizer = optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) return classifier optimizer = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam'] learning_rate = [0.001, 0.01, 0.1, 0.2, 0.3] param_grid = dict(optimizer=optimizer, learning_rate = learning_rate) # create model my_wrapper_model = KerasClassifier(build_fn=build_classifier, epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=0) grid = GridSearchCV(estimator=my_wrapper_model, param_grid=param_grid, n_jobs=-1, cv=3, scoring='accuracy') start_time = time.time() grid_result = grid.fit(x_train, y_train) print(f'--- {(time.time() - start_time)/60:.3f} minutes ---')

# summarize results print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) means = grid_result.cv_results_['mean_test_score'] stds = grid_result.cv_results_['std_test_score'] params = grid_result.cv_results_['params'] #for mean, stdev, param in zip(means, stds, params): #print("%f (%f) with: %r" % (mean, stdev, param))

model_nadam = build_fc_model() model_nadam.compile(optimizer=tf.keras.optimizers.Nadam(learning_rate=0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy']) model_nadam.fit(x_test, y_test, batch_size=BATCH_SIZE, epochs=EPOCHS) '''TODO: Use the evaluate method to test the model!''' test_loss, test_acc = model_nadam.evaluate(x= x_test, y=y_test) print('Test accuracy:', test_acc)

def build_cnn_m