import os
os.system ("pip install keras==2.4.3")
import keras
print('The keras version is {}.'.format(keras.__version__))
The keras version is 2.4.3.
# IPython display functions
import IPython
from IPython.display import display, HTML, SVG, Image
# General Plotting
import matplotlib.pyplot as plt
plt.style.use('seaborn-paper')
plt.rcParams['figure.figsize'] = [10, 6] ## plot size
plt.rcParams['axes.linewidth'] = 2.0 #set the value globally
## notebook style and settings
display(HTML("<style>.container { width:90% !important; }</style>"))
display(HTML("<style>.output_png { display: table-cell; text-align: center; vertical-align: middle; } </style>"))
display(HTML("<style>.MathJax {font-size: 100%;}</style>"))
# For changing background color
def set_background(color):
script = ( "var cell = this.closest('.code_cell');" "var editor = cell.querySelector('.input_area');" "editor.style.background='{}';" "this.parentNode.removeChild(this)" ).format(color)
display(HTML('<img src onerror="{}">'.format(script)))
import os
import sys
import random
import numpy as np
import pandas as pd
from os import walk
# Metrics
from sklearn.metrics import *
# Keras library for deep learning
# https://keras.io/
import tensorflow as tf
import keras
from keras.datasets import mnist # MNIST Data set
from keras.models import Sequential # Model building
from keras.layers import * # Model layers
from keras.preprocessing.image import *
from sklearn.model_selection import train_test_split
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
def displayConfusionMatrix(confusionMatrix, precisionNegative, precisionPositive, recallNegative, recallPositive, title):
# Set font size for the plots. You can ignore this line.
PLOT_FONT_SIZE = 14
# Set plot size. Please ignore this line
plt.rcParams['figure.figsize'] = [5, 5]
# Transpose of confusion matrix to align the plot with the actual precision recall values. Please ignore this as well.
confusionMatrix = np.transpose(confusionMatrix)
# Plotting the confusion matrix
plt.imshow(confusionMatrix, interpolation='nearest',cmap=plt.cm.Blues, vmin=0, vmax=100)
# Setting plot properties. You should ignore everything from here on.
xticks = np.array([-0.5, 0, 1,1.5])
plt.gca().set_xticks(xticks)
plt.gca().set_yticks(xticks)
plt.gca().set_xticklabels(["", "Healthy\nRecall=" + str(recallNegative) , "Pneumonia\nRecall=" + str(recallPositive), ""], fontsize=PLOT_FONT_SIZE)
plt.gca().set_yticklabels(["", "Healthy\nPrecision=" + str(precisionNegative) , "Pneumonia\nPrecision=" + str(precisionPositive), ""], fontsize=PLOT_FONT_SIZE)
plt.ylabel("Predicted Class", fontsize=PLOT_FONT_SIZE)
plt.xlabel("Actual Class", fontsize=PLOT_FONT_SIZE)
plt.title(title, fontsize=PLOT_FONT_SIZE)
# Add text in heatmap boxes
for i in range(2):
for j in range(2):
text = plt.text(j, i, confusionMatrix[i][j], ha="center", va="center", color="white", size=15) ### size here is the size of text inside a single box in the heatmap
plt.show()
def calculateMetricsAndPrint(predictions, predictionsProbabilities, actualLabels):
# Convert label format from [0,1](label 1) and [1,0](label 0) into single integers: 1 and 0.
actualLabels = [item[1] for item in actualLabels]
# Get probabilities for the class with label 1. That is all we need to compute AUCs. We don't need probabilities for class 0.
predictionsProbabilities = [item[1] for item in predictionsProbabilities]
# Calculate metrics using scikit-learn functions. The round function is used to round the numbers up to 2 decimal points.
accuracy = round(accuracy_score(actualLabels, predictions) * 100, 2)
precisionNegative = round(precision_score(actualLabels, predictions, average = None)[0] * 100, 2)
precisionPositive = round(precision_score(actualLabels, predictions, average = None)[1] * 100, 2)
recallNegative = round(recall_score(actualLabels, predictions, average = None)[0] * 100, 2)
recallPositive = round(recall_score(actualLabels, predictions, average = None)[1] * 100, 2)
auc = round(roc_auc_score(actualLabels, predictionsProbabilities) * 100, 2)
confusionMatrix = confusion_matrix(actualLabels, predictions)
# Print metrics. .%2f prints a number upto 2 decimal points only.
print("------------------------------------------------------------------------")
print("Accuracy: %.2f\nPrecisionNegative: %.2f\nPrecisionPositive: %.2f\nRecallNegative: %.2f\nRecallPositive: %.2f\nAUC Score: %.2f" %
(accuracy, precisionNegative, precisionPositive, recallNegative, recallPositive, auc))
print("------------------------------------------------------------------------")
print("+ Printing confusion matrix...\n")
# Display confusion matrix
displayConfusionMatrix(confusionMatrix, precisionNegative, precisionPositive, recallNegative, recallPositive, "Confusion Matrix")
print("+ Printing ROC curve...\n")
# ROC Curve
plt.rcParams['figure.figsize'] = [16, 8]
FONT_SIZE = 16
falsePositiveRateDt, truePositiveRateDt, _ = roc_curve(actualLabels, predictionsProbabilities)
plt.plot(falsePositiveRateDt, truePositiveRateDt, linewidth = 5, color='black')
plt.xticks(fontsize=FONT_SIZE)
plt.yticks(fontsize=FONT_SIZE)
plt.xlabel("False Positive Rate", fontsize=FONT_SIZE)
plt.ylabel("True Positive Rate", fontsize=FONT_SIZE)
plt.show()
return auc
def calculateMetrics(predictions, predictionsProbabilities, actualLabels):
# Convert label format from [0,1](label 1) and [1,0](label 0) into single integers: 1 and 0.
actualLabels = [item[1] for item in actualLabels]
# Get probabilities for the class with label 1. That is all we need to compute AUCs. We don't need probabilities for class 0.
predictionsProbabilities = [item[1] for item in predictionsProbabilities]
# Calculate metrics using scikit-learn functions. The round function is used to round the numbers up to 2 decimal points.
try:
accuracy = round(accuracy_score(actualLabels, predictions) * 100, 2)
precisionNegative = round(precision_score(actualLabels, predictions, average = None)[0] * 100, 2)
precisionPositive = round(precision_score(actualLabels, predictions, average = None)[1] * 100, 2)
recallNegative = round(recall_score(actualLabels, predictions, average = None)[0] * 100, 2)
recallPositive = round(recall_score(actualLabels, predictions, average = None)[1] * 100, 2)
except:
print("An exception occurred but was caught.")
auc = round(roc_auc_score(actualLabels, predictionsProbabilities) * 100, 2)
return auc
def getKagglePredictions(model, kaggleData, filename):
print("+ Writing kaggle test results in : results/%s..." % filename)
predictions = model.predict(kaggleData)
predictionProbs = [item[1] for item in predictions]
# Store predictions for kaggle
outputFile = open("results/" + str(filename), "w")
outputFile.write("Id,Prediction\n")
for i in range(0, len(predictionProbs)):
outputFile.write(str(i + 1) + "," + str(predictionProbs[i]) + "\n")
outputFile.close()
def calculateClasswiseTopNAccuracy(actualLabels, predictionsProbs, TOP_N):
"""
TOP_N is the top n% predictions you want to use for each class
"""
discreteActualLabels = [1 if item[1] > item[0] else 0 for item in actualLabels]
discretePredictions = [1 if item[1] > item[0] else 0 for item in predictionsProbs]
predictionProbsTopNHealthy, predictionProbsTopNPneumonia = [item[0] for item in predictionsProbs], [item[1] for item in predictionsProbs]
predictionProbsTopNHealthy = list(reversed(sorted(predictionProbsTopNHealthy)))[:int(len(predictionProbsTopNHealthy) * TOP_N / 100)][-1]
predictionProbsTopNPneumonia = list(reversed(sorted(predictionProbsTopNPneumonia)))[:int(len(predictionProbsTopNPneumonia) * TOP_N / 100)][-1]
# Calculate accuracy for both classes
accuracyHealthy = []
accuracyPneumonia = []
for i in range(0, len(discretePredictions)):
if discretePredictions[i] == 1:
# Pneumonia
if predictionsProbs[i][1] > predictionProbsTopNPneumonia:
accuracyPneumonia.append(int(discreteActualLabels[i]) == 1)
else:
# Healthy
if predictionsProbs[i][0] > predictionProbsTopNHealthy:
accuracyHealthy.append(int(discreteActualLabels[i]) == 0)
accuracyHealthy = round((accuracyHealthy.count(True) * 100) / len(accuracyHealthy), 2)
accuracyPneumonia = round((accuracyPneumonia.count(True) * 100) / len(accuracyPneumonia), 2)
return accuracyHealthy, accuracyPneumonia
# Load normal images
normalImagesPath = "data/train/normal"
normalImageFiles = []
for(_,_,files) in walk(normalImagesPath):
normalImageFiles.extend(files)
normalImagesPath2 = "data/train/normal2"
for(_,_,files) in walk(normalImagesPath2):
normalImageFiles.extend(files)
print(len(normalImageFiles))
# Load pneumonia images
pneumoniaImagesPath = "data/train/pneumonia"
pneumoniaImageFiles = []
for(_,_,files) in walk(pneumoniaImagesPath):
pneumoniaImageFiles.extend(files)
random.shuffle(pneumoniaImageFiles)
pneumoniaImageFiles = pneumoniaImageFiles[:len(normalImageFiles)]
print("Normal X-ray images: %d\nPneumonia X-ray images: %d" % (len(normalImageFiles), len(pneumoniaImageFiles)))
1436
Normal X-ray images: 1436
Pneumonia X-ray images: 1436
imagesData = []
imagesLabels = []
for file in normalImageFiles:
fullPath = normalImagesPath + "/" + file
if os.path.exists(fullPath) == False:
continue
imageData = load_img(normalImagesPath + "/" + file, color_mode = "grayscale") # load_img function comes from keras library when we do "from keras.preprocessing.image import *"
imageArray = img_to_array(imageData) / 255.0
imagesData.append(imageArray)
imagesLabels.append(0)
for file in pneumoniaImageFiles:
fullPath = pneumoniaImagesPath + "/" + file
if os.path.exists(fullPath) == False:
continue
imageData = load_img(pneumoniaImagesPath + "/" + file, color_mode = "grayscale") # load_img function comes from keras library when we do "from keras.preprocessing.image import *"
imageArray = img_to_array(imageData) / 255.0
imagesData.append(imageArray)
imagesLabels.append(1)
imagesData = np.array(imagesData)
imagesLabels = keras.utils.to_categorical(imagesLabels)
print("Input data shape: %s" % (imagesData.shape,))
Input data shape: (2154, 256, 256, 1)
testImagesPath = "data/test/"
testImageFiles = []
for(_,_,files) in walk(testImagesPath):
testImageFiles.extend(files)
testImageFiles = list(sorted(testImageFiles))
kaggleTestImages = []
for file in testImageFiles:
fullPath = testImagesPath + "/" + file
if os.path.exists(fullPath) == False:
continue
imageData = load_img(testImagesPath + "/" + file, color_mode = "grayscale") # load_img function comes from keras library when we do "from keras.preprocessing.image import *"
imageArray = img_to_array(imageData) / 255.0
kaggleTestImages.append(imageArray)
kaggleTestImages = np.array(kaggleTestImages)
print("Number of test images: %d" % len(kaggleTestImages))
Number of test images: 200
def trainTestSplit(data, labels):
"""
80-20 train-test data split
"""
trainData, trainLabels, testData, testLabels = [], [], [], []
for i in range(0, len(data)):
if i % 5 == 0:
testData.append(data[i])
testLabels.append(labels[i])
else:
trainData.append(data[i])
trainLabels.append(labels[i])
return np.array(trainData), np.array(testData), np.array(trainLabels), np.array(testLabels)
# In our context, since we have a private test data on kaggle, our test data here would actually mean validation data. We will use results on this validation(test) data to see how our model would perform on the actual test data.
# Split data into 80% training and 20% testing
trainData, testData, trainLabels, testLabels = trainTestSplit(imagesData, imagesLabels)
def createParameterizedConvolutionalNeuralNetwork(trainImages, numLayers, numFilters, kernelSize, maxPooling, dropoutValue, learningRate, numClasses):
# Create model object
model = Sequential()
# Add the first layer with dropout
model.add(Conv2D(numFilters, kernel_size=(kernelSize, kernelSize),
activation='elu', padding = 'same',
input_shape=trainImages.shape[1:]))
model.add(MaxPooling2D(pool_size=(maxPooling, maxPooling)))
model.add(Dropout(dropoutValue))
while numLayers > 1:
model.add(Conv2D(numFilters, kernel_size=(kernelSize, kernelSize),
activation='elu', padding = 'same'))
model.add(MaxPooling2D(pool_size=(maxPooling, maxPooling)))
model.add(Dropout(dropoutValue))
numLayers = numLayers - 1
# Convolutional layers are done, adding the remaining stuff. Please note that after conv layers, you should always use a Flatten() layer.
model.add(Flatten())
model.add(Dense(64, activation='elu'))
model.add(BatchNormalization(momentum=0.99))
model.add(Dropout(dropoutValue))
model.add(Dense(numClasses, activation='softmax'))
# Compile model. You can skip this line.
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=learningRate),
metrics=['accuracy'])
# Return model
return model
def createNuancedConvolutionalNeuralNetwork(trainImages, numClasses):
"""
You should try to edit this model as much as you can. Try adding/removing layers, setting different parameters for different layers etc. You have complete control of the model and you should try different things to see what works and what does not.
"""
# Create model object
model = Sequential()
# Add the first layer with dropout
model.add(Conv2D(filters = 64, kernel_size=(5, 5),
activation='relu', padding = 'same',
input_shape=trainImages.shape[1:]))
model.add(MaxPooling2D(pool_size=(3, 3)))
model.add(Dropout(0.5))
# Second layer with diffefiltersrent parameters
model.add(Conv2D(filters = 32, kernel_size=(3, 3),
activation='relu', padding = 'same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
# Convolutional layers are done, adding the remaining stuff. Please note that after conv layers, you should always use a Flatten() layer.
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(numClasses, activation='softmax'))
# Compile model. You can skip this line.
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.0001),
metrics=['accuracy'])
# Return model
return model
set_background('#fce53a')
#####################################################################################################################################################
# Things you can change
#####################################################################################################################################################
# You can change all these parameters for different results. Please go to the following links to read more about each parameter:
# https://keras.io/preprocessing/image/
# https://www.pyimagesearch.com/2019/07/08/keras-imagedatagenerator-and-data-augmentation/
dataAugmentation = ImageDataGenerator(
rotation_range=11,
width_shift_range=0.07,
height_shift_range=0.07,
horizontal_flip=True,
vertical_flip=False,
shear_range=0.0,
zoom_range=0.37)
set_background('#fce53a')
#####################################################################################################################################################
# Things you can change
#####################################################################################################################################################
numLayers = 3 # Number of layers in the neural network
numFilters = 55 # Number of units in each layer
kernelSize = 22 # filter size of a single filter
dropoutValue = 0.29 # Dropout probability
maxPooling = 3 # Max pooling
numClasses = 2 # Don't change this value for pneumonia since we have 2 classes i.e we are trying to recognize a digit among 10 digits. But for any other data set, this should be changed
batchSize = 64 # How many images should a single batch contain
learningRate = 0.00004807526976 # How fast should the model learn
epochs = 200
# Number of epochs to train your model for
USE_DATA_AUGMENTATION = True # You can set it to false if you do not want to use data augmentation. We recommend trying both true and false.
#####################################################################################################################################################
# Please do not change this line.
dataAugmentation.fit(trainData) # Training the augmentor in case we set USE_DATA_AUGMENTATION to True.
# Create model
parameterizedModel = createParameterizedConvolutionalNeuralNetwork(trainData, numLayers, numFilters, kernelSize, maxPooling, dropoutValue, learningRate, numClasses = 2)
print("+ Your parameterized model has been created...")
+ Your parameterized model has been created...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py:375: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")
# You can create the other model with the following line
nonParameterizedModel = createNuancedConvolutionalNeuralNetwork(imagesData, numClasses = 2)
print("+ Your non parameterized model has been created...")
+ Your non parameterized model has been created...
#####################################################################################################################################################
# Things you can change
#####################################################################################################################################################
# Please assign model the deep learning model you want to use i.e parameterizedModel or nonParameterizedModel
model = parameterizedModel
bestAcc = 0.0
bestEpoch = 0
bestAccPredictions, bestAccPredictionProbabilities = [], []
print("+ Starting training. Each epoch can take about 2-5 minutes, hold tight!")
print("-----------------------------------------------------------------------\n")
for epoch in range(epochs):
#################################################### Model Training ###############################################################
if USE_DATA_AUGMENTATION == True:
# Use data augmentation in alternate epochs
if epoch % 2 == 0:
# Alternate between training with and without augmented data. Training just on the augmented data might not be the best way to go.
############ You can change the "epoch % 2" to some other integer value to train on top of the augmented data
############ after a certain number of epochs e.g "epoch % 3" will train on augmented data after every 2 epochs ############
model.fit_generator(dataAugmentation.flow(trainData, trainLabels, batch_size=batchSize),
steps_per_epoch=len(trainData) / batchSize, epochs=1, verbose = 2)
else:
model.fit(trainData, trainLabels, batch_size=batchSize, epochs=1, verbose = 2)
else:
# Do not use data augmentation
model.fit(trainData, trainLabels, batch_size=batchSize, epochs=1, verbose = 2)
#################################################### Model Testing ###############################################################
# Calculate test accuracy
accuracy = round(model.evaluate(testData, testLabels)[1] * 100, 3)
predictions = model.predict(testData)
#get epoch-level AUCs
AccPredictionProbabilities = model.predict(testData)
AccPredictions = [1 if item[1] > item[0] else 0 for item in AccPredictionProbabilities]
epochAUC = calculateMetrics(AccPredictions, AccPredictionProbabilities, testLabels)
print("+ Test accuracy at epoch %d is: %.2f " % (epoch, accuracy))
print("+ Test AUC at epoch %d is: %.3f " % (epoch, epochAUC))
if accuracy > bestAcc:
bestEpoch = epoch
bestAcc = accuracy
bestAccPredictions = [1 if item[1] > item[0] else 0 for item in predictions]
bestAccPredictionProbabilities = predictions
##################################### Store predictions for kaggle ###########################################################
kaggleResultsFileName = "epoch-" + str(epoch) + "-results.csv"
getKagglePredictions(model, kaggleTestImages, kaggleResultsFileName)
##############################################################################################################################
print('\n')
print("------------------------------------------------------------------------")
##################################################### Printing best metrics ##########################################################
# Get more metrics for the best performing epoch
print("\n*** Printing our best validation results that we obtained in epoch %d ..." % bestEpoch)
calculateMetricsAndPrint(bestAccPredictions, bestAccPredictionProbabilities, testLabels)
+ Writing kaggle test results in : results/epoch-76-results.csv...
27/27 - 20s - loss: 0.1206 - accuracy: 0.9541
14/14 [==============================] - 2s 173ms/step - loss: 0.1379 - accuracy: 0.9490
+ Test accuracy at epoch 77 is: 94.90
+ Test AUC at epoch 77 is: 98.700
+ Writing kaggle test results in : results/epoch-77-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.3136 - accuracy: 0.8845
14/14 [==============================] - 2s 173ms/step - loss: 0.1896 - accuracy: 0.9258
+ Test accuracy at epoch 78 is: 92.58
+ Test AUC at epoch 78 is: 97.820
+ Writing kaggle test results in : results/epoch-78-results.csv...
27/27 - 20s - loss: 0.1168 - accuracy: 0.9565
14/14 [==============================] - 2s 173ms/step - loss: 0.2558 - accuracy: 0.9002
+ Test accuracy at epoch 79 is: 90.02
+ Test AUC at epoch 79 is: 98.340
+ Writing kaggle test results in : results/epoch-79-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2973 - accuracy: 0.8810
14/14 [==============================] - 2s 174ms/step - loss: 0.1711 - accuracy: 0.9443
+ Test accuracy at epoch 80 is: 94.43
+ Test AUC at epoch 80 is: 98.070
+ Writing kaggle test results in : results/epoch-80-results.csv...
27/27 - 20s - loss: 0.1185 - accuracy: 0.9600
14/14 [==============================] - 2s 173ms/step - loss: 0.3505 - accuracy: 0.8770
+ Test accuracy at epoch 81 is: 87.70
+ Test AUC at epoch 81 is: 98.380
+ Writing kaggle test results in : results/epoch-81-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2789 - accuracy: 0.8967
14/14 [==============================] - 2s 173ms/step - loss: 0.2674 - accuracy: 0.9072
+ Test accuracy at epoch 82 is: 90.72
+ Test AUC at epoch 82 is: 98.250
+ Writing kaggle test results in : results/epoch-82-results.csv...
27/27 - 20s - loss: 0.1204 - accuracy: 0.9600
14/14 [==============================] - 2s 173ms/step - loss: 0.1887 - accuracy: 0.9304
+ Test accuracy at epoch 83 is: 93.04
+ Test AUC at epoch 83 is: 98.820
+ Writing kaggle test results in : results/epoch-83-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2880 - accuracy: 0.8955
14/14 [==============================] - 2s 174ms/step - loss: 1.7497 - accuracy: 0.5429
+ Test accuracy at epoch 84 is: 54.29
+ Test AUC at epoch 84 is: 98.780
+ Writing kaggle test results in : results/epoch-84-results.csv...
27/27 - 20s - loss: 0.1354 - accuracy: 0.9501
14/14 [==============================] - 2s 174ms/step - loss: 0.5774 - accuracy: 0.7958
+ Test accuracy at epoch 85 is: 79.58
+ Test AUC at epoch 85 is: 98.200
+ Writing kaggle test results in : results/epoch-85-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2969 - accuracy: 0.8799
14/14 [==============================] - 2s 173ms/step - loss: 0.1662 - accuracy: 0.9397
+ Test accuracy at epoch 86 is: 93.97
+ Test AUC at epoch 86 is: 98.320
+ Writing kaggle test results in : results/epoch-86-results.csv...
27/27 - 20s - loss: 0.1166 - accuracy: 0.9559
14/14 [==============================] - 2s 174ms/step - loss: 0.1445 - accuracy: 0.9420
+ Test accuracy at epoch 87 is: 94.20
+ Test AUC at epoch 87 is: 98.750
+ Writing kaggle test results in : results/epoch-87-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2891 - accuracy: 0.8944
14/14 [==============================] - 2s 174ms/step - loss: 0.1764 - accuracy: 0.9350
+ Test accuracy at epoch 88 is: 93.50
+ Test AUC at epoch 88 is: 98.720
+ Writing kaggle test results in : results/epoch-88-results.csv...
27/27 - 20s - loss: 0.1102 - accuracy: 0.9611
14/14 [==============================] - 2s 174ms/step - loss: 0.1577 - accuracy: 0.9420
+ Test accuracy at epoch 89 is: 94.20
+ Test AUC at epoch 89 is: 98.340
+ Writing kaggle test results in : results/epoch-89-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.3052 - accuracy: 0.8862
14/14 [==============================] - 2s 173ms/step - loss: 1.1022 - accuracy: 0.6241
+ Test accuracy at epoch 90 is: 62.41
+ Test AUC at epoch 90 is: 98.580
+ Writing kaggle test results in : results/epoch-90-results.csv...
27/27 - 20s - loss: 0.1268 - accuracy: 0.9541
14/14 [==============================] - 2s 173ms/step - loss: 0.1512 - accuracy: 0.9443
+ Test accuracy at epoch 91 is: 94.43
+ Test AUC at epoch 91 is: 98.560
+ Writing kaggle test results in : results/epoch-91-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2636 - accuracy: 0.9002
14/14 [==============================] - 2s 174ms/step - loss: 0.1670 - accuracy: 0.9397
+ Test accuracy at epoch 92 is: 93.97
+ Test AUC at epoch 92 is: 98.210
+ Writing kaggle test results in : results/epoch-92-results.csv...
27/27 - 20s - loss: 0.1119 - accuracy: 0.9571
14/14 [==============================] - 2s 174ms/step - loss: 0.6983 - accuracy: 0.7726
+ Test accuracy at epoch 93 is: 77.26
+ Test AUC at epoch 93 is: 98.270
+ Writing kaggle test results in : results/epoch-93-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2768 - accuracy: 0.8973
14/14 [==============================] - 3s 175ms/step - loss: 0.1601 - accuracy: 0.9443
+ Test accuracy at epoch 94 is: 94.43
+ Test AUC at epoch 94 is: 98.840
+ Writing kaggle test results in : results/epoch-94-results.csv...
27/27 - 20s - loss: 0.0899 - accuracy: 0.9704
14/14 [==============================] - 2s 173ms/step - loss: 0.4601 - accuracy: 0.8445
+ Test accuracy at epoch 95 is: 84.45
+ Test AUC at epoch 95 is: 99.030
+ Writing kaggle test results in : results/epoch-95-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.3227 - accuracy: 0.8787
14/14 [==============================] - 2s 173ms/step - loss: 1.2887 - accuracy: 0.6868
+ Test accuracy at epoch 96 is: 68.68
+ Test AUC at epoch 96 is: 96.430
+ Writing kaggle test results in : results/epoch-96-results.csv...
27/27 - 20s - loss: 0.0963 - accuracy: 0.9652
14/14 [==============================] - 2s 173ms/step - loss: 0.1265 - accuracy: 0.9536
+ Test accuracy at epoch 97 is: 95.36
+ Test AUC at epoch 97 is: 98.880
+ Writing kaggle test results in : results/epoch-97-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2944 - accuracy: 0.8857
14/14 [==============================] - 2s 175ms/step - loss: 0.1861 - accuracy: 0.9304
+ Test accuracy at epoch 98 is: 93.04
+ Test AUC at epoch 98 is: 98.030
+ Writing kaggle test results in : results/epoch-98-results.csv...
27/27 - 20s - loss: 0.1120 - accuracy: 0.9629
14/14 [==============================] - 2s 173ms/step - loss: 0.1720 - accuracy: 0.9281
+ Test accuracy at epoch 99 is: 92.81
+ Test AUC at epoch 99 is: 98.640
+ Writing kaggle test results in : results/epoch-99-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2946 - accuracy: 0.8880
14/14 [==============================] - 2s 173ms/step - loss: 0.5519 - accuracy: 0.8097
+ Test accuracy at epoch 100 is: 80.97
+ Test AUC at epoch 100 is: 99.010
+ Writing kaggle test results in : results/epoch-100-results.csv...
27/27 - 20s - loss: 0.0880 - accuracy: 0.9698
14/14 [==============================] - 2s 173ms/step - loss: 0.1811 - accuracy: 0.9304
+ Test accuracy at epoch 101 is: 93.04
+ Test AUC at epoch 101 is: 98.760
+ Writing kaggle test results in : results/epoch-101-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2801 - accuracy: 0.9031
14/14 [==============================] - 2s 173ms/step - loss: 0.1646 - accuracy: 0.9443
+ Test accuracy at epoch 102 is: 94.43
+ Test AUC at epoch 102 is: 98.270
+ Writing kaggle test results in : results/epoch-102-results.csv...
27/27 - 20s - loss: 0.0884 - accuracy: 0.9646
14/14 [==============================] - 2s 174ms/step - loss: 0.1835 - accuracy: 0.9327
+ Test accuracy at epoch 103 is: 93.27
+ Test AUC at epoch 103 is: 98.940
+ Writing kaggle test results in : results/epoch-103-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2759 - accuracy: 0.8979
14/14 [==============================] - 2s 174ms/step - loss: 1.6313 - accuracy: 0.5916
+ Test accuracy at epoch 104 is: 59.16
+ Test AUC at epoch 104 is: 98.900
+ Writing kaggle test results in : results/epoch-104-results.csv...
27/27 - 20s - loss: 0.1123 - accuracy: 0.9559
14/14 [==============================] - 2s 174ms/step - loss: 0.3270 - accuracy: 0.8840
+ Test accuracy at epoch 105 is: 88.40
+ Test AUC at epoch 105 is: 98.710
+ Writing kaggle test results in : results/epoch-105-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2789 - accuracy: 0.8909
14/14 [==============================] - 2s 173ms/step - loss: 0.1491 - accuracy: 0.9420
+ Test accuracy at epoch 106 is: 94.20
+ Test AUC at epoch 106 is: 98.380
+ Writing kaggle test results in : results/epoch-106-results.csv...
27/27 - 20s - loss: 0.0864 - accuracy: 0.9704
14/14 [==============================] - 2s 173ms/step - loss: 0.1743 - accuracy: 0.9466
+ Test accuracy at epoch 107 is: 94.66
+ Test AUC at epoch 107 is: 98.840
+ Writing kaggle test results in : results/epoch-107-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.3082 - accuracy: 0.8886
14/14 [==============================] - 2s 174ms/step - loss: 0.4088 - accuracy: 0.8422
+ Test accuracy at epoch 108 is: 84.22
+ Test AUC at epoch 108 is: 97.270
+ Writing kaggle test results in : results/epoch-108-results.csv...
27/27 - 20s - loss: 0.0946 - accuracy: 0.9634
14/14 [==============================] - 2s 174ms/step - loss: 0.2757 - accuracy: 0.8933
+ Test accuracy at epoch 109 is: 89.33
+ Test AUC at epoch 109 is: 98.680
+ Writing kaggle test results in : results/epoch-109-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2424 - accuracy: 0.9054
14/14 [==============================] - 2s 173ms/step - loss: 0.1752 - accuracy: 0.9327
+ Test accuracy at epoch 110 is: 93.27
+ Test AUC at epoch 110 is: 98.510
+ Writing kaggle test results in : results/epoch-110-results.csv...
27/27 - 20s - loss: 0.0951 - accuracy: 0.9687
14/14 [==============================] - 2s 174ms/step - loss: 0.1305 - accuracy: 0.9490
+ Test accuracy at epoch 111 is: 94.90
+ Test AUC at epoch 111 is: 98.800
+ Writing kaggle test results in : results/epoch-111-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2611 - accuracy: 0.9071
14/14 [==============================] - 2s 173ms/step - loss: 0.2272 - accuracy: 0.9095
+ Test accuracy at epoch 112 is: 90.95
+ Test AUC at epoch 112 is: 98.830
+ Writing kaggle test results in : results/epoch-112-results.csv...
27/27 - 20s - loss: 0.0917 - accuracy: 0.9646
14/14 [==============================] - 2s 174ms/step - loss: 0.2556 - accuracy: 0.9165
+ Test accuracy at epoch 113 is: 91.65
+ Test AUC at epoch 113 is: 98.770
+ Writing kaggle test results in : results/epoch-113-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2502 - accuracy: 0.8915
14/14 [==============================] - 2s 173ms/step - loss: 0.1663 - accuracy: 0.9397
+ Test accuracy at epoch 114 is: 93.97
+ Test AUC at epoch 114 is: 97.860
+ Writing kaggle test results in : results/epoch-114-results.csv...
27/27 - 20s - loss: 0.0834 - accuracy: 0.9704
14/14 [==============================] - 2s 173ms/step - loss: 0.1762 - accuracy: 0.9420
+ Test accuracy at epoch 115 is: 94.20
+ Test AUC at epoch 115 is: 98.450
+ Writing kaggle test results in : results/epoch-115-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2870 - accuracy: 0.8961
14/14 [==============================] - 2s 173ms/step - loss: 0.1997 - accuracy: 0.9281
+ Test accuracy at epoch 116 is: 92.81
+ Test AUC at epoch 116 is: 98.660
+ Writing kaggle test results in : results/epoch-116-results.csv...
27/27 - 20s - loss: 0.0916 - accuracy: 0.9710
14/14 [==============================] - 2s 173ms/step - loss: 0.2209 - accuracy: 0.9026
+ Test accuracy at epoch 117 is: 90.25
+ Test AUC at epoch 117 is: 98.690
+ Writing kaggle test results in : results/epoch-117-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2699 - accuracy: 0.9008
14/14 [==============================] - 2s 173ms/step - loss: 0.2024 - accuracy: 0.9281
+ Test accuracy at epoch 118 is: 92.81
+ Test AUC at epoch 118 is: 98.060
+ Writing kaggle test results in : results/epoch-118-results.csv...
27/27 - 20s - loss: 0.0766 - accuracy: 0.9750
14/14 [==============================] - 2s 173ms/step - loss: 0.1585 - accuracy: 0.9420
+ Test accuracy at epoch 119 is: 94.20
+ Test AUC at epoch 119 is: 98.920
+ Writing kaggle test results in : results/epoch-119-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2570 - accuracy: 0.9042
14/14 [==============================] - 2s 173ms/step - loss: 0.1986 - accuracy: 0.9350
+ Test accuracy at epoch 120 is: 93.50
+ Test AUC at epoch 120 is: 98.530
+ Writing kaggle test results in : results/epoch-120-results.csv...
27/27 - 20s - loss: 0.0939 - accuracy: 0.9675
14/14 [==============================] - 2s 173ms/step - loss: 0.3417 - accuracy: 0.8538
+ Test accuracy at epoch 121 is: 85.38
+ Test AUC at epoch 121 is: 98.560
+ Writing kaggle test results in : results/epoch-121-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2570 - accuracy: 0.9013
14/14 [==============================] - 2s 174ms/step - loss: 0.2301 - accuracy: 0.9118
+ Test accuracy at epoch 122 is: 91.18
+ Test AUC at epoch 122 is: 97.570
+ Writing kaggle test results in : results/epoch-122-results.csv...
27/27 - 20s - loss: 0.0875 - accuracy: 0.9675
14/14 [==============================] - 2s 173ms/step - loss: 0.2402 - accuracy: 0.9072
+ Test accuracy at epoch 123 is: 90.72
+ Test AUC at epoch 123 is: 98.600
+ Writing kaggle test results in : results/epoch-123-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2581 - accuracy: 0.9025
14/14 [==============================] - 2s 173ms/step - loss: 0.9668 - accuracy: 0.7239
+ Test accuracy at epoch 124 is: 72.39
+ Test AUC at epoch 124 is: 96.710
+ Writing kaggle test results in : results/epoch-124-results.csv...
27/27 - 20s - loss: 0.0977 - accuracy: 0.9629
14/14 [==============================] - 2s 174ms/step - loss: 1.5064 - accuracy: 0.5870
+ Test accuracy at epoch 125 is: 58.70
+ Test AUC at epoch 125 is: 98.600
+ Writing kaggle test results in : results/epoch-125-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2699 - accuracy: 0.8868
14/14 [==============================] - 2s 175ms/step - loss: 0.1834 - accuracy: 0.9281
+ Test accuracy at epoch 126 is: 92.81
+ Test AUC at epoch 126 is: 98.040
+ Writing kaggle test results in : results/epoch-126-results.csv...
27/27 - 20s - loss: 0.0847 - accuracy: 0.9698
14/14 [==============================] - 2s 173ms/step - loss: 0.1527 - accuracy: 0.9443
+ Test accuracy at epoch 127 is: 94.43
+ Test AUC at epoch 127 is: 98.840
+ Writing kaggle test results in : results/epoch-127-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2489 - accuracy: 0.9042
14/14 [==============================] - 2s 173ms/step - loss: 1.6857 - accuracy: 0.6798
+ Test accuracy at epoch 128 is: 67.98
+ Test AUC at epoch 128 is: 96.460
+ Writing kaggle test results in : results/epoch-128-results.csv...
27/27 - 20s - loss: 0.0817 - accuracy: 0.9733
14/14 [==============================] - 2s 174ms/step - loss: 0.2056 - accuracy: 0.9165
+ Test accuracy at epoch 129 is: 91.65
+ Test AUC at epoch 129 is: 98.870
+ Writing kaggle test results in : results/epoch-129-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2593 - accuracy: 0.9089
14/14 [==============================] - 2s 173ms/step - loss: 0.5554 - accuracy: 0.8074
+ Test accuracy at epoch 130 is: 80.74
+ Test AUC at epoch 130 is: 97.600
+ Writing kaggle test results in : results/epoch-130-results.csv...
27/27 - 20s - loss: 0.0752 - accuracy: 0.9762
14/14 [==============================] - 2s 173ms/step - loss: 0.2186 - accuracy: 0.9304
+ Test accuracy at epoch 131 is: 93.04
+ Test AUC at epoch 131 is: 98.890
+ Writing kaggle test results in : results/epoch-131-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2578 - accuracy: 0.9008
14/14 [==============================] - 2s 173ms/step - loss: 0.4357 - accuracy: 0.8283
+ Test accuracy at epoch 132 is: 82.83
+ Test AUC at epoch 132 is: 97.800
+ Writing kaggle test results in : results/epoch-132-results.csv...
27/27 - 20s - loss: 0.0654 - accuracy: 0.9779
14/14 [==============================] - 2s 173ms/step - loss: 0.1527 - accuracy: 0.9490
+ Test accuracy at epoch 133 is: 94.90
+ Test AUC at epoch 133 is: 98.980
+ Writing kaggle test results in : results/epoch-133-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2459 - accuracy: 0.9071
14/14 [==============================] - 2s 173ms/step - loss: 0.4112 - accuracy: 0.8492
+ Test accuracy at epoch 134 is: 84.92
+ Test AUC at epoch 134 is: 97.640
+ Writing kaggle test results in : results/epoch-134-results.csv...
27/27 - 20s - loss: 0.0708 - accuracy: 0.9762
14/14 [==============================] - 2s 173ms/step - loss: 0.1557 - accuracy: 0.9466
+ Test accuracy at epoch 135 is: 94.66
+ Test AUC at epoch 135 is: 98.900
+ Writing kaggle test results in : results/epoch-135-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2522 - accuracy: 0.9071
14/14 [==============================] - 2s 174ms/step - loss: 0.2281 - accuracy: 0.9188
+ Test accuracy at epoch 136 is: 91.88
+ Test AUC at epoch 136 is: 98.810
+ Writing kaggle test results in : results/epoch-136-results.csv...
27/27 - 20s - loss: 0.0687 - accuracy: 0.9750
14/14 [==============================] - 2s 175ms/step - loss: 0.1566 - accuracy: 0.9397
+ Test accuracy at epoch 137 is: 93.97
+ Test AUC at epoch 137 is: 98.790
+ Writing kaggle test results in : results/epoch-137-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2508 - accuracy: 0.9054
14/14 [==============================] - 2s 173ms/step - loss: 0.1774 - accuracy: 0.9443
+ Test accuracy at epoch 138 is: 94.43
+ Test AUC at epoch 138 is: 98.350
+ Writing kaggle test results in : results/epoch-138-results.csv...
27/27 - 20s - loss: 0.0674 - accuracy: 0.9803
14/14 [==============================] - 2s 174ms/step - loss: 0.1285 - accuracy: 0.9490
+ Test accuracy at epoch 139 is: 94.90
+ Test AUC at epoch 139 is: 98.870
+ Writing kaggle test results in : results/epoch-139-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2493 - accuracy: 0.9095
14/14 [==============================] - 2s 174ms/step - loss: 0.1973 - accuracy: 0.9211
+ Test accuracy at epoch 140 is: 92.11
+ Test AUC at epoch 140 is: 98.190
+ Writing kaggle test results in : results/epoch-140-results.csv...
27/27 - 20s - loss: 0.0709 - accuracy: 0.9797
14/14 [==============================] - 2s 173ms/step - loss: 0.1416 - accuracy: 0.9490
+ Test accuracy at epoch 141 is: 94.90
+ Test AUC at epoch 141 is: 98.570
+ Writing kaggle test results in : results/epoch-141-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2436 - accuracy: 0.9019
14/14 [==============================] - 2s 173ms/step - loss: 0.5555 - accuracy: 0.8167
+ Test accuracy at epoch 142 is: 81.67
+ Test AUC at epoch 142 is: 96.980
+ Writing kaggle test results in : results/epoch-142-results.csv...
27/27 - 20s - loss: 0.0652 - accuracy: 0.9826
14/14 [==============================] - 2s 174ms/step - loss: 0.2421 - accuracy: 0.9165
+ Test accuracy at epoch 143 is: 91.65
+ Test AUC at epoch 143 is: 98.890
+ Writing kaggle test results in : results/epoch-143-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2297 - accuracy: 0.9158
14/14 [==============================] - 2s 173ms/step - loss: 0.2092 - accuracy: 0.9327
+ Test accuracy at epoch 144 is: 93.27
+ Test AUC at epoch 144 is: 98.730
+ Writing kaggle test results in : results/epoch-144-results.csv...
27/27 - 20s - loss: 0.0638 - accuracy: 0.9779
14/14 [==============================] - 2s 174ms/step - loss: 0.1276 - accuracy: 0.9513
+ Test accuracy at epoch 145 is: 95.13
+ Test AUC at epoch 145 is: 98.880
+ Writing kaggle test results in : results/epoch-145-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2374 - accuracy: 0.9106
14/14 [==============================] - 2s 173ms/step - loss: 0.1319 - accuracy: 0.9513
+ Test accuracy at epoch 146 is: 95.13
+ Test AUC at epoch 146 is: 98.840
+ Writing kaggle test results in : results/epoch-146-results.csv...
27/27 - 20s - loss: 0.0606 - accuracy: 0.9826
14/14 [==============================] - 2s 174ms/step - loss: 0.4608 - accuracy: 0.8469
+ Test accuracy at epoch 147 is: 84.69
+ Test AUC at epoch 147 is: 98.660
+ Writing kaggle test results in : results/epoch-147-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2484 - accuracy: 0.9042
14/14 [==============================] - 2s 173ms/step - loss: 0.1466 - accuracy: 0.9466
+ Test accuracy at epoch 148 is: 94.66
+ Test AUC at epoch 148 is: 98.610
+ Writing kaggle test results in : results/epoch-148-results.csv...
27/27 - 20s - loss: 0.0578 - accuracy: 0.9849
14/14 [==============================] - 2s 174ms/step - loss: 0.1239 - accuracy: 0.9513
+ Test accuracy at epoch 149 is: 95.13
+ Test AUC at epoch 149 is: 98.870
+ Writing kaggle test results in : results/epoch-149-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2548 - accuracy: 0.9095
14/14 [==============================] - 2s 173ms/step - loss: 0.2062 - accuracy: 0.9234
+ Test accuracy at epoch 150 is: 92.34
+ Test AUC at epoch 150 is: 98.550
+ Writing kaggle test results in : results/epoch-150-results.csv...
27/27 - 20s - loss: 0.0583 - accuracy: 0.9826
14/14 [==============================] - 2s 173ms/step - loss: 0.2682 - accuracy: 0.8979
+ Test accuracy at epoch 151 is: 89.79
+ Test AUC at epoch 151 is: 99.010
+ Writing kaggle test results in : results/epoch-151-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2425 - accuracy: 0.9100
14/14 [==============================] - 2s 173ms/step - loss: 0.2412 - accuracy: 0.9258
+ Test accuracy at epoch 152 is: 92.58
+ Test AUC at epoch 152 is: 97.710
+ Writing kaggle test results in : results/epoch-152-results.csv...
27/27 - 20s - loss: 0.0657 - accuracy: 0.9779
14/14 [==============================] - 2s 173ms/step - loss: 0.1247 - accuracy: 0.9559
+ Test accuracy at epoch 153 is: 95.59
+ Test AUC at epoch 153 is: 98.850
+ Writing kaggle test results in : results/epoch-153-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2409 - accuracy: 0.9013
14/14 [==============================] - 2s 173ms/step - loss: 0.1533 - accuracy: 0.9420
+ Test accuracy at epoch 154 is: 94.20
+ Test AUC at epoch 154 is: 98.930
+ Writing kaggle test results in : results/epoch-154-results.csv...
27/27 - 20s - loss: 0.0597 - accuracy: 0.9832
14/14 [==============================] - 2s 173ms/step - loss: 0.1477 - accuracy: 0.9513
+ Test accuracy at epoch 155 is: 95.13
+ Test AUC at epoch 155 is: 98.710
+ Writing kaggle test results in : results/epoch-155-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2458 - accuracy: 0.9199
14/14 [==============================] - 2s 173ms/step - loss: 0.1661 - accuracy: 0.9350
+ Test accuracy at epoch 156 is: 93.50
+ Test AUC at epoch 156 is: 98.910
+ Writing kaggle test results in : results/epoch-156-results.csv...
27/27 - 20s - loss: 0.0625 - accuracy: 0.9808
14/14 [==============================] - 2s 174ms/step - loss: 0.1930 - accuracy: 0.9327
+ Test accuracy at epoch 157 is: 93.27
+ Test AUC at epoch 157 is: 98.610
+ Writing kaggle test results in : results/epoch-157-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2257 - accuracy: 0.9147
14/14 [==============================] - 2s 173ms/step - loss: 0.9946 - accuracy: 0.7425
+ Test accuracy at epoch 158 is: 74.25
+ Test AUC at epoch 158 is: 97.580
+ Writing kaggle test results in : results/epoch-158-results.csv...
27/27 - 20s - loss: 0.0665 - accuracy: 0.9774
14/14 [==============================] - 2s 173ms/step - loss: 0.3552 - accuracy: 0.8585
+ Test accuracy at epoch 159 is: 85.85
+ Test AUC at epoch 159 is: 98.960
+ Writing kaggle test results in : results/epoch-159-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2514 - accuracy: 0.9031
14/14 [==============================] - 2s 173ms/step - loss: 0.2191 - accuracy: 0.9234
+ Test accuracy at epoch 160 is: 92.34
+ Test AUC at epoch 160 is: 97.950
+ Writing kaggle test results in : results/epoch-160-results.csv...
27/27 - 20s - loss: 0.0556 - accuracy: 0.9826
14/14 [==============================] - 2s 174ms/step - loss: 0.1311 - accuracy: 0.9606
+ Test accuracy at epoch 161 is: 96.06
+ Test AUC at epoch 161 is: 98.900
+ Writing kaggle test results in : results/epoch-161-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2366 - accuracy: 0.9095
14/14 [==============================] - 2s 174ms/step - loss: 0.5509 - accuracy: 0.8190
+ Test accuracy at epoch 162 is: 81.90
+ Test AUC at epoch 162 is: 98.910
+ Writing kaggle test results in : results/epoch-162-results.csv...
27/27 - 20s - loss: 0.0615 - accuracy: 0.9803
14/14 [==============================] - 2s 173ms/step - loss: 0.1434 - accuracy: 0.9466
+ Test accuracy at epoch 163 is: 94.66
+ Test AUC at epoch 163 is: 98.880
+ Writing kaggle test results in : results/epoch-163-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2291 - accuracy: 0.9124
14/14 [==============================] - 2s 173ms/step - loss: 0.1366 - accuracy: 0.9582
+ Test accuracy at epoch 164 is: 95.82
+ Test AUC at epoch 164 is: 98.660
+ Writing kaggle test results in : results/epoch-164-results.csv...
27/27 - 20s - loss: 0.0530 - accuracy: 0.9849
14/14 [==============================] - 2s 174ms/step - loss: 0.1319 - accuracy: 0.9536
+ Test accuracy at epoch 165 is: 95.36
+ Test AUC at epoch 165 is: 98.890
+ Writing kaggle test results in : results/epoch-165-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2541 - accuracy: 0.9176
14/14 [==============================] - 2s 173ms/step - loss: 1.5303 - accuracy: 0.6891
+ Test accuracy at epoch 166 is: 68.91
+ Test AUC at epoch 166 is: 95.780
+ Writing kaggle test results in : results/epoch-166-results.csv...
27/27 - 20s - loss: 0.0661 - accuracy: 0.9774
14/14 [==============================] - 2s 174ms/step - loss: 0.1123 - accuracy: 0.9536
+ Test accuracy at epoch 167 is: 95.36
+ Test AUC at epoch 167 is: 99.000
+ Writing kaggle test results in : results/epoch-167-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2511 - accuracy: 0.9060
14/14 [==============================] - 2s 173ms/step - loss: 0.3162 - accuracy: 0.8747
+ Test accuracy at epoch 168 is: 87.47
+ Test AUC at epoch 168 is: 98.140
+ Writing kaggle test results in : results/epoch-168-results.csv...
27/27 - 20s - loss: 0.0571 - accuracy: 0.9814
14/14 [==============================] - 2s 173ms/step - loss: 0.2552 - accuracy: 0.9095
+ Test accuracy at epoch 169 is: 90.95
+ Test AUC at epoch 169 is: 98.940
+ Writing kaggle test results in : results/epoch-169-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2257 - accuracy: 0.9089
14/14 [==============================] - 2s 173ms/step - loss: 0.1438 - accuracy: 0.9466
+ Test accuracy at epoch 170 is: 94.66
+ Test AUC at epoch 170 is: 98.530
+ Writing kaggle test results in : results/epoch-170-results.csv...
27/27 - 20s - loss: 0.0537 - accuracy: 0.9814
14/14 [==============================] - 2s 173ms/step - loss: 0.1568 - accuracy: 0.9350
+ Test accuracy at epoch 171 is: 93.50
+ Test AUC at epoch 171 is: 98.760
+ Writing kaggle test results in : results/epoch-171-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2284 - accuracy: 0.9112
14/14 [==============================] - 2s 174ms/step - loss: 0.6270 - accuracy: 0.7935
+ Test accuracy at epoch 172 is: 79.35
+ Test AUC at epoch 172 is: 97.340
+ Writing kaggle test results in : results/epoch-172-results.csv...
27/27 - 20s - loss: 0.0605 - accuracy: 0.9814
14/14 [==============================] - 2s 173ms/step - loss: 0.1197 - accuracy: 0.9606
+ Test accuracy at epoch 173 is: 96.06
+ Test AUC at epoch 173 is: 98.980
+ Writing kaggle test results in : results/epoch-173-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2285 - accuracy: 0.9100
14/14 [==============================] - 2s 173ms/step - loss: 0.4233 - accuracy: 0.8631
+ Test accuracy at epoch 174 is: 86.31
+ Test AUC at epoch 174 is: 98.780
+ Writing kaggle test results in : results/epoch-174-results.csv...
27/27 - 20s - loss: 0.0560 - accuracy: 0.9832
14/14 [==============================] - 2s 174ms/step - loss: 0.1227 - accuracy: 0.9536
+ Test accuracy at epoch 175 is: 95.36
+ Test AUC at epoch 175 is: 98.890
+ Writing kaggle test results in : results/epoch-175-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2160 - accuracy: 0.9263
14/14 [==============================] - 2s 173ms/step - loss: 0.1848 - accuracy: 0.9281
+ Test accuracy at epoch 176 is: 92.81
+ Test AUC at epoch 176 is: 98.950
+ Writing kaggle test results in : results/epoch-176-results.csv...
27/27 - 20s - loss: 0.0395 - accuracy: 0.9896
14/14 [==============================] - 2s 172ms/step - loss: 0.1553 - accuracy: 0.9559
+ Test accuracy at epoch 177 is: 95.59
+ Test AUC at epoch 177 is: 98.850
+ Writing kaggle test results in : results/epoch-177-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2102 - accuracy: 0.9153
14/14 [==============================] - 2s 173ms/step - loss: 0.2903 - accuracy: 0.9026
+ Test accuracy at epoch 178 is: 90.25
+ Test AUC at epoch 178 is: 98.960
+ Writing kaggle test results in : results/epoch-178-results.csv...
27/27 - 20s - loss: 0.0451 - accuracy: 0.9884
14/14 [==============================] - 2s 173ms/step - loss: 0.1227 - accuracy: 0.9536
+ Test accuracy at epoch 179 is: 95.36
+ Test AUC at epoch 179 is: 98.890
+ Writing kaggle test results in : results/epoch-179-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2152 - accuracy: 0.9135
14/14 [==============================] - 2s 173ms/step - loss: 0.1739 - accuracy: 0.9513
+ Test accuracy at epoch 180 is: 95.13
+ Test AUC at epoch 180 is: 98.460
+ Writing kaggle test results in : results/epoch-180-results.csv...
27/27 - 20s - loss: 0.0464 - accuracy: 0.9843
14/14 [==============================] - 2s 174ms/step - loss: 0.1212 - accuracy: 0.9559
+ Test accuracy at epoch 181 is: 95.59
+ Test AUC at epoch 181 is: 98.950
+ Writing kaggle test results in : results/epoch-181-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2366 - accuracy: 0.9129
14/14 [==============================] - 2s 173ms/step - loss: 0.1357 - accuracy: 0.9513
+ Test accuracy at epoch 182 is: 95.13
+ Test AUC at epoch 182 is: 98.670
+ Writing kaggle test results in : results/epoch-182-results.csv...
27/27 - 20s - loss: 0.0483 - accuracy: 0.9855
14/14 [==============================] - 2s 173ms/step - loss: 0.2126 - accuracy: 0.9281
+ Test accuracy at epoch 183 is: 92.81
+ Test AUC at epoch 183 is: 99.010
+ Writing kaggle test results in : results/epoch-183-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2287 - accuracy: 0.9141
14/14 [==============================] - 2s 173ms/step - loss: 0.4107 - accuracy: 0.8237
+ Test accuracy at epoch 184 is: 82.37
+ Test AUC at epoch 184 is: 98.180
+ Writing kaggle test results in : results/epoch-184-results.csv...
27/27 - 20s - loss: 0.0557 - accuracy: 0.9843
14/14 [==============================] - 2s 174ms/step - loss: 0.1259 - accuracy: 0.9536
+ Test accuracy at epoch 185 is: 95.36
+ Test AUC at epoch 185 is: 98.740
+ Writing kaggle test results in : results/epoch-185-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2166 - accuracy: 0.9176
14/14 [==============================] - 2s 173ms/step - loss: 0.2052 - accuracy: 0.9397
+ Test accuracy at epoch 186 is: 93.97
+ Test AUC at epoch 186 is: 97.740
+ Writing kaggle test results in : results/epoch-186-results.csv...
27/27 - 20s - loss: 0.0443 - accuracy: 0.9884
14/14 [==============================] - 2s 173ms/step - loss: 0.1677 - accuracy: 0.9350
+ Test accuracy at epoch 187 is: 93.50
+ Test AUC at epoch 187 is: 98.980
+ Writing kaggle test results in : results/epoch-187-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2499 - accuracy: 0.9129
14/14 [==============================] - 2s 173ms/step - loss: 0.1416 - accuracy: 0.9443
+ Test accuracy at epoch 188 is: 94.43
+ Test AUC at epoch 188 is: 98.660
+ Writing kaggle test results in : results/epoch-188-results.csv...
27/27 - 20s - loss: 0.0364 - accuracy: 0.9919
14/14 [==============================] - 2s 174ms/step - loss: 0.1324 - accuracy: 0.9466
+ Test accuracy at epoch 189 is: 94.66
+ Test AUC at epoch 189 is: 98.850
+ Writing kaggle test results in : results/epoch-189-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2099 - accuracy: 0.9158
14/14 [==============================] - 2s 174ms/step - loss: 0.1451 - accuracy: 0.9513
+ Test accuracy at epoch 190 is: 95.13
+ Test AUC at epoch 190 is: 98.500
+ Writing kaggle test results in : results/epoch-190-results.csv...
27/27 - 20s - loss: 0.0391 - accuracy: 0.9913
14/14 [==============================] - 2s 174ms/step - loss: 0.1371 - accuracy: 0.9536
+ Test accuracy at epoch 191 is: 95.36
+ Test AUC at epoch 191 is: 99.080
+ Writing kaggle test results in : results/epoch-191-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2301 - accuracy: 0.9153
14/14 [==============================] - 2s 174ms/step - loss: 0.3031 - accuracy: 0.8910
+ Test accuracy at epoch 192 is: 89.09
+ Test AUC at epoch 192 is: 98.510
+ Writing kaggle test results in : results/epoch-192-results.csv...
27/27 - 20s - loss: 0.0428 - accuracy: 0.9878
14/14 [==============================] - 2s 173ms/step - loss: 0.1495 - accuracy: 0.9490
+ Test accuracy at epoch 193 is: 94.90
+ Test AUC at epoch 193 is: 98.770
+ Writing kaggle test results in : results/epoch-193-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2051 - accuracy: 0.9257
14/14 [==============================] - 2s 174ms/step - loss: 0.1475 - accuracy: 0.9374
+ Test accuracy at epoch 194 is: 93.73
+ Test AUC at epoch 194 is: 98.580
+ Writing kaggle test results in : results/epoch-194-results.csv...
27/27 - 20s - loss: 0.0506 - accuracy: 0.9843
14/14 [==============================] - 2s 173ms/step - loss: 0.1277 - accuracy: 0.9513
+ Test accuracy at epoch 195 is: 95.13
+ Test AUC at epoch 195 is: 98.880
+ Writing kaggle test results in : results/epoch-195-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2528 - accuracy: 0.9129
14/14 [==============================] - 2s 174ms/step - loss: 0.9000 - accuracy: 0.7494
+ Test accuracy at epoch 196 is: 74.94
+ Test AUC at epoch 196 is: 97.440
+ Writing kaggle test results in : results/epoch-196-results.csv...
27/27 - 20s - loss: 0.0476 - accuracy: 0.9861
14/14 [==============================] - 2s 173ms/step - loss: 0.2341 - accuracy: 0.9072
+ Test accuracy at epoch 197 is: 90.72
+ Test AUC at epoch 197 is: 99.110
+ Writing kaggle test results in : results/epoch-197-results.csv...
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:1969: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
warnings.warn('`Model.fit_generator` is deprecated and '
26/26 - 21s - loss: 0.2584 - accuracy: 0.9054
14/14 [==============================] - 2s 173ms/step - loss: 0.3018 - accuracy: 0.9118
+ Test accuracy at epoch 198 is: 91.18
+ Test AUC at epoch 198 is: 98.760
+ Writing kaggle test results in : results/epoch-198-results.csv...
27/27 - 20s - loss: 0.0450 - accuracy: 0.9878
14/14 [==============================] - 2s 173ms/step - loss: 0.1473 - accuracy: 0.9490
+ Test accuracy at epoch 199 is: 94.90
+ Test AUC at epoch 199 is: 98.830
+ Writing kaggle test results in : results/epoch-199-results.csv...
------------------------------------------------------------------------
*** Printing our best validation results that we obtained in epoch 161 ...
------------------------------------------------------------------------
Accuracy: 96.06
PrecisionNegative: 96.35
PrecisionPositive: 95.92
RecallNegative: 91.67
RecallPositive: 98.26
AUC Score: 98.90
------------------------------------------------------------------------
+ Printing confusion matrix...
+ Printing ROC curve...
################################## You can change values inside the following list ###########################
topNValues = [5, 10, 20, 30,40, 50, 60, 70, 80, 90,100]
##############################################################################################################
accuraciesHealthy, accuraciesPneumonia = [], []
for topn in topNValues:
accuracyHealthy, accuracyPneumonia = calculateClasswiseTopNAccuracy(testLabels, bestAccPredictionProbabilities, topn)
accuraciesHealthy.append(accuracyHealthy)
accuraciesPneumonia.append(accuracyPneumonia)
print("+ Accuracy for top %d percent predictions for healthy: %.2f, pneumonia: %.2f" % (topn, accuracyHealthy, accuracyPneumonia))
# Plot results
x = np.arange(len(accuraciesHealthy))
plt.plot(x, accuraciesHealthy, linewidth = 3, color = '#e01111')
scatterHealthy = plt.scatter(x, accuraciesHealthy, marker = 's', s = 100, color = '#e01111')
plt.plot(x, accuraciesPneumonia, linewidth = 3, color = '#0072ff')
scatterPneumonia = plt.scatter(x, accuraciesPneumonia, marker = 'o', s = 100, color = '#0072ff')
plt.xticks(x, topNValues, fontsize = 15)
plt.yticks(fontsize = 15)
plt.xlabel("Top N%", fontsize = 15)
plt.ylabel("Accuracy", fontsize = 15)
plt.legend([scatterHealthy, scatterPneumonia], ["Accuracy for Healthy", "Accuracy for Pneumonia"], fontsize = 17)
plt.ylim(0, 110)
plt.show()
+ Accuracy for top 5 percent predictions for healthy: 100.00, pneumonia: 100.00
+ Accuracy for top 10 percent predictions for healthy: 100.00, pneumonia: 100.00
+ Accuracy for top 20 percent predictions for healthy: 100.00, pneumonia: 100.00
+ Accuracy for top 30 percent predictions for healthy: 97.66, pneumonia: 100.00
+ Accuracy for top 40 percent predictions for healthy: 96.35, pneumonia: 100.00
+ Accuracy for top 50 percent predictions for healthy: 96.35, pneumonia: 99.07
+ Accuracy for top 60 percent predictions for healthy: 96.35, pneumonia: 98.44
+ Accuracy for top 70 percent predictions for healthy: 96.35, pneumonia: 95.92
+ Accuracy for top 80 percent predictions for healthy: 96.35, pneumonia: 95.92
+ Accuracy for top 90 percent predictions for healthy: 96.35, pneumonia: 95.92
+ Accuracy for top 100 percent predictions for healthy: 96.35, pneumonia: 95.92
print(model.summary())
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 256, 256, 55) 26675
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 85, 85, 55) 0
_________________________________________________________________
dropout (Dropout) (None, 85, 85, 55) 0
_________________________________________________________________
conv2d_1 (Conv2D) (None, 85, 85, 55) 1464155
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 28, 28, 55) 0
_________________________________________________________________
dropout_1 (Dropout) (None, 28, 28, 55) 0
_________________________________________________________________
conv2d_2 (Conv2D) (None, 28, 28, 55) 1464155
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 9, 9, 55) 0
_________________________________________________________________
dropout_2 (Dropout) (None, 9, 9, 55) 0
_________________________________________________________________
flatten (Flatten) (None, 4455) 0
_________________________________________________________________
dense (Dense) (None, 64) 285184
_________________________________________________________________
batch_normalization (BatchNo (None, 64) 256
_________________________________________________________________
dropout_3 (Dropout) (None, 64) 0
_________________________________________________________________
dense_1 (Dense) (None, 2) 130
=================================================================
Total params: 3,240,555
Trainable params: 3,240,427
Non-trainable params: 128
_________________________________________________________________
None