from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
a=1
hidden_layer1 = Dense(units=12, activation="relu", input_dim=24)
hidden_layer2 = Dense(units=12, activation="sigmoid")
#[...]
output_layer = Dense(units=3, activation="linear")
model = Sequential()
model.add(hidden_layer1)
model.add(hidden_layer2)
#[...]
model.add(output_layer)
model.compile(optimizer="sgd", loss="mse", metrics=["accuracy"])
#model.fit(X_train, y_train, verbose=2, epochs=10, batch_size=200)
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.datasets import mnist, boston_housing
from tensorflow.keras.utils import to_categorical
def prepare_mnist():
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape((x_train.shape[0], -1))
x_test = x_test.reshape((x_test.shape[0], -1))
scaler = MinMaxScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
return x_train, x_test, y_train, y_test
def prepare_boston():
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
scaler_x = MinMaxScaler()
scaler_x.fit(x_train)
x_train = scaler_x.transform(x_train)
x_test = scaler_x.transform(x_test)
scaler_y = MinMaxScaler()
scaler_y.fit(y_train.reshape((-1, 1)))
y_train = scaler_y.transform(y_train.reshape((-1, 1)))
y_test = scaler_y.transform(y_test.reshape((-1, 1)))
return x_train, x_test, y_train, y_test
x_train, x_test, y_train, y_test = prepare_mnist()
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz
11493376/11490434 [==============================] - 0s 0us/step
x_train, x_test, y_train, y_test = prepare_boston()
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/boston_housing.npz
57344/57026 [==============================] - 0s 0us/step
print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)
print(y_train.min(), y_train.max())
(404, 13) (404, 1) (102, 13) (102, 1)
0.0 1.0
model_boston = Sequential(
[
Dense(units=1, activation="linear", input_dim=13)
]
)
model_boston.compile(optimizer="adam", loss="mse")
h = model_boston.fit(x_train, y_train, verbose=1, epochs=30)
Epoch 1/30
13/13 [==============================] - 0s 797us/step - loss: 0.4267
Epoch 2/30
13/13 [==============================] - 0s 596us/step - loss: 0.3406
Epoch 3/30
13/13 [==============================] - 0s 586us/step - loss: 0.2703
Epoch 4/30
13/13 [==============================] - 0s 560us/step - loss: 0.2130
Epoch 5/30
13/13 [==============================] - 0s 589us/step - loss: 0.1714
Epoch 6/30
13/13 [==============================] - 0s 562us/step - loss: 0.1397
Epoch 7/30
13/13 [==============================] - 0s 573us/step - loss: 0.1173
Epoch 8/30
13/13 [==============================] - 0s 624us/step - loss: 0.1010
Epoch 9/30
13/13 [==============================] - 0s 598us/step - loss: 0.0906
Epoch 10/30
13/13 [==============================] - 0s 574us/step - loss: 0.0835
Epoch 11/30
13/13 [==============================] - 0s 639us/step - loss: 0.0787
Epoch 12/30
13/13 [==============================] - 0s 542us/step - loss: 0.0756
Epoch 13/30
13/13 [==============================] - 0s 2ms/step - loss: 0.0731
Epoch 14/30
13/13 [==============================] - 0s 585us/step - loss: 0.0715
Epoch 15/30
13/13 [==============================] - 0s 584us/step - loss: 0.0703
Epoch 16/30
13/13 [==============================] - 0s 592us/step - loss: 0.0691
Epoch 17/30
13/13 [==============================] - 0s 615us/step - loss: 0.0679
Epoch 18/30
13/13 [==============================] - 0s 637us/step - loss: 0.0669
Epoch 19/30
13/13 [==============================] - 0s 555us/step - loss: 0.0658
Epoch 20/30
13/13 [==============================] - 0s 582us/step - loss: 0.0647
Epoch 21/30
13/13 [==============================] - 0s 2ms/step - loss: 0.0637
Epoch 22/30
13/13 [==============================] - 0s 585us/step - loss: 0.0627
Epoch 23/30
13/13 [==============================] - 0s 714us/step - loss: 0.0616
Epoch 24/30
13/13 [==============================] - 0s 776us/step - loss: 0.0606
Epoch 25/30
13/13 [==============================] - 0s 636us/step - loss: 0.0596
Epoch 26/30
13/13 [==============================] - 0s 622us/step - loss: 0.0586
Epoch 27/30
13/13 [==============================] - 0s 574us/step - loss: 0.0576
Epoch 28/30
13/13 [==============================] - 0s 622us/step - loss: 0.0566
Epoch 29/30
13/13 [==============================] - 0s 2ms/step - loss: 0.0556
Epoch 30/30
13/13 [==============================] - 0s 565us/step - loss: 0.0546
import matplotlib.pyplot as plt
plt.plot(h.history['loss'])
from tensorflow.keras.layers import Input
x_train, x_test, y_train, y_test = prepare_mnist()
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=10)
plt.plot(h.history['loss'])
Epoch 1/10
1875/1875 [==============================] - 2s 1ms/step - loss: 0.4718 - accuracy: 0.8765
Epoch 2/10
1875/1875 [==============================] - 2s 1ms/step - loss: 0.3039 - accuracy: 0.9146
Epoch 3/10
1875/1875 [==============================] - 2s 972us/step - loss: 0.2835 - accuracy: 0.9201
Epoch 4/10
1875/1875 [==============================] - 2s 993us/step - loss: 0.2731 - accuracy: 0.9232
Epoch 5/10
1875/1875 [==============================] - 2s 973us/step - loss: 0.2666 - accuracy: 0.9254
Epoch 6/10
1875/1875 [==============================] - 2s 981us/step - loss: 0.2617 - accuracy: 0.9266
Epoch 7/10
1875/1875 [==============================] - 2s 980us/step - loss: 0.2583 - accuracy: 0.9280
Epoch 8/10
1875/1875 [==============================] - 2s 973us/step - loss: 0.2551 - accuracy: 0.9300
Epoch 9/10
1875/1875 [==============================] - 2s 987us/step - loss: 0.2534 - accuracy: 0.9300
Epoch 10/10
1875/1875 [==============================] - 2s 997us/step - loss: 0.2508 - accuracy: 0.9305
plt.plot(h.history['accuracy'])
from tensorflow.keras.layers import Input
x_train, x_test, y_train, y_test = prepare_mnist()
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=10)
plt.plot(h.history['loss'])
Epoch 1/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.2253 - accuracy: 0.9329
Epoch 2/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0971 - accuracy: 0.9700
Epoch 3/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0705 - accuracy: 0.9774
Epoch 4/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0572 - accuracy: 0.9816
Epoch 5/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0440 - accuracy: 0.9859
Epoch 6/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0382 - accuracy: 0.9879
Epoch 7/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0326 - accuracy: 0.9896
Epoch 8/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0286 - accuracy: 0.9904
Epoch 9/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0266 - accuracy: 0.9914
Epoch 10/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0228 - accuracy: 0.9927
print(model_mnist.count_params())
model_mnist.summary()
N = model_mnist.count_params() / (10 + x_train.shape[1])
print(N)
134794
Model: "sequential_3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_5 (Dense) (None, 128) 100480
_________________________________________________________________
dense_6 (Dense) (None, 128) 16512
_________________________________________________________________
dense_7 (Dense) (None, 128) 16512
_________________________________________________________________
dense_8 (Dense) (None, 10) 1290
=================================================================
Total params: 134,794
Trainable params: 134,794
Non-trainable params: 0
_________________________________________________________________
169.76574307304787
print(y_train[:100].argmax(axis=-1))
[5 0 4 1 9 2 1 3 1 4 3 5 3 6 1 7 2 8 6 9 4 0 9 1 1 2 4 3 2 7 3 8 6 9 0 5 6
0 7 6 1 8 7 9 3 9 8 5 9 3 3 0 7 4 9 8 0 9 4 1 4 4 6 0 4 5 6 1 0 0 1 7 1 6
3 0 2 1 1 7 9 0 2 6 7 8 3 9 0 4 6 7 4 6 8 0 7 8 3 1]
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=169, activation="relu"),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=10, batch_size=100)
Epoch 1/10
600/600 [==============================] - 2s 3ms/step - loss: 0.3113 - accuracy: 0.9139
Epoch 2/10
600/600 [==============================] - 2s 3ms/step - loss: 0.1416 - accuracy: 0.9595
Epoch 3/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0982 - accuracy: 0.9721
Epoch 4/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0745 - accuracy: 0.9783
Epoch 5/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0587 - accuracy: 0.9825
Epoch 6/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0467 - accuracy: 0.9866
Epoch 7/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0374 - accuracy: 0.9893
Epoch 8/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0303 - accuracy: 0.9916
Epoch 9/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0245 - accuracy: 0.9932
Epoch 10/10
600/600 [==============================] - 2s 3ms/step - loss: 0.0195 - accuracy: 0.9950
from tensorflow.keras.layers import Input, Dense
x_train, x_test, y_train, y_test = prepare_mnist()
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=10, validation_data=(x_test, y_test))
Epoch 1/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2250 - accuracy: 0.9323 - val_loss: 0.1133 - val_accuracy: 0.9648
Epoch 2/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1003 - accuracy: 0.9691 - val_loss: 0.1027 - val_accuracy: 0.9691
Epoch 3/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0700 - accuracy: 0.9781 - val_loss: 0.0860 - val_accuracy: 0.9743
Epoch 4/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0549 - accuracy: 0.9826 - val_loss: 0.1003 - val_accuracy: 0.9713
Epoch 5/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0458 - accuracy: 0.9857 - val_loss: 0.0879 - val_accuracy: 0.9750
Epoch 6/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0371 - accuracy: 0.9880 - val_loss: 0.0917 - val_accuracy: 0.9756
Epoch 7/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0328 - accuracy: 0.9894 - val_loss: 0.0910 - val_accuracy: 0.9769
Epoch 8/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0293 - accuracy: 0.9905 - val_loss: 0.1046 - val_accuracy: 0.9752
Epoch 9/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0265 - accuracy: 0.9911 - val_loss: 0.0952 - val_accuracy: 0.9767
Epoch 10/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0220 - accuracy: 0.9928 - val_loss: 0.1065 - val_accuracy: 0.9747
from tensorflow.keras.optimizers import Adam
model_mnist.compile(optimizer=Adam(learning_rate=.01), loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=10, validation_data=(x_test, y_test))
Epoch 1/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2528 - accuracy: 0.9348 - val_loss: 0.1990 - val_accuracy: 0.9496
Epoch 2/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1951 - accuracy: 0.9514 - val_loss: 0.1892 - val_accuracy: 0.9558
Epoch 3/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1729 - accuracy: 0.9580 - val_loss: 0.2141 - val_accuracy: 0.9526
Epoch 4/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1745 - accuracy: 0.9590 - val_loss: 0.2396 - val_accuracy: 0.9471
Epoch 5/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1584 - accuracy: 0.9640 - val_loss: 0.2280 - val_accuracy: 0.9564
Epoch 6/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1524 - accuracy: 0.9665 - val_loss: 0.1919 - val_accuracy: 0.9639
Epoch 7/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1394 - accuracy: 0.9690 - val_loss: 0.2365 - val_accuracy: 0.9527
Epoch 8/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1568 - accuracy: 0.9670 - val_loss: 0.1792 - val_accuracy: 0.9638
Epoch 9/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1413 - accuracy: 0.9702 - val_loss: 0.4430 - val_accuracy: 0.9555
Epoch 10/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1416 - accuracy: 0.9709 - val_loss: 0.2161 - val_accuracy: 0.9620
from tensorflow.keras.optimizers import SGD
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer=SGD(learning_rate=.5), loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=10, validation_data=(x_test, y_test))
Epoch 1/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.3328 - accuracy: 0.8994 - val_loss: 0.2204 - val_accuracy: 0.9341
Epoch 2/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.1431 - accuracy: 0.9588 - val_loss: 0.1169 - val_accuracy: 0.9676
Epoch 3/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.1098 - accuracy: 0.9685 - val_loss: 0.1074 - val_accuracy: 0.9683
Epoch 4/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0873 - accuracy: 0.9745 - val_loss: 0.0997 - val_accuracy: 0.9719
Epoch 5/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0766 - accuracy: 0.9774 - val_loss: 0.1238 - val_accuracy: 0.9692
Epoch 6/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0683 - accuracy: 0.9799 - val_loss: 0.1572 - val_accuracy: 0.9591
Epoch 7/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0620 - accuracy: 0.9817 - val_loss: 0.1326 - val_accuracy: 0.9717
Epoch 8/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0578 - accuracy: 0.9833 - val_loss: 0.1346 - val_accuracy: 0.9664
Epoch 9/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0514 - accuracy: 0.9841 - val_loss: 0.1496 - val_accuracy: 0.9653
Epoch 10/10
1875/1875 [==============================] - 4s 2ms/step - loss: 0.0451 - accuracy: 0.9862 - val_loss: 0.1101 - val_accuracy: 0.9746
import matplotlib.pyplot as plt
plt.plot(h.history['loss'], color='red')
plt.plot(h.history['val_loss'], color='blue')
from tensorflow.keras.callbacks import ModelCheckpoint
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer='adam', loss="categorical_crossentropy", metrics=["accuracy"])
path_to_file = "model.hdf5"
model_checkpoint_callback = ModelCheckpoint(
filepath=path_to_file,
save_freq='epoch',
monitor='val_accuracy',
mode='max',
save_best_only=True)
# Model weights are saved at the end of every epoch, if it's the best seen
# so far.
model_mnist.fit(x_train, y_train, verbose=1, epochs=10, validation_data=(x_test, y_test),
callbacks=[model_checkpoint_callback])
Epoch 1/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2296 - accuracy: 0.9309 - val_loss: 0.1202 - val_accuracy: 0.9636
Epoch 2/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1023 - accuracy: 0.9686 - val_loss: 0.0915 - val_accuracy: 0.9734
Epoch 3/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0714 - accuracy: 0.9780 - val_loss: 0.0828 - val_accuracy: 0.9743
Epoch 4/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0562 - accuracy: 0.9822 - val_loss: 0.0853 - val_accuracy: 0.9755
Epoch 5/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0462 - accuracy: 0.9854 - val_loss: 0.1049 - val_accuracy: 0.9714
Epoch 6/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0413 - accuracy: 0.9869 - val_loss: 0.0779 - val_accuracy: 0.9767
Epoch 7/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0329 - accuracy: 0.9902 - val_loss: 0.0926 - val_accuracy: 0.9759
Epoch 8/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0297 - accuracy: 0.9907 - val_loss: 0.0885 - val_accuracy: 0.9793
Epoch 9/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0267 - accuracy: 0.9913 - val_loss: 0.1082 - val_accuracy: 0.9768
Epoch 10/10
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0242 - accuracy: 0.9919 - val_loss: 0.0937 - val_accuracy: 0.9784
!ls -alh "model.hdf5"
-rw-r--r-- 1 jovyan users 1.6M Nov 30 15:37 model.hdf5
from tensorflow.keras.regularizers import L2
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu", kernel_regularizer=L2(l2=0.05)),
Dense(units=128, activation="relu", kernel_regularizer=L2(l2=0.05)),
Dense(units=128, activation="relu", kernel_regularizer=L2(l2=0.05)),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer='adam', loss="categorical_crossentropy", metrics=["accuracy"])
model_mnist.fit(x_train, y_train, verbose=1, epochs=10, validation_data=(x_test, y_test))
Epoch 1/10
1875/1875 [==============================] - 5s 3ms/step - loss: 1.7476 - accuracy: 0.8131 - val_loss: 1.0019 - val_accuracy: 0.8689
Epoch 2/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.9802 - accuracy: 0.8534 - val_loss: 0.9291 - val_accuracy: 0.8549
Epoch 3/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.8856 - accuracy: 0.8686 - val_loss: 0.8279 - val_accuracy: 0.8810
Epoch 4/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.8292 - accuracy: 0.8759 - val_loss: 0.7755 - val_accuracy: 0.8843
Epoch 5/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.7879 - accuracy: 0.8813 - val_loss: 0.7610 - val_accuracy: 0.8884
Epoch 6/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.7582 - accuracy: 0.8876 - val_loss: 0.7840 - val_accuracy: 0.8684
Epoch 7/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.7348 - accuracy: 0.8917 - val_loss: 0.7225 - val_accuracy: 0.8847
Epoch 8/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.7118 - accuracy: 0.8959 - val_loss: 0.6843 - val_accuracy: 0.9035
Epoch 9/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.6971 - accuracy: 0.8967 - val_loss: 0.6809 - val_accuracy: 0.9002
Epoch 10/10
1875/1875 [==============================] - 5s 3ms/step - loss: 0.6820 - accuracy: 0.9003 - val_loss: 0.6462 - val_accuracy: 0.9067
from tensorflow.keras.layers import Dropout
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu"),
Dropout(rate=0.5),
Dense(units=128, activation="relu"),
Dropout(rate=0.5),
Dense(units=128, activation="relu"),
Dropout(rate=0.5),
Dense(units=10, activation="softmax")
]
)
model_mnist.compile(optimizer='adam', loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=5, validation_data=(x_test, y_test))
Epoch 1/5
1875/1875 [==============================] - 5s 3ms/step - loss: 0.6406 - accuracy: 0.8024 - val_loss: 0.2065 - val_accuracy: 0.9416
Epoch 2/5
1875/1875 [==============================] - 5s 3ms/step - loss: 0.3350 - accuracy: 0.9086 - val_loss: 0.1601 - val_accuracy: 0.9537
Epoch 3/5
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2840 - accuracy: 0.9230 - val_loss: 0.1431 - val_accuracy: 0.9588
Epoch 4/5
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2573 - accuracy: 0.9297 - val_loss: 0.1338 - val_accuracy: 0.9626
Epoch 5/5
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2395 - accuracy: 0.9349 - val_loss: 0.1199 - val_accuracy: 0.9655
import matplotlib.pyplot as plt
plt.plot(h.history['loss'], color='red')
plt.plot(h.history['val_loss'], color='blue')
new_model = Sequential()
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu", name="hidden1"),
Dense(units=128, activation="relu", name="hidden2"),
Dense(units=128, activation="relu", name="hidden3"),
Dense(units=10, activation="softmax")
]
)
for layer in model_mnist.layers:
new_model.add(layer)
if layer.name.startswith("hidden"):
new_model.add(Dropout(rate=0.1))
new_model.compile(optimizer='adam', loss="categorical_crossentropy", metrics=["accuracy"])
new_model.fit(x_train, y_train, verbose=2, epochs=1)
new_model.summary()
1875/1875 - 4s - loss: 0.2735 - accuracy: 0.9171
Model: "sequential_10"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
hidden1 (Dense) multiple 100480
_________________________________________________________________
dropout_3 (Dropout) (32, 128) 0
_________________________________________________________________
hidden2 (Dense) multiple 16512
_________________________________________________________________
dropout_4 (Dropout) (32, 128) 0
_________________________________________________________________
hidden3 (Dense) multiple 16512
_________________________________________________________________
dropout_5 (Dropout) (32, 128) 0
_________________________________________________________________
dense_31 (Dense) multiple 1290
=================================================================
Total params: 134,794
Trainable params: 134,794
Non-trainable params: 0
_________________________________________________________________
from tensorflow.keras.callbacks import EarlyStopping
model_mnist = Sequential(
[
Input(x_train.shape[1]),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=128, activation="relu"),
Dense(units=10, activation="softmax")
]
)
es = EarlyStopping(monitor="val_loss", patience=5)
model_mnist.compile(optimizer='adam', loss="categorical_crossentropy", metrics=["accuracy"])
h = model_mnist.fit(x_train, y_train, verbose=1, epochs=30, validation_data=(x_test, y_test), callbacks=[es])
Epoch 1/30
1875/1875 [==============================] - 5s 3ms/step - loss: 0.2323 - accuracy: 0.9306 - val_loss: 0.1169 - val_accuracy: 0.9635
Epoch 2/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.1004 - accuracy: 0.9688 - val_loss: 0.0963 - val_accuracy: 0.9697
Epoch 3/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0724 - accuracy: 0.9772 - val_loss: 0.0870 - val_accuracy: 0.9739
Epoch 4/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0574 - accuracy: 0.9814 - val_loss: 0.0790 - val_accuracy: 0.9755
Epoch 5/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0463 - accuracy: 0.9852 - val_loss: 0.0817 - val_accuracy: 0.9780
Epoch 6/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0373 - accuracy: 0.9884 - val_loss: 0.1018 - val_accuracy: 0.9740
Epoch 7/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0347 - accuracy: 0.9891 - val_loss: 0.0971 - val_accuracy: 0.9746
Epoch 8/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0307 - accuracy: 0.9903 - val_loss: 0.0938 - val_accuracy: 0.9759
Epoch 9/30
1875/1875 [==============================] - 5s 2ms/step - loss: 0.0258 - accuracy: 0.9914 - val_loss: 0.0995 - val_accuracy: 0.9773
import numpy
x_train, x_valid, y_train, y_valid = prepare_mnist()
n_samples = 10000
indices_train = numpy.random.permutation(x_train.shape[0])
x_train = x_train[indices_train][:n_samples]
y_train = y_train[indices_train][:n_samples]
print(x_train.shape, y_train.shape)
def build_model(hp):
layer1 = Dense(units=hp.Int('units1', min_value=64, max_value=192, step=64),
activation="relu", input_dim=784)
layer2 = Dense(units=hp.Int('units2', min_value=64, max_value=192, step=64),
activation="relu")
layer3 = Dense(units=hp.Int('units3', min_value=64, max_value=192, step=64),
activation="relu")
layer4 = Dense(units=10, activation="softmax")
model = Sequential()
model.add(layer1)
model.add(layer2)
model.add(layer3)
model.add(layer4)
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
return model
(10000, 784) (10000, 10)
from kerastuner.tuners.hyperband import Hyperband
# If you do not remember what R and eta are, refer to your course...
R = 81
eta = 3
tuner = Hyperband(build_model, objective="val_loss", max_epochs=R, factor=eta)
tuner.search_space_summary()
Search space summary
Default search space size: 3
units1 (Int)
{'default': None, 'conditions': [], 'min_value': 64, 'max_value': 192, 'step': 64, 'sampling': None}
units2 (Int)
{'default': None, 'conditions': [], 'min_value': 64, 'max_value': 192, 'step': 64, 'sampling': None}
units3 (Int)
{'default': None, 'conditions': [], 'min_value': 64, 'max_value': 192, 'step': 64, 'sampling': None}
tuner.search(x_train, y_train, validation_data=(x_test, y_test))
Unsupported output type: clearOutput
Trial 25 Complete [00h 00m 01s]
val_loss: 0.23440952599048615
Best val_loss So Far: 0.21717898547649384
Total elapsed time: 00h 00m 43s
INFO:tensorflow:Oracle triggered exit
tuner.results_summary()
Results summary
Results in ./untitled_project
Showing 10 best trials
Objective(name='val_loss', direction='min')
Trial summary
Hyperparameters:
units1: 192
units2: 192
units3: 128
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.21717898547649384
Trial summary
Hyperparameters:
units1: 128
units2: 192
units3: 192
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.22496572136878967
Trial summary
Hyperparameters:
units1: 192
units2: 128
units3: 192
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.2271118462085724
Trial summary
Hyperparameters:
units1: 192
units2: 64
units3: 192
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.23230962455272675
Trial summary
Hyperparameters:
units1: 192
units2: 192
units3: 64
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.23440952599048615
Trial summary
Hyperparameters:
units1: 192
units2: 128
units3: 128
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.24034607410430908
Trial summary
Hyperparameters:
units1: 192
units2: 192
units3: 192
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.2415589839220047
Trial summary
Hyperparameters:
units1: 128
units2: 192
units3: 64
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.25818660855293274
Trial summary
Hyperparameters:
units1: 192
units2: 64
units3: 64
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.2590256929397583
Trial summary
Hyperparameters:
units1: 128
units2: 128
units3: 128
tuner/epochs: 1
tuner/initial_epoch: 0
tuner/bracket: 4
tuner/round: 0
Score: 0.2635345757007599
best_model = tuner.get_best_models(1)[0]
best_model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 192) 150720
_________________________________________________________________
dense_1 (Dense) (None, 192) 37056
_________________________________________________________________
dense_2 (Dense) (None, 128) 24704
_________________________________________________________________
dense_3 (Dense) (None, 10) 1290
=================================================================
Total params: 213,770
Trainable params: 213,770
Non-trainable params: 0
_________________________________________________________________