deeplearning
July 7, 2024
[2]: import numpy as np
import matplotlib.pyplot as plt
import random
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from tensorflow.keras.utils import to_categorical
[3]: # The MNIST data is split between 60,000 28 x 28 pixel training images and␣
↪10,000 28 x 28 pixel images
(X_train, y_train), (X_test, y_test) = mnist.load_data()
print("X_train shape", X_train.shape)
print("y_train shape", y_train.shape)
print("X_test shape", X_test.shape)
print("y_test shape", y_test.shape)
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-
datasets/mnist.npz
11490434/11490434 [==============================] - 0s 0us/step
X_train shape (60000, 28, 28)
y_train shape (60000,)
X_test shape (10000, 28, 28)
y_test shape (10000,)
[4]: plt.rcParams['figure.figsize'] = (9,9)
for i in range(9):
plt.subplot(3,3,i+1)
num = random.randint(0, len(X_train))
plt.imshow(X_train[num], cmap='gray', interpolation='none')
plt.title("Class {}".format(y_train[num]))
plt.tight_layout()
1
[5]: # just a little function for pretty printing a matrix
def matprint(mat, fmt="g"):
col_maxes = [max([len(("{:"+fmt+"}").format(x)) for x in col]) for col in␣
↪mat.T]
for x in mat:
for i, y in enumerate(x):
print(("{:"+str(col_maxes[i])+fmt+"}").format(y), end=" ")
print("")
# now print!
matprint(X_train[num])
2
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 114 12 89 207 253
176 253 253 84 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 29 85 101 225 253 152 197 252 252
253 252 252 84 0 0 0 0 0 0
0 0 0 0 19 166 197 197 198 215 252 252 252 196 161 43 149 195
253 252 252 84 0 0 0 0 0 0
0 0 0 0 178 252 252 252 253 252 252 138 84 0 47 0 0 0
253 252 252 84 0 0 0 0 0 0
0 0 0 0 146 252 252 252 253 252 141 37 0 0 0 0 0 0
253 252 220 37 0 0 0 0 0 0
0 0 0 0 0 57 240 140 141 94 0 0 0 0 0 0 13 191
255 253 196 0 0 0 0 0 0 0
0 0 0 0 0 6 24 0 0 0 0 0 0 0 0 0 154 252
253 252 148 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 187 252
253 186 12 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 79 252 252
253 151 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 85 252 252 252
112 12 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 147 253 253 253
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 225 252 242 89
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 57 231 252 192 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 16 253 252 214 28 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 140 253 252 118 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 92 253 255 215 31 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 19 196 252 253 167 0 0 0
0 0 0 0 0 0 0 0 0 0
3
0 0 0 0 0 0 0 0 0 0 57 252 252 253 167 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 57 252 252 253 74 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 25 205 252 190 12 0 0 0
0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0
[6]: X_train = X_train.reshape(60000, 784) # reshape 60,000 28 x 28 matrices into␣
↪60,000 784-length vectors.
X_test = X_test.reshape(10000, 784) # reshape 10,000 28 x 28 matrices into␣
↪10,000 784-length vectors.
X_train = X_train.astype('float32') # change integers to 32-bit floating␣
↪point numbers
X_test = X_test.astype('float32')
X_train /= 255 # normalize each value for each pixel for␣
↪the entire vector for each input
X_test /= 255
print("Training matrix shape", X_train.shape)
print("Testing matrix shape", X_test.shape)
Training matrix shape (60000, 784)
Testing matrix shape (10000, 784)
[7]: nb_classes = 10 # number of unique digits
Y_train = to_categorical(y_train, nb_classes)
Y_test = to_categorical(y_test, nb_classes)
[8]: # The Sequential model is a linear stack of layers and is very common.
model = Sequential()
[9]: # The first hidden layer is a set of 512 nodes (artificial neurons).
# Each node will receive an element from each input vector and apply some␣
↪weight and bias to it.
model.add(Dense(512, input_shape=(784,))) #(784,) is not a typo -- that␣
↪represents a 784 length vector!
[10]: # An "activation" is a non-linear function applied to the output of the layer␣
↪above.
4
# It checks the new value of the node, and decides whether that artifical␣
↪neuron has fired.
# The Rectified Linear Unit (ReLU) converts all negative inputs to nodes in the␣
↪next layer to be zero.
# Those inputs are then not considered to be fired.
# Positive values of a node are unchanged.
model.add(Activation('relu'))
[11]: # Dropout zeroes a selection of random outputs (i.e., disables their activation)
# Dropout helps protect the model from memorizing or "overfitting" the training␣
↪data.
model.add(Dropout(0.2))
[12]: # The second hidden layer appears identical to our first layer.
# However, instead of each of the 512-node receiving 784-inputs from the input␣
↪image data,
# they receive 512 inputs from the output of the first 512-node layer.
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.2))
[13]: # The final layer of 10 neurons in fully-connected to the previous 512-node␣
↪layer.
# The final layer of a FCN should be equal to the number of desired classes (10␣
↪in this case).
model.add(Dense(10))
[14]: # The "softmax" activation represents a probability distribution over K␣
↪different possible outcomes.
# Its values are all non-negative and sum to 1.
model.add(Activation('softmax'))
[15]: # Summarize the built model
model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 512) 401920
activation (Activation) (None, 512) 0
5
dropout (Dropout) (None, 512) 0
dense_1 (Dense) (None, 512) 262656
activation_1 (Activation) (None, 512) 0
dropout_1 (Dropout) (None, 512) 0
dense_2 (Dense) (None, 10) 5130
activation_2 (Activation) (None, 10) 0
=================================================================
Total params: 669706 (2.55 MB)
Trainable params: 669706 (2.55 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
[16]: # Let's use the Adam optimizer for learning
model.compile(loss='categorical_crossentropy', optimizer='adam',␣
↪metrics=['accuracy'])
[17]: model.fit(X_train, Y_train,
batch_size=128, epochs=5,
verbose=1)
Epoch 1/5
469/469 [==============================] - 12s 23ms/step - loss: 0.2507 -
accuracy: 0.9239
Epoch 2/5
469/469 [==============================] - 14s 31ms/step - loss: 0.1006 -
accuracy: 0.9692
Epoch 3/5
469/469 [==============================] - 12s 26ms/step - loss: 0.0721 -
accuracy: 0.9776
Epoch 4/5
469/469 [==============================] - 11s 24ms/step - loss: 0.0566 -
accuracy: 0.9818
Epoch 5/5
469/469 [==============================] - 9s 20ms/step - loss: 0.0466 -
accuracy: 0.9843
[17]: <keras.src.callbacks.History at 0x7f7327872680>
[18]: score = model.evaluate(X_test, Y_test)
print('Test score:', score[0])
print('Test accuracy:', score[1])
6
313/313 [==============================] - 1s 4ms/step - loss: 0.0714 -
accuracy: 0.9790
Test score: 0.07143609970808029
Test accuracy: 0.9789999723434448
[19]: # The predict_classes function outputs the highest probability class
# according to the trained classifier for each input example.
predictions = model.predict(X_test)
predicted_classes = np.argmax(predictions, axis=1)
# Check which items we got right / wrong
correct_indices = np.nonzero(predicted_classes == y_test)[0]
incorrect_indices = np.nonzero(predicted_classes != y_test)[0]
313/313 [==============================] - 2s 6ms/step
[20]: plt.figure()
for i, correct in enumerate(correct_indices[:9]):
plt.subplot(3,3,i+1)
plt.imshow(X_test[correct].reshape(28,28), cmap='gray',␣
↪interpolation='none')
plt.title("Predicted {}, Class {}".format(predicted_classes[correct],␣
↪y_test[correct]))
plt.tight_layout()
plt.figure()
for i, incorrect in enumerate(incorrect_indices[:9]):
plt.subplot(3,3,i+1)
plt.imshow(X_test[incorrect].reshape(28,28), cmap='gray',␣
↪interpolation='none')
plt.title("Predicted {}, Class {}".format(predicted_classes[incorrect],␣
↪y_test[incorrect]))
plt.tight_layout()
7
8
[41]: # import some additional tools
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D,␣
↪GlobalAveragePooling2D, Flatten
from keras.layers import BatchNormalization
[22]: # Reload the MNIST data
(X_train, y_train), (X_test, y_test) = mnist.load_data()
[23]: # Again, do some formatting
9
# Except we do not flatten each image into a 784-length vector because we want␣
↪to perform convolutions first
X_train = X_train.reshape(60000, 28, 28, 1) #add an additional dimension to␣
↪represent the single-channel
X_test = X_test.reshape(10000, 28, 28, 1)
X_train = X_train.astype('float32') # change integers to 32-bit␣
↪floating point numbers
X_test = X_test.astype('float32')
X_train /= 255 # normalize each value for each␣
↪pixel for the entire vector for each input
X_test /= 255
print("Training matrix shape", X_train.shape)
print("Testing matrix shape", X_test.shape)
Training matrix shape (60000, 28, 28, 1)
Testing matrix shape (10000, 28, 28, 1)
[24]: # one-hot format classes
nb_classes = 10 # number of unique digits
Y_train = to_categorical(y_train, nb_classes)
Y_test = to_categorical(y_test, nb_classes)
[25]: model = Sequential() # Linear stacking of layers
# Convolution Layer 1
model.add(Conv2D(32, (3, 3), input_shape=(28,28,1))) # 32 different 3x3 kernels␣
↪-- so 32 feature maps
model.add(BatchNormalization(axis=-1)) # normalize each feature␣
↪map before activation
convLayer01 = Activation('relu') # activation
model.add(convLayer01)
# Convolution Layer 2
model.add(Conv2D(32, (3, 3))) # 32 different 3x3 kernels␣
↪-- so 32 feature maps
model.add(BatchNormalization(axis=-1)) # normalize each feature␣
↪map before activation
model.add(Activation('relu')) # activation
convLayer02 = MaxPooling2D(pool_size=(2,2)) # Pool the max values over␣
↪a 2x2 kernel
10
model.add(convLayer02)
# Convolution Layer 3
model.add(Conv2D(64,(3, 3))) # 64 different 3x3 kernels␣
↪-- so 64 feature maps
model.add(BatchNormalization(axis=-1)) # normalize each feature␣
↪map before activation
convLayer03 = Activation('relu') # activation
model.add(convLayer03)
# Convolution Layer 4
model.add(Conv2D(64, (3, 3))) # 64 different 3x3 kernels␣
↪-- so 64 feature maps
model.add(BatchNormalization(axis=-1)) # normalize each feature␣
↪map before activation
model.add(Activation('relu')) # activation
convLayer04 = MaxPooling2D(pool_size=(2,2)) # Pool the max values over␣
↪a 2x2 kernel
model.add(convLayer04)
model.add(Flatten()) # Flatten final 4x4x64␣
↪output matrix into a 1024-length vector
# Fully Connected Layer 5
model.add(Dense(512)) # 512 FCN nodes
model.add(BatchNormalization()) # normalization
model.add(Activation('relu')) # activation
# Fully Connected Layer 6
model.add(Dropout(0.2)) # 20% dropout of randomly␣
↪selected nodes
model.add(Dense(10)) # final 10 FCN nodes
model.add(Activation('softmax')) # softmax activation
[26]: model.summary()
Model: "sequential_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 26, 26, 32) 320
batch_normalization (Batch (None, 26, 26, 32) 128
Normalization)
activation_3 (Activation) (None, 26, 26, 32) 0
conv2d_1 (Conv2D) (None, 24, 24, 32) 9248
11
batch_normalization_1 (Bat (None, 24, 24, 32) 128
chNormalization)
activation_4 (Activation) (None, 24, 24, 32) 0
max_pooling2d (MaxPooling2 (None, 12, 12, 32) 0
D)
conv2d_2 (Conv2D) (None, 10, 10, 64) 18496
batch_normalization_2 (Bat (None, 10, 10, 64) 256
chNormalization)
activation_5 (Activation) (None, 10, 10, 64) 0
conv2d_3 (Conv2D) (None, 8, 8, 64) 36928
batch_normalization_3 (Bat (None, 8, 8, 64) 256
chNormalization)
activation_6 (Activation) (None, 8, 8, 64) 0
max_pooling2d_1 (MaxPoolin (None, 4, 4, 64) 0
g2D)
flatten (Flatten) (None, 1024) 0
dense_3 (Dense) (None, 512) 524800
batch_normalization_4 (Bat (None, 512) 2048
chNormalization)
activation_7 (Activation) (None, 512) 0
dropout_2 (Dropout) (None, 512) 0
dense_4 (Dense) (None, 10) 5130
activation_8 (Activation) (None, 10) 0
=================================================================
Total params: 597738 (2.28 MB)
Trainable params: 596330 (2.27 MB)
Non-trainable params: 1408 (5.50 KB)
_________________________________________________________________
12
[27]: # we'll use the same optimizer
model.compile(loss='categorical_crossentropy', optimizer='adam',␣
↪metrics=['accuracy'])
[28]: # data augmentation prevents overfitting by slightly changing the data randomly
# Keras has a great built-in feature to do automatic augmentation
gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08,␣
↪shear_range=0.3,
height_shift_range=0.08, zoom_range=0.08)
test_gen = ImageDataGenerator()
[29]: # We can then feed our augmented data in batches
# Besides loss function considerations as before, this method actually results␣
↪in significant memory savings
# because we are actually LOADING the data into the network in batches before␣
↪processing each batch
# Before the data was all loaded into memory, but then processed in batches.
train_generator = gen.flow(X_train, Y_train, batch_size=128)
test_generator = test_gen.flow(X_test, Y_test, batch_size=128)
[30]: # We can now train our model which is fed data by our batch loader
# Steps per epoch should always be total size of the set divided by the batch␣
↪size
# SIGNIFICANT MEMORY SAVINGS (important for larger, deeper networks)
model.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=5,␣
↪verbose=1,
validation_data=test_generator, validation_steps=10000//128)
Epoch 1/5
<ipython-input-30-758777735d04>:6: UserWarning: `Model.fit_generator` is
deprecated and will be removed in a future version. Please use `Model.fit`,
which supports generators.
model.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=5,
verbose=1,
468/468 [==============================] - 201s 425ms/step - loss: 0.1306 -
accuracy: 0.9590 - val_loss: 0.2779 - val_accuracy: 0.9135
Epoch 2/5
468/468 [==============================] - 211s 450ms/step - loss: 0.0506 -
accuracy: 0.9847 - val_loss: 0.0316 - val_accuracy: 0.9895
13
Epoch 3/5
468/468 [==============================] - 206s 441ms/step - loss: 0.0393 -
accuracy: 0.9877 - val_loss: 0.0306 - val_accuracy: 0.9904
Epoch 4/5
468/468 [==============================] - 207s 442ms/step - loss: 0.0336 -
accuracy: 0.9897 - val_loss: 0.0228 - val_accuracy: 0.9923
Epoch 5/5
468/468 [==============================] - 195s 417ms/step - loss: 0.0293 -
accuracy: 0.9906 - val_loss: 0.0364 - val_accuracy: 0.9889
[30]: <keras.src.callbacks.History at 0x7f7327d1ea70>
[31]: score = model.evaluate(X_test, Y_test)
print('Test score:', score[0])
print('Test accuracy:', score[1])
313/313 [==============================] - 7s 22ms/step - loss: 0.0364 -
accuracy: 0.9888
Test score: 0.036419935524463654
Test accuracy: 0.9887999892234802
[46]: from keras import backend as K
# choose any image to want by specifying the index
img = X_test[3]
img = np.expand_dims(img, axis=0) # Keras requires the image to be in 4D, so we␣
↪add an extra dimension to it.
# Not important to understand how this function work -- It just plots a␣
↪convolution layer
def visualize(layer):
inputs = [K.learning_phase()] + model.inputs
_convout1_f = K.function(inputs, [layer.output])
def convout1_f(X):
# The [0] is to disable the training phase flag
return _convout1_f([0] + [X])
convolutions = convout1_f(img)
convolutions = np.squeeze(convolutions)
print ('Shape of conv:', convolutions.shape)
m = convolutions.shape[2]
n = int(np.ceil(np.sqrt(m)))
14
# Visualization of each filter of the layer
fig = plt.figure(figsize=(15,12))
for i in range(m):
ax = fig.add_subplot(n,n,i+1)
ax.imshow(convolutions[:,:,i], cmap='gray')
[47]: plt.figure()
plt.imshow(X_test[3].reshape(28,28), cmap='gray', interpolation='none')
[47]: <matplotlib.image.AxesImage at 0x7f7319933310>
[44]: visualize(convLayer01, input_image)
15
1/1 [==============================] - 0s 41ms/step
[ ]:
16
