How to do it...

1. Import the libraries as follows:

import numpy as np 
import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

from keras.models import Sequential
from keras.layers import Dense
from keras.utils import to_categorical
from keras.callbacks import Callback

from keras.datasets import mnist

SEED = 2017

1. Load the MNIST dataset:

(X_train, y_train), (X_val, y_val) = mnist.load_data()

1. Show an example of each label and print the count per label:

# Plot first image of each label
unique_labels = set(y_train)
plt.figure(figsize=(12, 12))

i = 1
for label in unique_labels:
 image = X_train[y_train.tolist().index(label)]
 plt.subplot(10, 10, i)
 plt.axis('off')
 plt.title("{0}: ({1})".format(label, y_train.tolist().count(label)))
 i += 1
 _ = plt.imshow(image, cmap='gray')
plt.show()

We obtain the following result:

1. Preprocess the data:

# Normalize data
X_train = X_train.astype('float32')/255.
X_val = X_val.astype('float32')/255.

# One-Hot-Encode labels
y_train = np_utils.to_categorical(y_train, 10)
y_val = np_utils.to_categorical(y_val, 10)

# Flatten data - we threat the image as a sequential array of values
X_train = np.reshape(X_train, (60000, 784))
X_val = np.reshape(X_val, (10000, 784))

1. Define the model with the sigmoid activation function:

model_sigmoid = Sequential()
model_sigmoid.add(Dense(700, input_dim=784, activation='sigmoid'))
model_sigmoid.add(Dense(700, activation='sigmoid'))
model_sigmoid.add(Dense(700, activation='sigmoid'))
model_sigmoid.add(Dense(700, activation='sigmoid'))
model_sigmoid.add(Dense(700, activation='sigmoid')) 
model_sigmoid.add(Dense(350, activation='sigmoid')) 
model_sigmoid.add(Dense(100, activation='sigmoid')) 
model_sigmoid.add(Dense(10, activation='softmax'))

# Compile model with SGD
model_sigmoid.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])

1. Define the model with the ReLU activation function:

model_relu = Sequential()
model_relu.add(Dense(700, input_dim=784, activation='relu'))
model_relu.add(Dense(700, activation='relu'))
model_relu.add(Dense(700, activation='relu'))
model_relu.add(Dense(700, activation='relu'))
model_relu.add(Dense(700, activation='relu')) 
model_relu.add(Dense(350, activation='relu')) 
model_relu.add(Dense(100, activation='relu')) 
model_relu.add(Dense(10, activation='softmax'))

# Compile model with SGD
model_relu.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])

1. Create a callback function to store the loss values per batch:

class history_loss(cb.Callback):
 def on_train_begin(self, logs={}):
 self.losses = []

 def on_batch_end(self, batch, logs={}):
 batch_loss = logs.get('loss')
 self.losses.append(batch_loss)

1. Run models:

n_epochs = 10
batch_size = 256
validation_split = 0.2

history_sigmoid = history_loss()
model_sigmoid.fit(X_train, y_train, epochs=n_epochs, batch_size=batch_size,
 callbacks=[history_sigmoid],
 validation_split=validation_split, verbose=2)

history_relu = history_loss()
model_relu.fit(X_train, y_train, epochs=n_epochs, batch_size=batch_size,
 callbacks=[history_relu],
 validation_split=validation_split, verbose=2)

1. Plot losses:

plt.plot(np.arange(len(history_sigmoid.losses)), sigmoid, label='sigmoid')
plt.plot(np.arange(len(history_relu.losses)), relu, label='relu')
plt.title('Losses')
plt.xlabel('number of batches')
plt.ylabel('loss')
plt.legend(loc=1)
plt.show()

This code gives us the following result:

1. Extract the maximum weights of each model per layer:

w_sigmoid = []
w_relu = []
for i in range(len(model_sigmoid.layers)):
    w_sigmoid.append(max(model_sigmoid.layers[i].get_weights()[1]))
    w_relu.append(max(model_relu.layers[i].get_weights()[1]))

1. Plot the weights of both models:

fig, ax = plt.subplots()

index = np.arange(len(model_sigmoid.layers))
bar_width = 0.35

plt.bar(index, w_sigmoid, bar_width, label='sigmoid', color='b', alpha=0.4)
plt.bar(index + bar_width, w_relu, bar_width, label='relu', color='r', alpha=0.4)
plt.title('Weights across layers')
plt.xlabel('layer number')
plt.ylabel('maximum weight')
plt.legend(loc=0)

plt.xticks(index + bar_width / 2, np.arange(8))
plt.show()

We obtain the following result: