DETECTION OF TUBERS WITH CONVOLUTIONAL NEURAL NETWORKS¶
Import packages and functions¶
In [1]:
# Import packages
%matplotlib inline
from PIL import Image
import numpy as np
import os
from skimage.color import gray2rgb
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib as mpl
from sklearn.utils import shuffle
import tensorflow as tf
from keras import activations
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, GaussianNoise, BatchNormalization, GlobalAveragePooling2D, Conv2D, MaxPooling2D
from keras.optimizers import Adam
from keras.backend.tensorflow_backend import set_session
from keras.applications.inception_v3 import InceptionV3
from keras.applications.resnet50 import ResNet50
from keras.models import Model
from keras import backend as K
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_auc_score
!pip install keras-vis
from vis.visualization import visualize_cam, visualize_saliency, overlay
!pip install hp5y
from keras.models import model_from_json
mpl.rcParams['figure.figsize'] = (14,16)
FIRST PART: DATA INGESTION AND DATA AUGMENTATION¶
Import original images from local computer¶
We used Magnetic Resonance Imaging (MRI) scans from 114 patients with tuberous sclerosis complex (TSC) and from 114 patients with structurally normal MRI (controls).
For each MRI, we manually selected representative axial T2 and T2 FLAIR slices with tubers (in patients with TSC) and with normal findings (in controls). These axial slices were converted to deidentified .jpg images.
We created three folders per TSC and three folders for controls: TSCtrain (69 patients, 566 images), TSCval (20 patients, 130 images), and TSCtest (25 patients, 210 images) and Controltrain (69 patients, 561 images), Controlval (20 patients, 118 images), and Controltest (25 patients, 226 images). Individual patients belonged to only one of the categories (none of the patients had images in different folders).
The model development part was done in a cloud computer environment and we only used the train and validation subsets. Once the developed models were ready, we selected the one with lowest validation error (binary cross-entropy loss) to test in the local computer on the test set.
In [2]:
# Import zipped files from local computer
from google.colab import files
uploaded = files.upload()
In [3]:
# Unzip files
!unzip Controltrain.zip
!unzip Controlval.zip
!unzip TSCtrain.zip
!unzip TSCval.zip
In [0]:
# Remove the zipped files
!rm Controltrain.zip
!rm Controlval.zip
!rm TSCtrain.zip
!rm TSCval.zip
Path to original images folder¶
In [0]:
# Path to the folder with the original images
pathtoimagesControltrain = './Controltrain/'
pathtoimagesControlval = './Controlval/'
pathtoimagesTSCtrain = './TSCtrain/'
pathtoimagesTSCval = './TSCval/'
The reason for generating augmented images and saving them in a folder (rather than applying data augmentation directly to the model) is twofold:
1.We preferred to visualize the augmented images to make sure that they were images that were reasonably expected to happen naturally (not too rotated, not too zoomed, etc.).
2.We wanted to use several models and compare them. For doing that, we wanted them to have access to the same set of images (rather than augmenting images individually per model).
Create path to augmented images folder¶
In [0]:
# Create folders to save the augmented images
!mkdir augmentedControltrain augmentedTSCtrain
In [0]:
# Create here the directory to save the augmented images
pathtoaugmentedimagesControltrain = './augmentedControltrain/'
pathtoaugmentedimagesTSCtrain = './augmentedTSCtrain/'
Read in the images and augment them for Controltrain¶
In [0]:
# Define the image size
image_size = (224, 224)
# Read in the training images
Controltrain_images = []
Controltrain_dir = pathtoimagesControltrain
Controltrain_files = os.listdir(Controltrain_dir)
# For each image
for f in Controltrain_files:
# Open the image
img = Image.open(Controltrain_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
Controltrain_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
Controltrain_MRIs = np.array(Controltrain_images)
# Transform image files because GPU expects values to be 32-bit floats
Controltrain_MRIs = Controltrain_MRIs.astype(np.float32)
# Rescale the pixel values to be between 0 and 1
Controltrain_MRIs /= 255.
In [9]:
# Example of an image to make sure they were converted right
plt.imshow(Controltrain_MRIs[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
In [0]:
# Define the characteristics for data augmentation
datagen = ImageDataGenerator(
rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.25,
horizontal_flip=True,
fill_mode='nearest')
# Path to images
Controltrain_files = os.listdir(pathtoimagesControltrain)
# Augment the images
for f in Controltrain_files:
img = load_img(Controltrain_dir + f)
x = img_to_array(img)
x = x.reshape((1,) + x.shape)
# Save the augmented images into a directory of augmented images
i = 0
for batch in datagen.flow(x, batch_size=1,
save_to_dir=pathtoaugmentedimagesControltrain,
save_prefix='augmented'+str(np.random.randint(low=1, high=1e8))+str(np.random.randint(low=1e4, high=1e6))+str(np.random.randint(low=1e4, high=1e6))+str(np.random.randint(low=1e4, high=1e6))+
str(np.random.randint(low=1e5, high=1e8))+str(np.random.randint(low=1e2, high=1e7))+str(np.random.randint(low=1e3, high=1e5))+str(np.random.randint(low=1e2, high=1e8)),
save_format='jpg'):
i += 1
if i > 3:
break # Break the cycle after having created 4 augmented images per image, otherwise the generator would loop indefinitely
Read in the images for Controlval¶
In [0]:
# Define the image size
image_size = (224, 224)
# Read in the validation images
Controlval_images = []
Controlval_dir = pathtoimagesControlval
Controlval_files = os.listdir(Controlval_dir)
# For each image
for f in Controlval_files:
# Open the image
img = Image.open(Controlval_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
Controlval_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
Controlval_MRIs = np.array(Controlval_images)
# Transform image files because GPU expects values to be 32-bit floats
Controlval_MRIs = Controlval_MRIs.astype(np.float32)
# Rescale the pixel values to be between 0 and 1
Controlval_MRIs /= 255.
In [12]:
# Example of an image to make sure they were converted right
plt.imshow(Controlval_MRIs[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
Read in the images and augment them for TSCtrain¶
In [0]:
# Define the image size
image_size = (224, 224)
# Read in the training images
TSCtrain_images = []
TSCtrain_dir = pathtoimagesTSCtrain
TSCtrain_files = os.listdir(TSCtrain_dir)
# For each image
for f in TSCtrain_files:
# Open the image
img = Image.open(TSCtrain_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
TSCtrain_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
TSCtrain_MRIs = np.array(TSCtrain_images)
# Transform image files because GPU expects values to be 32-bit floats
TSCtrain_MRIs = TSCtrain_MRIs.astype(np.float32)
# Rescale the pixel values to be between 0 and 1
TSCtrain_MRIs /= 255.
In [14]:
# Example of an image to make sure they were converted right
plt.imshow(TSCtrain_MRIs[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
In [0]:
# Define the characteristics for data augmentation
datagen = ImageDataGenerator(
rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.25,
horizontal_flip=True,
fill_mode='nearest')
# Path to images
TSCtrain_files = os.listdir(pathtoimagesTSCtrain)
# Augment the images
for f in TSCtrain_files:
img = load_img(TSCtrain_dir + f)
x = img_to_array(img)
x = x.reshape((1,) + x.shape)
# Save the augmented images into a directory of augmented images
i = 0
for batch in datagen.flow(x, batch_size=1,
save_to_dir=pathtoaugmentedimagesTSCtrain, save_prefix='augmented'+str(np.random.randint(low=1e5, high=1e8))+str(np.random.randint(low=1e3, high=1e5))+str(np.random.randint(low=1e2, high=1e7))+
str(np.random.randint(low=1e4, high=1e6))+str(np.random.randint(low=1e2, high=1e8))+str(np.random.randint(low=1e4, high=1e6))+str(np.random.randint(low=1e4, high=1e6))
, save_format='jpg'):
i += 1
if i > 3:
break # Break the cycle after having created 4 augmented images per image, otherwise the generator would loop indefinitely
Read in the images for TSCval¶
In [0]:
# Define the image size
image_size = (224, 224)
# Read in the validation images
TSCval_images = []
TSCval_dir = pathtoimagesTSCval
TSCval_files = os.listdir(TSCval_dir)
# For each image
for f in TSCval_files:
# Open the image
img = Image.open(TSCval_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
TSCval_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
TSCval_MRIs = np.array(TSCval_images)
# Transform image files because GPU expects values to be 32-bit floats
TSCval_MRIs = TSCval_MRIs.astype(np.float32)
# Rescale the pixel values to be between 0 and 1
TSCval_MRIs /= 255.
In [17]:
# Example of an image to make sure they were converted right
plt.imshow(TSCval_MRIs[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
SECOND PART: IMPORTATION OF FINAL DATA¶
Path to the final images¶
The final images are the ones used for training and validating the models.
For the training sets they consist of the original training images and their corresponding augmented images.
For the validation set they consist of the original validation images.
In [0]:
# Create directories for the final images
!mkdir FinalControltrain FinalControlval FinalTSCtrain FinalTSCval
In [0]:
# Copy all original images and augmented images to the final folders
!cp ./Controltrain/* ./FinalControltrain/
!cp ./augmentedControltrain/* ./FinalControltrain/
!cp ./Controlval/* ./FinalControlval/
!cp ./TSCtrain/* ./FinalTSCtrain/
!cp ./augmentedTSCtrain/* ./FinalTSCtrain/
!cp ./TSCval/* ./FinalTSCval/
In [0]:
## Path to final images
pathtofinalControltrain = './FinalControltrain/'
pathtofinalControlval = './FinalControlval/'
pathtofinalTSCtrain = './FinalTSCtrain/'
pathtofinalTSCval = './FinalTSCval/'
Import images and labels for the train set¶
In [0]:
## CONTROLS
# Define the image size
image_size = (224, 224)
# Read in the training images for controls
Controltrain_images = []
Controltrain_dir = pathtofinalControltrain
Controltrain_files = os.listdir(Controltrain_dir)
# For each image
for f in Controltrain_files:
# Open the image
img = Image.open(Controltrain_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
Controltrain_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
Controltrain_X = np.array(Controltrain_images)
# Create an array of labels (0 for controls)
Controltrain_y = np.array([[0]*Controltrain_X.shape[0]]).T
## TSC
# Read in the training images for TSC
TSCtrain_images = []
TSCtrain_dir = pathtofinalTSCtrain
TSCtrain_files = os.listdir(TSCtrain_dir)
# For each image
for f in TSCtrain_files:
# Open the image
img = Image.open(TSCtrain_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
TSCtrain_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
TSCtrain_X = np.array(TSCtrain_images)
# Create an array of labels (1 for TSC)
TSCtrain_y = np.array([[1]*TSCtrain_X.shape[0]]).T
## MERGE CONTROLS AND TSC
# Train merge files
train_X = np.concatenate([Controltrain_X, TSCtrain_X])
train_y = np.vstack((Controltrain_y, TSCtrain_y))
# GPU expects values to be 32-bit floats
train_X = train_X.astype(np.float32)
# Rescale the pixel values to be between 0 and 1
train_X /= 255.
In [0]:
# Shuffle in unison the train_X and the train_y array (123 is just a random number for reproducibility)
shuffled_train_X, shuffled_train_y = shuffle(train_X, train_y, random_state=123)
In [23]:
# Make sure that the dimensions are as expected
shuffled_train_X.shape
Out[23]:
In [24]:
# Example of an image to make sure they were converted right
plt.imshow(shuffled_train_X[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
In [25]:
# Make sure that the dimensions are as expected
shuffled_train_y.shape
Out[25]:
In [26]:
# Make sure that the label is correct for the image
shuffled_train_y[0]
Out[26]:
Import images and labels for the validation set¶
In [0]:
## CONTROLS
# Define the image size
image_size = (224, 224)
# Read in the validation images for controls
Controlval_images = []
Controlval_dir = pathtofinalControlval
Controlval_files = os.listdir(Controlval_dir)
# For each image
for f in Controlval_files:
# Open the image
img = Image.open(Controlval_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
Controlval_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
Controlval_X = np.array(Controlval_images)
# Create an array of labels (0 for controls)
Controlval_y = np.array([[0]*Controlval_X.shape[0]]).T
## TSC
# Read in the validation images for TSC
TSCval_images = []
TSCval_dir = pathtofinalTSCval
TSCval_files = os.listdir(TSCval_dir)
# For each image
for f in TSCval_files:
# Open the image
img = Image.open(TSCval_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
TSCval_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
TSCval_X = np.array(TSCval_images)
# Create an array of labels (1 for TSC)
TSCval_y = np.array([[1]*TSCval_X.shape[0]]).T
## MERGE CONTROLS AND TSC
# Val merge files
val_X = np.concatenate([Controlval_X, TSCval_X])
val_y = np.vstack((Controlval_y, TSCval_y))
# GPU expects pixel values to be 32-bit floats
val_X = val_X.astype(np.float32)
# Rescale the pixel values to be between 0 and 1
val_X /= 255.
In [0]:
# Shuffle in unison the val_X and the val_y array (123 is just a random number for reproducibility)
shuffled_val_X, shuffled_val_y = shuffle(val_X, val_y, random_state=123)
In [29]:
shuffled_val_X.shape
Out[29]:
In [30]:
# Example of an image to make sure they were converted right
plt.imshow(shuffled_val_X[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
In [31]:
shuffled_val_y.shape
Out[31]:
In [32]:
shuffled_val_y[0]
Out[32]:
Delete unnecessary files¶
In [33]:
!ls
In [0]:
# Remove the zipped files
!rm -r Controltrain
!rm -r augmentedControltrain
!rm -r Controlval
!rm -r TSCtrain
!rm -r augmentedTSCtrain
!rm -r TSCval
In [35]:
!ls
THIRD PART: EVALUATE NEURAL NETWORKS¶
Tuberous sclerosis complex convolutional neural network (TSCCNN)¶
This is a neural network consisting of:
-A set of: one Conv2D layer with 64 3x3 filters (the input layer), one Conv2D layer with 64 3x3 filters, and one MaxPooling layer with 2x2 pooling.
-One layer of GaussianNoise with 0.2 standard deviation to make the model more robust to overfitting.
-A set of: one Conv2D layer with 128 3x3 filters, one Conv2D layer with 128 3x3 filters, one BathNormalization layer, and one MaxPooling layer with 2x2 pooling.
-A set of: one Conv2D layer with 256 3x3 filters, one Conv2D layer with 256 3x3 filters, and one MaxPooling layer with 2x2 pooling.
-A set of: one Conv2D layer with 512 3x3 filters, one Conv2D layer with 512 3x3 filters, one Conv2D layer with 512 3x3 filters, one Conv2D layer with 512 3x3 filters, and one MaxPooling layer with 2x2 pooling.
-A set of: one Conv2D layer with 512 3x3 filters, one Conv2D layer with 512 3x3 filters, one BathNormalization layer, and one MaxPooling layer with 2x2 pooling.
-A GlobalAveragePooling layer to flatten the input.
-A set of: one Dense (fully connected) layer with 4096 units and ReLU activation, one dropout layer with dropout rate 0.5, one Dense (fully connected) layer with 4096 units and ReLU activation.
-One output layer with one unit and a sigmoid activation function.
The optimizer was Adam with a learning rate of 0.00025. The loss function was binary cross-entropy.
We trained the neural network for 100 epochs with a batch size of 64.
In [36]:
# Define the neural network layers
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size = (3,3), input_shape = train_X.shape[1:]))
model.add(Conv2D(filters = 64, kernel_size = (3,3), activation='relu'))
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(GaussianNoise(0.2))
model.add(Conv2D(filters = 128, kernel_size = (3,3), activation='relu'))
model.add(Conv2D(filters = 128, kernel_size = (3,3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Conv2D(filters = 256, kernel_size = (3,3), activation='relu'))
model.add(Conv2D(filters = 256, kernel_size = (3,3), activation='relu'))
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Conv2D(filters = 512, kernel_size = (3,3), activation='relu'))
model.add(Conv2D(filters = 512, kernel_size = (3,3), activation='relu'))
model.add(Conv2D(filters = 512, kernel_size = (3,3), activation='relu'))
model.add(Conv2D(filters = 512, kernel_size = (3,3), activation='relu'))
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(Conv2D(filters = 512, kernel_size = (3,3), activation='relu'))
model.add(Conv2D(filters = 512, kernel_size = (3,3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size = (2, 2)))
model.add(GlobalAveragePooling2D())
model.add(Dense(units = 4096, activation = 'relu'))
model.add(Dropout(rate = 0.5))
model.add(Dense(units = 4096, activation = 'relu'))
model.add(Dense(units = 1, activation = 'sigmoid'))
# Define the neural network optimizer
opt = Adam(lr = 0.00025)
# Compile the model
model.compile(optimizer = opt, loss = 'binary_crossentropy', metrics = ['accuracy'])
# Fit the model in the training set
historyTSCCNN = model.fit(shuffled_train_X, shuffled_train_y, validation_data = [shuffled_val_X, shuffled_val_y], epochs = 100, batch_size = 64)
print('\n')
print('\n')
# AUC in train and validation set
auc_trainTSCCNN = roc_auc_score(shuffled_train_y, model.predict(shuffled_train_X))
print('The AUC in the train set is {}.'.format(auc_trainTSCCNN))
print('\n')
print('\n')
auc_validTSCCNN = roc_auc_score(shuffled_val_y, model.predict(shuffled_val_X))
print('The AUC in the validation set is {}.'.format(auc_validTSCCNN))
print('\n')
print('\n')
print('\n')
print('\n')
# Plot history of loss during training
plt.plot(historyTSCCNN.history['loss'], label='Train', color='red')
plt.plot(historyTSCCNN.history['val_loss'], label='Validation', color='blue')
plt.title('Loss in train and validation set', size=28)
plt.xlabel('Epoch', size=20)
plt.ylabel('Binary cross-entropy loss', size=20)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.legend(fontsize=20)
plt.ylim(ymin=0, ymax=2)
plt.grid(b=None)
plt.rcParams['axes.facecolor']='lightgrey'
plt.show()
print('\n')
print('\n')
print('\n')
print('\n')
# Plot history of accuracy
plt.plot(historyTSCCNN.history['acc'], label='Train', color='red')
plt.plot(historyTSCCNN.history['val_acc'], label='Validation', color='blue')
plt.title('Accuracy in train and validation set', size=28)
plt.xlabel('Epoch', size=20)
plt.ylabel('Accuracy', size=20)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.legend(fontsize=20)
plt.ylim(ymin=0, ymax=1)
plt.grid(b=None)
plt.rcParams['axes.facecolor']='lightgrey'
plt.show()
In [37]:
# Generate predictions in the form of probabilities for the validation set
valTSCNN = model.predict(shuffled_val_X, batch_size = 16)
valTSCNN
Out[37]:
In [38]:
# Generate the confusion matrix in the validation set
y_true = shuffled_val_y
y_predTSCNN = valTSCNN > 0.5
confusion_matrix(y_true, y_predTSCNN)
Out[38]:
In [39]:
# Calculate accuracy in the validation set
accuracy_TSCCNN = (confusion_matrix(y_true, y_predTSCNN)[0, 0] + confusion_matrix(y_true, y_predTSCNN)[1, 1]) / (confusion_matrix(y_true, y_predTSCNN)[0, 0] + confusion_matrix(y_true, y_predTSCNN)[0, 1] + confusion_matrix(y_true, y_predTSCNN)[1, 0] + confusion_matrix(y_true, y_predTSCNN)[1, 1])
print('The accuracy in the validation set is {}.'.format(accuracy_TSCCNN))
In [40]:
# Calculate AUC in the validation set
auc_validTSCCNN = roc_auc_score(shuffled_val_y, model.predict(shuffled_val_X))
print('The AUC in the validation set is {}.'.format(auc_validTSCCNN))
Save TSCNN¶
In [0]:
# Serialize model to JSON
model_json = model.to_json()
with open("TSCNN.json", "w") as json_file:
json_file.write(model_json)
# Serialize weights to HDF5
model.save_weights("TSCNN.h5")
Model visualization for TSCNN¶
Visualization with class activation map¶
In [42]:
# Visualize the structure and layers of the model
model.layers
Out[42]:
In [43]:
# Iterate through the first 8 MRI images in validation set (otherwise there are issues with the amount of space in the output and running time)
# Print title
print('\033[1m' + 'CLASS ACTIVATION MAP IN THE LAST CONVOLUTIONAL LAYER OF THE NEURONAL NETWORK' + '\033[0m' + '\n \n \n \n')
for i in range(8):
# Print real classification of the image
print('Real classification of the image: {}'.format('TSC' if y_true[i][0]==1 else 'NO TSC'))
# Original image
plt.subplot(1,3,1)
plt.imshow(shuffled_val_X[i])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map
plt.subplot(1,3,2)
heat_map = visualize_cam(model, layer_idx=17, filter_indices=None, seed_input=shuffled_val_X[i])
plt.imshow(heat_map)
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map superimposed on original image
plt.subplot(1,3,3)
plt.imshow(shuffled_val_X[i])
plt.imshow(heat_map, alpha = 0.8 * valTSCNN[i][0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Shot the image
plt.show()
# Print model classification and model probability of TSC
print('Model classification of this image: {} \nEstimated probability of tuber(s): {} \n \n \n \n'.format('TSC' if valTSCNN[i][0]>0.5 else 'NO TSC', valTSCNN[i][0]))
InceptionV3 training all the layers¶
This is the InceptionV3 neural network initialized with the ImageNet weights.
We did not use the top layer, and we used an output layer with 1 units and a sigmoid activation function.
We added a GlobalAveragePooling2D and a Dense (fully-connected layer with 4096 units) before the output layer.
We used Adam optimizer with learning rate of 0.00025.
We used binary cross-entropy as loss function.
We trained the neural network for 100 epochs with a batch size of 64.
In [44]:
# Use InceptionV3 as the base model
base_model = InceptionV3(weights = 'imagenet', include_top = False, input_shape=(224, 224, 3))
# Get the output of the base model
x = base_model.output
# Add a 2D global average pooling layer
x = GlobalAveragePooling2D()(x)
# Add a fully-connected layer
x = Dense(4096, activation='relu')(x)
# Ad a layer for binary classification
predictions = Dense(units = 1, activation = 'sigmoid')(x)
# Define the model to be trained
model = Model(inputs = base_model.input, outputs = predictions)
# Train all layers in the base model
for layer in base_model.layers:
layer.trainable = True
# Compile the model
opt = Adam(lr = 0.00025)
model.compile(optimizer = opt, loss = 'binary_crossentropy', metrics = ['accuracy'])
# Fit and test the model in the validation set
historyInceptionV3 = model.fit(shuffled_train_X, shuffled_train_y, validation_data = [shuffled_val_X, shuffled_val_y], epochs = 100, batch_size = 64)
print('\n')
print('\n')
# AUC in train and validation set
auc_trainInceptionV3 = roc_auc_score(shuffled_train_y, model.predict(shuffled_train_X))
print('The AUC in the train set is {}.'.format(auc_trainInceptionV3))
print('\n')
print('\n')
auc_validInceptionV3 = roc_auc_score(shuffled_val_y, model.predict(shuffled_val_X))
print('The AUC in the validation set is {}.'.format(auc_validInceptionV3))
print('\n')
print('\n')
print('\n')
print('\n')
# Plot history of loss during training
plt.plot(historyInceptionV3.history['loss'], label='Train', color='red')
plt.plot(historyInceptionV3.history['val_loss'], label='Validation', color='blue')
plt.title('Loss in train and validation set', size=28)
plt.xlabel('Epoch', size=20)
plt.ylabel('Binary cross-entropy loss', size=20)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.legend(fontsize=20)
plt.ylim(ymin=0, ymax=2)
plt.grid(b=None)
plt.rcParams['axes.facecolor']='lightgrey'
plt.show()
print('\n')
print('\n')
print('\n')
print('\n')
# Plot history of accuracy
plt.plot(historyInceptionV3.history['acc'], label='Train', color='red')
plt.plot(historyInceptionV3.history['val_acc'], label='Validation', color='blue')
plt.title('Accuracy in train and validation set', size=28)
plt.xlabel('Epoch', size=20)
plt.ylabel('Accuracy', size=20)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.legend(fontsize=20)
plt.ylim(ymin=0, ymax=1)
plt.grid(b=None)
plt.rcParams['axes.facecolor']='lightgrey'
plt.show()
In [45]:
# Generate predictions in the form of probabilities in the validation set
valInceptionV3 = model.predict(shuffled_val_X, batch_size = 16)
valInceptionV3
Out[45]:
In [46]:
# Generate the confusion matrix in the validation set
y_true = shuffled_val_y
y_predInceptionV3 = valInceptionV3 > 0.5
confusion_matrix(y_true, y_predInceptionV3)
Out[46]:
In [47]:
# Calculate accuracy in the validation set
accuracy_InceptionV3 = (confusion_matrix(y_true, y_predInceptionV3)[0, 0] + confusion_matrix(y_true, y_predInceptionV3)[1, 1]) / (confusion_matrix(y_true, y_predInceptionV3)[0, 0] + confusion_matrix(y_true, y_predInceptionV3)[0, 1] + confusion_matrix(y_true, y_predInceptionV3)[1, 0] + confusion_matrix(y_true, y_predInceptionV3)[1, 1])
print('The accuracy in the validation set is {}.'.format(accuracy_InceptionV3))
In [48]:
# Calculate AUC in the validation set
auc_validInceptionV3 = roc_auc_score(shuffled_val_y, model.predict(shuffled_val_X))
print('The AUC in the validation set is {}.'.format(auc_validInceptionV3))
Save InceptionV3¶
In [0]:
# Serialize model to JSON
model_json = model.to_json()
with open("InceptionV3.json", "w") as json_file:
json_file.write(model_json)
# Serialize weights to HDF5
model.save_weights("InceptionV3.h5")
Model visualization for InceptionV3¶
Visualization with class activation map¶
In [50]:
# Visualize the structure and layers of the model
model.layers
Out[50]:
In [51]:
# Iterate through the first 8 MRI images in validation set (otherwise there are issues with the amount of space in the output and running time)
# Print title
print('\033[1m' + 'CLASS ACTIVATION MAP IN THE LAST CONVOLUTIONAL LAYER OF THE NEURONAL NETWORK' + '\033[0m' + '\n \n \n \n')
for i in range(8):
# Print real classification of the image
print('Real classification of the image: {}'.format('TSC' if y_true[i][0]==1 else 'NO TSC'))
# Original image
plt.subplot(1,3,1)
plt.imshow(shuffled_val_X[i])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map
plt.subplot(1,3,2)
heat_map = visualize_cam(model, layer_idx=300, filter_indices=None, seed_input=shuffled_val_X[i])
plt.imshow(heat_map)
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map superimposed on original image
plt.subplot(1,3,3)
plt.imshow(shuffled_val_X[i])
plt.imshow(heat_map, alpha = 0.8 * valInceptionV3[i][0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Shot the image
plt.show()
# Print model classification and model probability of TSC
print('Model classification of this image: {} \nEstimated probability of tuber(s): {} \n \n \n \n'.format('TSC' if valInceptionV3[i][0]>0.5 else 'NO TSC', valInceptionV3[i][0]))
ResNet50 training all the layers¶
This is the ResNet50 neural network initialized with the ImageNet weights.
We did not use the top layer, and we used an output layer with 1 units and a sigmoid activation function.
We added a GlobalAveragePooling2D.
We used Adam optimizer with learning rate of 0.00025.
We used binary cross-entropy as loss function.
We trained the neural network for 100 epochs with a batch size of 64.
In [52]:
# Use ResNet50 as the base model
base_model = ResNet50(weights = 'imagenet', include_top = False, input_shape=(224, 224, 3))
# Get the output of the base model
x = base_model.output
# Add a 2D global average pooling layer
x = GlobalAveragePooling2D()(x)
# Ad a layer for binary classification
predictions = Dense(units = 1, activation = 'sigmoid')(x)
# Define the model to be trained
model = Model(inputs = base_model.input, outputs = predictions)
# Train all layers in the base model
for layer in base_model.layers:
layer.trainable = True
# Compile the model
opt = Adam(lr = 0.00025)
model.compile(optimizer = opt, loss = 'binary_crossentropy', metrics = ['accuracy'])
# Fit and test the model in the validation set
historyResNet50 = model.fit(shuffled_train_X, shuffled_train_y, validation_data = [shuffled_val_X, shuffled_val_y], epochs = 100, batch_size = 64)
print('\n')
print('\n')
# AUC in train and validation set
auc_trainResNet50 = roc_auc_score(shuffled_train_y, model.predict(shuffled_train_X))
print('The AUC in the train set is {}.'.format(auc_trainResNet50))
print('\n')
print('\n')
auc_validResNet50 = roc_auc_score(shuffled_val_y, model.predict(shuffled_val_X))
print('The AUC in the validation set is {}.'.format(auc_validResNet50))
print('\n')
print('\n')
print('\n')
print('\n')
# Plot history of loss during training
plt.plot(historyResNet50.history['loss'], label='Train', color='red')
plt.plot(historyResNet50.history['val_loss'], label='Validation', color='blue')
plt.title('Loss in train and validation set', size=28)
plt.xlabel('Epoch', size=20)
plt.ylabel('Binary cross-entropy loss', size=20)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.legend(fontsize=20)
plt.ylim(ymin=0, ymax=2)
plt.grid(b=None)
plt.rcParams['axes.facecolor']='lightgrey'
plt.show()
print('\n')
print('\n')
print('\n')
print('\n')
# Plot history of accuracy
plt.plot(historyResNet50.history['acc'], label='Train', color='red')
plt.plot(historyResNet50.history['val_acc'], label='Validation', color='blue')
plt.title('Accuracy in train and validation set', size=28)
plt.xlabel('Epoch', size=20)
plt.ylabel('Accuracy', size=20)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.legend(fontsize=20)
plt.ylim(ymin=0, ymax=1)
plt.grid(b=None)
plt.rcParams['axes.facecolor']='lightgrey'
plt.show()
In [53]:
# Generate predictions in the form of probabilities in the validation set
valResNet50 = model.predict(shuffled_val_X, batch_size = 16)
valResNet50
Out[53]:
In [54]:
# Generate the confusion matrix in the validation set
y_true = shuffled_val_y
y_predResNet50 = valResNet50 > 0.5
confusion_matrix(y_true, y_predResNet50)
Out[54]:
In [55]:
# Calculate accuracy in the validation set
accuracy_ResNet50 = (confusion_matrix(y_true, y_predResNet50)[0, 0] + confusion_matrix(y_true, y_predResNet50)[1, 1]) / (confusion_matrix(y_true, y_predResNet50)[0, 0] + confusion_matrix(y_true, y_predResNet50)[0, 1] + confusion_matrix(y_true, y_predResNet50)[1, 0] + confusion_matrix(y_true, y_predResNet50)[1, 1])
print('The accuracy in the validation set is {}.'.format(accuracy_ResNet50))
In [56]:
# Calculate AUC in the validation set
auc_validResNet50 = roc_auc_score(shuffled_val_y, model.predict(shuffled_val_X))
print('The AUC in the validation set is {}.'.format(auc_validResNet50))
Save ResNet50¶
In [0]:
# Serialize model to JSON
model_json = model.to_json()
with open("ResNet50.json", "w") as json_file:
json_file.write(model_json)
# Serialize weights to HDF5
model.save_weights("ResNet50.h5")
Model visualization for ResNet50¶
Visualization with class activation map¶
In [58]:
# Visualize the structure and layers of the model
model.layers
Out[58]:
In [59]:
# Iterate through the first 8 MRI images in validation set (otherwise there are issues with the amount of space in the output and running time)
# Print title
print('\033[1m' + 'CLASS ACTIVATION MAP IN THE LAST CONVOLUTIONAL LAYER OF THE NEURONAL NETWORK' + '\033[0m' + '\n \n \n \n')
for i in range(8):
# Print real classification of the image
print('Real classification of the image: {}'.format('TSC' if y_true[i][0]==1 else 'NO TSC'))
# Original image
plt.subplot(1,3,1)
plt.imshow(shuffled_val_X[i])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map
plt.subplot(1,3,2)
heat_map = visualize_cam(model, layer_idx=171, filter_indices=None, seed_input=shuffled_val_X[i])
plt.imshow(heat_map)
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map superimposed on original image
plt.subplot(1,3,3)
plt.imshow(shuffled_val_X[i])
plt.imshow(heat_map, alpha = 0.8 * valResNet50[i][0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Shot the image
plt.show()
# Print model classification and model probability of TSC
print('Model classification of this image: {} \nEstimated probability of tuber(s): {} \n \n \n \n'.format('TSC' if valResNet50[i][0]>0.5 else 'NO TSC', valResNet50[i][0]))
