Skip to content
Snippets Groups Projects
Commit e39ca41e authored by Jean-Luc Parouty's avatar Jean-Luc Parouty
Browse files

Update VAE / batch stuffs

parent a4d1cb55
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2228 additions and 171 deletions
GTSRB/05-Full-convolutions.ipynb
+ 14
− 14
View file @ e39ca41e
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img> <img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
# <!-- TITLE --> [GTS5] - CNN with GTSRB dataset - Full convolutions # <!-- TITLE --> [GTS5] - CNN with GTSRB dataset - Full convolutions
<!-- DESC --> Episode 5 : A lot of models, a lot of datasets and a lot of results. <!-- DESC --> Episode 5 : A lot of models, a lot of datasets and a lot of results.
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) --> <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives : ## Objectives :
- Try multiple solutions - Try multiple solutions
- Design a generic and batch-usable code - Design a generic and batch-usable code
The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes. The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
The final aim is to recognise them ! The final aim is to recognise them !
Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
## What we're going to do : ## What we're going to do :
Our main steps: Our main steps:
- Try n models with n datasets - Try n models with n datasets
- Save a Pandas/h5 report - Save a Pandas/h5 report
- Write to be run in batch mode - Write to be run in batch mode
## Step 1 - Import ## Step 1 - Import
### 1.1 - Python ### 1.1 - Python
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import tensorflow as tf import tensorflow as tf
from tensorflow import keras from tensorflow import keras
import numpy as np import numpy as np
import h5py import h5py
import sys,os,time,json import sys,os,time,json
import random import random
from IPython.display import display from IPython.display import display
VERSION='1.6' VERSION='1.6'
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.2 - Where are we ? ### 1.2 - Where are we ?
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# At GRICAD # At GRICAD
# dataset_dir = '/bettik/PROJECTS/pr-fidle/datasets/GTSRB/' # dataset_dir = '/bettik/PROJECTS/pr-fidle/datasets/GTSRB/'
# At IDRIS # At IDRIS
dataset_dir = f'{os.getenv("ALL_CCFRWORK","nowhere")}/datasets/GTSRB' dataset_dir = f'{os.getenv("ALL_CCFRWORK","nowhere")}/datasets/GTSRB'
# At Home # At Home
# dataset_dir = f'{os.getenv("HOME","nowhere")}/datasets/GTSRB' # dataset_dir = f'{os.getenv("HOME","nowhere")}/datasets/GTSRB'
print(f'We will use : dataset_dir={dataset_dir}') print(f'We will use : dataset_dir={dataset_dir}')
``` ```
%% Output %% Output
We will use : dataset_dir=/gpfswork/rech/mlh/commun/datasets/GTSRB We will use : dataset_dir=/gpfswork/rech/mlh/commun/datasets/GTSRB
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 2 - Init and start ## Step 2 - Init and start
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# ---- Where I am ? # ---- Where I am ?
now = time.strftime("%A %d %B %Y - %Hh%Mm%Ss") now = time.strftime("%A %d %B %Y - %Hh%Mm%Ss")
here = os.getcwd() here = os.getcwd()
random.seed(time.time()) random.seed(time.time())
tag_id = '{:06}'.format(random.randint(0,99999)) tag_id = '{:06}'.format(random.randint(0,99999))
# ---- Who I am ? # ---- Who I am ?
oar_id = os.getenv("OAR_JOB_ID", "??") oar_id = os.getenv("OAR_JOB_ID", "??")
slurm_id = os.getenv("SLURM_JOBID", "??") slurm_id = os.getenv("SLURM_JOBID", "??")
print('\nFull Convolutions Notebook') print('\nFull Convolutions Notebook')
print(' Version : {}'.format(VERSION)) print(' Version : {}'.format(VERSION))
print(' Now is : {}'.format(now)) print(' Now is : {}'.format(now))
print(' OAR id : {}'.format(oar_id)) print(' OAR id : {}'.format(oar_id))
print(' SLURM id : {}'.format(slurm_id)) print(' SLURM id : {}'.format(slurm_id))
print(' Tag id : {}'.format(tag_id)) print(' Tag id : {}'.format(tag_id))
print(' Working directory : {}'.format(here)) print(' Working directory : {}'.format(here))
print(' Output directory : ./run') print(' Output directory : ./run')
print(' Dataset_dir : {}'.format(dataset_dir)) print(' Dataset_dir : {}'.format(dataset_dir))
print(' TensorFlow version :',tf.__version__) print(' TensorFlow version :',tf.__version__)
print(' Keras version :',tf.keras.__version__) print(' Keras version :',tf.keras.__version__)
print(' for tensorboard : --logdir {}/run/logs_{}'.format(here,tag_id)) print(' for tensorboard : --logdir {}/run/logs_{}'.format(here,tag_id))
``` ```
%% Output %% Output
Full Convolutions Notebook Full Convolutions Notebook
Version : 1.6 Version : 1.6
Now is : Wednesday 09 September 2020 - 16h32m32s Now is : Wednesday 09 September 2020 - 16h32m32s
OAR id : ?? OAR id : ??
SLURM id : 231099 SLURM id : 231099
Tag id : 029938 Tag id : 029938
Working directory : /gpfsdswork/projects/rech/mlh/uja62cb/fidle/GTSRB Working directory : /gpfsdswork/projects/rech/mlh/uja62cb/fidle/GTSRB
Dataset_dir : /gpfswork/rech/mlh/commun/datasets/GTSRB Dataset_dir : /gpfswork/rech/mlh/commun/datasets/GTSRB
TensorFlow version : 2.2.0 TensorFlow version : 2.2.0
Keras version : 2.3.0-tf Keras version : 2.3.0-tf
for tensorboard : --logdir /gpfsdswork/projects/rech/mlh/uja62cb/fidle/GTSRB/run/logs_029938 for tensorboard : --logdir /gpfsdswork/projects/rech/mlh/uja62cb/fidle/GTSRB/run/logs_029938
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# ---- Uncomment for batch tests # ---- Uncomment for batch tests
# #
# print("\n\n*** Test mode - Exit before making big treatments... ***\n\n") # print("\n\n*** Test mode - Exit before making big treatments... ***\n\n")
# sys.exit() # sys.exit()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 3 - Dataset loading ## Step 3 - Dataset loading
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def read_dataset(dataset_dir, name): def read_dataset(dataset_dir, name):
'''Reads h5 dataset from dataset_dir '''Reads h5 dataset from dataset_dir
Args: Args:
dataset_dir : datasets dir dataset_dir : datasets dir
name : dataset name, without .h5 name : dataset name, without .h5
Returns: x_train,y_train,x_test,y_test data''' Returns: x_train,y_train,x_test,y_test data'''
# ---- Read dataset # ---- Read dataset
filename=f'{dataset_dir}/{name}.h5' filename=f'{dataset_dir}/{name}.h5'
with h5py.File(filename,'r') as f: with h5py.File(filename,'r') as f:
x_train = f['x_train'][:] x_train = f['x_train'][:]
y_train = f['y_train'][:] y_train = f['y_train'][:]
x_test = f['x_test'][:] x_test = f['x_test'][:]
y_test = f['y_test'][:] y_test = f['y_test'][:]
# ---- done # ---- done
return x_train,y_train,x_test,y_test return x_train,y_train,x_test,y_test
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 4 - Models collection ## Step 4 - Models collection
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# A basic model # A basic model
# #
def get_model_v1(lx,ly,lz): def get_model_v1(lx,ly,lz):
model = keras.models.Sequential() model = keras.models.Sequential()
model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz))) model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz)))
model.add( keras.layers.MaxPooling2D((2, 2))) model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.2)) model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Conv2D(192, (3, 3), activation='relu')) model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D((2, 2))) model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.2)) model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Flatten()) model.add( keras.layers.Flatten())
model.add( keras.layers.Dense(1500, activation='relu')) model.add( keras.layers.Dense(1500, activation='relu'))
model.add( keras.layers.Dropout(0.5)) model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Dense(43, activation='softmax')) model.add( keras.layers.Dense(43, activation='softmax'))
return model return model
# A more sophisticated model # A more sophisticated model
# #
def get_model_v2(lx,ly,lz): def get_model_v2(lx,ly,lz):
model = keras.models.Sequential() model = keras.models.Sequential()
model.add( keras.layers.Conv2D(64, (3, 3), padding='same', input_shape=(lx,ly,lz), activation='relu')) model.add( keras.layers.Conv2D(64, (3, 3), padding='same', input_shape=(lx,ly,lz), activation='relu'))
model.add( keras.layers.Conv2D(64, (3, 3), activation='relu')) model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D(pool_size=(2, 2))) model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add( keras.layers.Dropout(0.2)) model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Conv2D(128, (3, 3), padding='same', activation='relu')) model.add( keras.layers.Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add( keras.layers.Conv2D(128, (3, 3), activation='relu')) model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D(pool_size=(2, 2))) model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add( keras.layers.Dropout(0.2)) model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Conv2D(256, (3, 3), padding='same',activation='relu')) model.add( keras.layers.Conv2D(256, (3, 3), padding='same',activation='relu'))
model.add( keras.layers.Conv2D(256, (3, 3), activation='relu')) model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D(pool_size=(2, 2))) model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add( keras.layers.Dropout(0.2)) model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Flatten()) model.add( keras.layers.Flatten())
model.add( keras.layers.Dense(512, activation='relu')) model.add( keras.layers.Dense(512, activation='relu'))
model.add( keras.layers.Dropout(0.5)) model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Dense(43, activation='softmax')) model.add( keras.layers.Dense(43, activation='softmax'))
return model return model
def get_model_v3(lx,ly,lz): def get_model_v3(lx,ly,lz):
model = keras.models.Sequential() model = keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(lx,ly,lz))) model.add(tf.keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(lx,ly,lz)))
model.add(tf.keras.layers.BatchNormalization(axis=-1)) model.add(tf.keras.layers.BatchNormalization(axis=-1))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2))) model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Dropout(0.2)) model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Conv2D(64, (5, 5), padding='same', activation='relu')) model.add(tf.keras.layers.Conv2D(64, (5, 5), padding='same', activation='relu'))
model.add(tf.keras.layers.BatchNormalization(axis=-1)) model.add(tf.keras.layers.BatchNormalization(axis=-1))
model.add(tf.keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu')) model.add(tf.keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu'))
model.add(tf.keras.layers.BatchNormalization(axis=-1)) model.add(tf.keras.layers.BatchNormalization(axis=-1))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2))) model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Dropout(0.2)) model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(512, activation='relu')) model.add(tf.keras.layers.Dense(512, activation='relu'))
model.add(tf.keras.layers.BatchNormalization()) model.add(tf.keras.layers.BatchNormalization())
model.add(tf.keras.layers.Dropout(0.4)) model.add(tf.keras.layers.Dropout(0.4))
model.add(tf.keras.layers.Dense(43, activation='softmax')) model.add(tf.keras.layers.Dense(43, activation='softmax'))
return model return model
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 5 - Multiple datasets, multiple models ;-) ## Step 5 - Multiple datasets, multiple models ;-)
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def multi_run(dataset_dir, datasets, models, datagen=None, def multi_run(dataset_dir, datasets, models, datagen=None,
train_size=1, test_size=1, batch_size=64, epochs=16, train_size=1, test_size=1, batch_size=64, epochs=16,
verbose=0, extension_dir='last'): verbose=0, extension_dir='last'):
""" """
Launches a dataset-model combination Launches a dataset-model combination
args: args:
dataset_dir : Directory of the datasets dataset_dir : Directory of the datasets
datasets : List of dataset (whitout .h5) datasets : List of dataset (whitout .h5)
models : List of model like { "model name":get_model(), ...} models : List of model like { "model name":get_model(), ...}
datagen : Data generator or None (None) datagen : Data generator or None (None)
train_size : % of train dataset to use. 1 mean all. (1) train_size : % of train dataset to use. 1 mean all. (1)
test_size : % of test dataset to use. 1 mean all. (1) test_size : % of test dataset to use. 1 mean all. (1)
batch_size : Batch size (64) batch_size : Batch size (64)
epochs : Number of epochs (16) epochs : Number of epochs (16)
verbose : Verbose level (0) verbose : Verbose level (0)
extension_dir : postfix for logs and models dir (_last) extension_dir : postfix for logs and models dir (_last)
return: return:
report : Report as a dict for Pandas. report : Report as a dict for Pandas.
""" """
# ---- Logs and models dir # ---- Logs and models dir
# #
os.makedirs(f'./run/logs_{extension_dir}', mode=0o750, exist_ok=True) os.makedirs(f'./run/logs_{extension_dir}', mode=0o750, exist_ok=True)
os.makedirs(f'./run/models_{extension_dir}', mode=0o750, exist_ok=True) os.makedirs(f'./run/models_{extension_dir}', mode=0o750, exist_ok=True)
# ---- Columns of output # ---- Columns of output
# #
output={} output={}
output['Dataset'] = [] output['Dataset'] = []
output['Size'] = [] output['Size'] = []
for m in models: for m in models:
output[m+'_Accuracy'] = [] output[m+'_Accuracy'] = []
output[m+'_Duration'] = [] output[m+'_Duration'] = []
# ---- Let's go # ---- Let's go
# #
for d_name in datasets: for d_name in datasets:
print("\nDataset : ",d_name) print("\nDataset : ",d_name)
# ---- Read dataset # ---- Read dataset
x_train,y_train,x_test,y_test = read_dataset(dataset_dir, d_name) x_train,y_train,x_test,y_test = read_dataset(dataset_dir, d_name)
d_size=os.path.getsize(f'{dataset_dir}/{d_name}.h5')/(1024*1024) d_size=os.path.getsize(f'{dataset_dir}/{d_name}.h5')/(1024*1024)
output['Dataset'].append(d_name) output['Dataset'].append(d_name)
output['Size'].append(d_size) output['Size'].append(d_size)
# ---- Get the shape # ---- Get the shape
(n,lx,ly,lz) = x_train.shape (n,lx,ly,lz) = x_train.shape
n_train = int( x_train.shape[0] * train_size ) n_train = int( x_train.shape[0] * train_size )
n_test = int( x_test.shape[0] * test_size ) n_test = int( x_test.shape[0] * test_size )
# ---- For each model # ---- For each model
for m_name,m_function in models.items(): for m_name,m_function in models.items():
print(" Run model {} : ".format(m_name), end='') print(" Run model {} : ".format(m_name), end='')
# ---- get model # ---- get model
try: try:
model=m_function(lx,ly,lz) model=m_function(lx,ly,lz)
# ---- Compile it # ---- Compile it
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
# ---- Callbacks tensorboard # ---- Callbacks tensorboard
log_dir = f"./run/logs_{extension_dir}/tb_{d_name}_{m_name}" log_dir = f"./run/logs_{extension_dir}/tb_{d_name}_{m_name}"
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
# ---- Callbacks bestmodel # ---- Callbacks bestmodel
save_dir = f"./run/models_{extension_dir}/model_{d_name}_{m_name}.h5" save_dir = f"./run/models_{extension_dir}/model_{d_name}_{m_name}.h5"
bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True) bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True)
# ---- Train # ---- Train
start_time = time.time() start_time = time.time()
if datagen==None: if datagen==None:
# ---- No data augmentation (datagen=None) -------------------------------------- # ---- No data augmentation (datagen=None) --------------------------------------
history = model.fit(x_train[:n_train], y_train[:n_train], history = model.fit(x_train[:n_train], y_train[:n_train],
batch_size = batch_size, batch_size = batch_size,
epochs = epochs, epochs = epochs,
verbose = verbose, verbose = verbose,
validation_data = (x_test[:n_test], y_test[:n_test]), validation_data = (x_test[:n_test], y_test[:n_test]),
callbacks = [tensorboard_callback, bestmodel_callback]) callbacks = [tensorboard_callback, bestmodel_callback])
else: else:
# ---- Data augmentation (datagen given) ---------------------------------------- # ---- Data augmentation (datagen given) ----------------------------------------
datagen.fit(x_train) datagen.fit(x_train)
history = model.fit(datagen.flow(x_train, y_train, batch_size=batch_size), history = model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch = int(n_train/batch_size), steps_per_epoch = int(n_train/batch_size),
epochs = epochs, epochs = epochs,
verbose = verbose, verbose = verbose,
validation_data = (x_test[:n_test], y_test[:n_test]), validation_data = (x_test[:n_test], y_test[:n_test]),
callbacks = [tensorboard_callback, bestmodel_callback]) callbacks = [tensorboard_callback, bestmodel_callback])
# ---- Result # ---- Result
end_time = time.time() end_time = time.time()
duration = end_time-start_time duration = end_time-start_time
accuracy = max(history.history["val_accuracy"])*100 accuracy = max(history.history["val_accuracy"])*100
# #
output[m_name+'_Accuracy'].append(accuracy) output[m_name+'_Accuracy'].append(accuracy)
output[m_name+'_Duration'].append(duration) output[m_name+'_Duration'].append(duration)
print(f"Accuracy={accuracy:.2f} and Duration={duration:.2f}") print(f"Accuracy={accuracy:.2f} and Duration={duration:.2f}")
except: except:
output[m_name+'_Accuracy'].append('0') output[m_name+'_Accuracy'].append('0')
output[m_name+'_Duration'].append('999') output[m_name+'_Duration'].append('999')
print('-') print('-')
return output return output
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 6 - Run ! ## Step 6 - Run !
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
start_time = time.time() start_time = time.time()
print('\n---- Run','-'*50) print('\n---- Run','-'*50)
# --------- Datasets, models, and more.. ----------------------------------- # --------- Datasets, models, and more.. -----------------------------------
# #
# ---- For tests # ---- For tests
# datasets = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-RGB'] # datasets = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-RGB']
# models = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3} # models = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3}
# batch_size = 64 # batch_size = 64
# epochs = 5 # epochs = 5
# train_size = 0.2 # train_size = 0.2
# test_size = 0.2 # test_size = 0.2
# with_datagen = False # with_datagen = False
# verbose = 0 # verbose = 0
# #
# ---- All possibilities # ---- All possibilities
datasets = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE'] # datasets = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']
models = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3} # models = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3}
batch_size = 64
epochs = 16
train_size = 1
test_size = 1
with_datagen = False
verbose = 0
#
# ---- Data augmentation
# datasets = ['set-48x48-RGB']
# models = {'v2':get_model_v2}
# batch_size = 64 # batch_size = 64
# epochs = 20 # epochs = 16
# train_size = 1 # train_size = 1
# test_size = 1 # test_size = 1
# with_datagen = True # with_datagen = False
# verbose = 0 # verbose = 0
# #
# ---- Data augmentation
datasets = ['set-48x48-RGB']
models = {'v2':get_model_v2}
batch_size = 64
epochs = 20
train_size = 1
test_size = 1
with_datagen = True
verbose = 0
#
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# ---- Data augmentation # ---- Data augmentation
# #
if with_datagen : if with_datagen :
datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False, datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False, featurewise_std_normalization=False,
width_shift_range=0.1, width_shift_range=0.1,
height_shift_range=0.1, height_shift_range=0.1,
zoom_range=0.2, zoom_range=0.2,
shear_range=0.1, shear_range=0.1,
rotation_range=10.) rotation_range=10.)
else: else:
datagen=None datagen=None
# ---- Run # ---- Run
# #
output = multi_run(dataset_dir, output = multi_run(dataset_dir,
datasets, models, datasets, models,
datagen=datagen, datagen=datagen,
train_size=train_size, test_size=test_size, train_size=train_size, test_size=test_size,
batch_size=batch_size, epochs=epochs, batch_size=batch_size, epochs=epochs,
verbose=verbose, verbose=verbose,
extension_dir=tag_id) extension_dir=tag_id)
# ---- Save report # ---- Save report
# #
report={} report={}
report['output']=output report['output']=output
report['description']='train_size={} test_size={} batch_size={} epochs={} data_aug={}'.format(train_size,test_size,batch_size,epochs,with_datagen) report['description']='train_size={} test_size={} batch_size={} epochs={} data_aug={}'.format(train_size,test_size,batch_size,epochs,with_datagen)
report_name=f'./run/report_{tag_id}.json' report_name=f'./run/report_{tag_id}.json'
with open(report_name, 'w') as file: with open(report_name, 'w') as file:
json.dump(report, file) json.dump(report, file)
print('\nReport saved as ',report_name) print('\nReport saved as ',report_name)
end_time = time.time() end_time = time.time()
duration = end_time-start_time duration = end_time-start_time
print(f'Duration : {duration:.2f} s') print(f'Duration : {duration:.2f} s')
print('-'*59) print('-'*59)
``` ```
%% Output %% Output
---- Run -------------------------------------------------- ---- Run --------------------------------------------------
Dataset : set-24x24-L Dataset : set-24x24-L
Run model v1 : Accuracy=86.42 and Duration=8.58 Run model v1 : Accuracy=86.42 and Duration=8.58
Run model v2 : Accuracy=89.23 and Duration=5.82 Run model v2 : Accuracy=89.23 and Duration=5.82
Run model v3 : Accuracy=88.20 and Duration=7.61 Run model v3 : Accuracy=88.20 and Duration=7.61
Dataset : set-24x24-RGB Dataset : set-24x24-RGB
Run model v1 : Accuracy=88.80 and Duration=6.63 Run model v1 : Accuracy=88.80 and Duration=6.63
Run model v2 : Accuracy=90.30 and Duration=5.70 Run model v2 : Accuracy=90.30 and Duration=5.70
Run model v3 : Accuracy=89.23 and Duration=7.30 Run model v3 : Accuracy=89.23 and Duration=7.30
Dataset : set-48x48-RGB Dataset : set-48x48-RGB
Run model v1 : Accuracy=89.79 and Duration=22.75 Run model v1 : Accuracy=89.79 and Duration=22.75
Run model v2 : Accuracy=91.81 and Duration=12.17 Run model v2 : Accuracy=91.81 and Duration=12.17
Run model v3 : Accuracy=93.39 and Duration=13.33 Run model v3 : Accuracy=93.39 and Duration=13.33
Report saved as ./run/report_029938.json Report saved as ./run/report_029938.json
Duration : 95.78 s Duration : 95.78 s
----------------------------------------------------------- -----------------------------------------------------------
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 7 - That's all folks.. ## Step 7 - That's all folks..
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print('\n{}'.format(time.strftime("%A %-d %B %Y, %H:%M:%S"))) print('\n{}'.format(time.strftime("%A %-d %B %Y, %H:%M:%S")))
print("The work is done.\n") print("The work is done.\n")
``` ```
%% Output %% Output
Wednesday 9 September 2020, 16:34:07 Wednesday 9 September 2020, 16:34:07
The work is done. The work is done.
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
--- ---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img> <img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
GTSRB/07-Show-report.ipynb
+ 343
− 35
View file @ e39ca41e
This diff is collapsed.
GTSRB/batch_slurm.sh
+ 1
− 1
View file @ e39ca41e
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
#SBATCH --gres=gpu:1 # nombre de GPU à réserver (un unique GPU ici) #SBATCH --gres=gpu:1 # nombre de GPU à réserver (un unique GPU ici)
#SBATCH --cpus-per-task=10 # nombre de coeurs à réserver (un quart du noeud) #SBATCH --cpus-per-task=10 # nombre de coeurs à réserver (un quart du noeud)
#SBATCH --hint=nomultithread # on réserve des coeurs physiques et non logiques #SBATCH --hint=nomultithread # on réserve des coeurs physiques et non logiques
#SBATCH --time=02:00:00 # temps exécution maximum demande (HH:MM:SS) #SBATCH --time=03:00:00 # temps exécution maximum demande (HH:MM:SS)
#SBATCH --output="_batch/GTSRB_%j.out" # nom du fichier de sortie #SBATCH --output="_batch/GTSRB_%j.out" # nom du fichier de sortie
#SBATCH --error="_batch/GTSRB_%j.err" # nom du fichier d'erreur (ici commun avec la sortie) #SBATCH --error="_batch/GTSRB_%j.err" # nom du fichier d'erreur (ici commun avec la sortie)
#SBATCH --mail-user=Jean-Luc.Parouty@grenoble-inp.fr #SBATCH --mail-user=Jean-Luc.Parouty@grenoble-inp.fr
......
VAE/04-Prepare-CelebA-datasets.ipynb
+ 9
− 9
View file @ e39ca41e
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img> <img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
# <!-- TITLE --> [VAE4] - Preparation of the CelebA dataset # <!-- TITLE --> [VAE4] - Preparation of the CelebA dataset
<!-- DESC --> Episode 4 : Preparation of a clustered dataset, batchable <!-- DESC --> Episode 4 : Preparation of a clustered dataset, batchable
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) --> <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives : ## Objectives :
- Formatting our dataset in **cluster files**, using batch mode - Formatting our dataset in **cluster files**, using batch mode
- Adapting a notebook for batch use - Adapting a notebook for batch use
The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3). The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3).
## What we're going to do : ## What we're going to do :
- Lire les images - Lire les images
- redimensionner et normaliser celles-ci, - redimensionner et normaliser celles-ci,
- Constituer des clusters d'images en format npy - Constituer des clusters d'images en format npy
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 1 - Import and init ## Step 1 - Import and init
### 1.2 - Import ### 1.2 - Import
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import pandas as pd import pandas as pd
from skimage import io, transform from skimage import io, transform
import os,time,sys,json,glob import os,time,sys,json,glob
import csv import csv
import math, random import math, random
from importlib import reload from importlib import reload
sys.path.append('..') sys.path.append('..')
import fidle.pwk as ooo import fidle.pwk as ooo
place, datasets_dir = ooo.init() place, datasets_dir = ooo.init()
``` ```
%% Output %% Output
FIDLE 2020 - Practical Work Module FIDLE 2020 - Practical Work Module
Version : 0.57 DEV Version : 0.57 DEV
Run time : Friday 11 September 2020, 09:32:00 Run time : Friday 11 September 2020, 11:24:05
TensorFlow version : 2.2.0 TensorFlow version : 2.2.0
Keras version : 2.3.0-tf Keras version : 2.3.0-tf
Current place : Fidle at IDRIS Current place : Fidle at IDRIS
Datasets dir : /gpfswork/rech/mlh/commun/datasets Datasets dir : /gpfswork/rech/mlh/commun/datasets
Update keras cache : Done Update keras cache : Done
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.2 - Directories and files : ### 1.2 - Directories and files :
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dataset_csv = f'{datasets_dir}/celeba/origine/list_attr_celeba.csv' dataset_csv = f'{datasets_dir}/celeba/origine/list_attr_celeba.csv'
dataset_img = f'{datasets_dir}/celeba/origine/img_align_celeba' dataset_img = f'{datasets_dir}/celeba/origine/img_align_celeba'
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 2 - Read and shuffle filenames catalog ## Step 2 - Read and shuffle filenames catalog
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dataset_desc = pd.read_csv(dataset_csv, header=0) dataset_desc = pd.read_csv(dataset_csv, header=0)
dataset_desc = dataset_desc.reindex(np.random.permutation(dataset_desc.index)) dataset_desc = dataset_desc.reindex(np.random.permutation(dataset_desc.index))
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 3 - Save as clusters of n images ## Step 3 - Save as clusters of n images
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 4.2 - Cooking function ### 4.2 - Cooking function
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def read_and_save( dataset_img, dataset_desc, def read_and_save( dataset_img, dataset_desc,
cluster_size=1000, cluster_dir='./dataset_cluster', cluster_name='images', cluster_size=1000, cluster_dir='./dataset_cluster', cluster_name='images',
image_size=(128,128)): image_size=(128,128)):
def save_cluster(imgs,desc,cols,id): def save_cluster(imgs,desc,cols,id):
file_img = f'{cluster_dir}/{cluster_name}-{id:03d}.npy' file_img = f'{cluster_dir}/{cluster_name}-{id:03d}.npy'
file_desc = f'{cluster_dir}/{cluster_name}-{id:03d}.csv' file_desc = f'{cluster_dir}/{cluster_name}-{id:03d}.csv'
np.save(file_img, np.array(imgs)) np.save(file_img, np.array(imgs))
df=pd.DataFrame(data=desc,columns=cols) df=pd.DataFrame(data=desc,columns=cols)
df.to_csv(file_desc, index=False) df.to_csv(file_desc, index=False)
return [],[],id+1 return [],[],id+1
start_time = time.time() start_time = time.time()
cols = list(dataset_desc.columns) cols = list(dataset_desc.columns)
# ---- Check if cluster files exist # ---- Check if cluster files exist
# #
if os.path.isfile(f'{cluster_dir}/images-000.npy'): if os.path.isfile(f'{cluster_dir}/images-000.npy'):
print('\n*** Oops. There are already clusters in the target folder!\n') print('\n*** Oops. There are already clusters in the target folder!\n')
return 0,0 return 0,0
# ---- Create cluster_dir # ---- Create cluster_dir
# #
os.makedirs(cluster_dir, mode=0o750, exist_ok=True) os.makedirs(cluster_dir, mode=0o750, exist_ok=True)
# ---- Read and save clusters # ---- Read and save clusters
# #
imgs, desc, cluster_id = [],[],0 imgs, desc, cluster_id = [],[],0
# #
for i,row in dataset_desc.iterrows(): for i,row in dataset_desc.iterrows():
# #
filename = f'{dataset_img}/{row.image_id}' filename = f'{dataset_img}/{row.image_id}'
# #
# ---- Read image, resize (and normalize) # ---- Read image, resize (and normalize)
# #
img = io.imread(filename) img = io.imread(filename)
img = transform.resize(img, image_size) img = transform.resize(img, image_size)
# #
# ---- Add image and description # ---- Add image and description
# #
imgs.append( img ) imgs.append( img )
desc.append( row.values ) desc.append( row.values )
# #
# ---- Progress bar # ---- Progress bar
# #
ooo.update_progress(f'Cluster {cluster_id:03d} :',len(imgs),cluster_size) ooo.update_progress(f'Cluster {cluster_id:03d} :',len(imgs),cluster_size)
# #
# ---- Save cluster if full # ---- Save cluster if full
# #
if len(imgs)==cluster_size: if len(imgs)==cluster_size:
imgs,desc,cluster_id=save_cluster(imgs,desc,cols, cluster_id) imgs,desc,cluster_id=save_cluster(imgs,desc,cols, cluster_id)
# ---- Save uncomplete cluster # ---- Save uncomplete cluster
if len(imgs)>0 : imgs,desc,cluster_id=save_cluster(imgs,desc,cols,cluster_id) if len(imgs)>0 : imgs,desc,cluster_id=save_cluster(imgs,desc,cols,cluster_id)
duration=time.time()-start_time duration=time.time()-start_time
return cluster_id,duration return cluster_id,duration
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 4.3 - Cluster building ### 4.3 - Cluster building
Reading the 200,000 images can take a long time (>20 minutes) Reading the 200,000 images can take a long time (>20 minutes)
200,000 images will be used for training (x_train), the rest for validation (x_test) 200,000 images will be used for training (x_train), the rest for validation (x_test)
If the target folder is not empty, the construction is blocked. If the target folder is not empty, the construction is blocked.
Image Sizes: 128x128 : 74 GB Image Sizes: 128x128 : 74 GB
Image Sizes: 192x160 : 138 GB Image Sizes: 192x160 : 138 GB
Cluster size : 1000 for tests, 10000 or real use Cluster size : 1000 for tests, 10000 or real use
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# ---- Cluster size # ---- Cluster size
cluster_size_train = 1000 cluster_size_train = 10000
cluster_size_test = 1000 cluster_size_test = 10000
image_size = (128,128) image_size = (128,128)
# ---- To write dataset in the project place # ---- To write dataset in the project place
# #
# output_dir = dataset_dir PLEASE, DON'T DO THIS DURING THE FORMATION ;-) ! # output_dir = dataset_dir PLEASE, DON'T DO THIS DURING THE FORMATION ;-) !
# ---- To write dataset in a test place # ---- To write dataset in a test place
# For small tests only ! # For small tests only !
# #
output_dir = './data' output_dir = './data'
# ---- Clusters location # ---- Clusters location
# #
ooo.mkdir(output_dir) ooo.mkdir(output_dir)
train_dir = f'{output_dir}/clusters.train' train_dir = f'{output_dir}/clusters-s.train'
test_dir = f'{output_dir}/clusters.test' test_dir = f'{output_dir}/clusters-s.test'
# ---- x_train, x_test # ---- x_train, x_test
# #
n1,d1 = read_and_save(dataset_img, dataset_desc[:200000], n1,d1 = read_and_save(dataset_img, dataset_desc[:200000],
cluster_size = cluster_size_train, cluster_size = cluster_size_train,
cluster_dir = train_dir, cluster_dir = train_dir,
image_size = image_size ) image_size = image_size )
n2,d2 = read_and_save(dataset_img, dataset_desc[200000:], n2,d2 = read_and_save(dataset_img, dataset_desc[200000:],
cluster_size = cluster_size_test, cluster_size = cluster_size_test,
cluster_dir = test_dir, cluster_dir = test_dir,
image_size = image_size ) image_size = image_size )
print(f'\n\nDuration : {d1+d2:.2f} s or {ooo.hdelay(d1+d2)}') print(f'\n\nDuration : {d1+d2:.2f} s or {ooo.hdelay(d1+d2)}')
print(f'Train clusters : {train_dir}') print(f'Train clusters : {train_dir}')
print(f'Test clusters : {test_dir}') print(f'Test clusters : {test_dir}')
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
--- ---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img> <img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
VAE/05-Check-CelebA.ipynb
+ 147
− 17
View file @ e39ca41e
This diff is collapsed.
VAE/06-VAE-with-CelebA-s.ipynb
+ 191
− 23
View file @ e39ca41e
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img> <img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
# <!-- TITLE --> [VAE6] - Variational AutoEncoder (VAE) with CelebA (small) # <!-- TITLE --> [VAE6] - Variational AutoEncoder (VAE) with CelebA (small)
<!-- DESC --> Episode 6 : Variational AutoEncoder (VAE) with CelebA (small res.) <!-- DESC --> Episode 6 : Variational AutoEncoder (VAE) with CelebA (small res.)
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) --> <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives : ## Objectives :
- Build and train a VAE model with a large dataset in **small resolution(>70 GB)** - Build and train a VAE model with a large dataset in **small resolution(>70 GB)**
- Understanding a more advanced programming model with **data generator** - Understanding a more advanced programming model with **data generator**
The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3). The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3).
## What we're going to do : ## What we're going to do :
- Defining a VAE model - Defining a VAE model
- Build the model - Build the model
- Train it - Train it
- Follow the learning process with Tensorboard - Follow the learning process with Tensorboard
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 1 - Setup environment ## Step 1 - Setup environment
### 1.1 - Python stuff ### 1.1 - Python stuff
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import tensorflow as tf import tensorflow as tf
import numpy as np import numpy as np
import os,sys import os,sys
from importlib import reload from importlib import reload
import modules.vae import modules.vae
import modules.data_generator import modules.data_generator
reload(modules.data_generator) reload(modules.data_generator)
reload(modules.vae) reload(modules.vae)
from modules.vae import VariationalAutoencoder from modules.vae import VariationalAutoencoder
from modules.data_generator import DataGenerator from modules.data_generator import DataGenerator
sys.path.append('..') sys.path.append('..')
import fidle.pwk as ooo import fidle.pwk_ns as ooo
reload(ooo)
ooo.init() place, datasets_dir = ooo.init()
VariationalAutoencoder.about() VariationalAutoencoder.about()
DataGenerator.about() DataGenerator.about()
``` ```
%% Output
FIDLE 2020 - Practical Work Module
Version : 0.57 DEV
Run time : Sunday 13 September 2020, 10:15:25
TensorFlow version : 2.2.0
Keras version : 2.3.0-tf
Current place : Fidle at IDRIS
Datasets dir : /gpfswork/rech/mlh/commun/datasets
Update keras cache : Done
FIDLE 2020 - Variational AutoEncoder (VAE)
TensorFlow version : 2.2.0
VAE version : 1.28
FIDLE 2020 - DataGenerator
Version : 0.4.1
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.2 - The good place ### 1.2 - The good place
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv("SCRATCH_DIR","")}/PROJECTS/pr-fidle/datasets/celeba', train_dir = f'{datasets_dir}/celeba/clusters-s.train'
'IDRIS' : f'{os.getenv("WORK","")}/datasets/celeba', test_dir = f'{datasets_dir}/celeba/clusters-s.test'
'HOME' : f'{os.getenv("HOME","")}/datasets/celeba'} )
# ---- train/test datasets
train_dir = f'{dataset_dir}/clusters.train'
test_dir = f'{dataset_dir}/clusters.test'
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 2 - DataGenerator and validation data ## Step 2 - DataGenerator and validation data
Ok, everything's perfect, now let's instantiate our generator for the entire dataset. Ok, everything's perfect, now let's instantiate our generator for the entire dataset.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
data_gen = DataGenerator(train_dir, 32, k_size=1) data_gen = DataGenerator(train_dir, 32, k_size=1)
x_test = np.load(f'{test_dir}/images-000.npy') x_test = np.load(f'{test_dir}/images-000.npy')
print(f'Data generator : {len(data_gen)} batchs of {data_gen.batch_size} images, or {data_gen.dataset_size} images') print(f'Data generator : {len(data_gen)} batchs of {data_gen.batch_size} images, or {data_gen.dataset_size} images')
print(f'x_test : {len(x_test)} images') print(f'x_test : {len(x_test)} images')
``` ```
%% Output
Data generator : 6250 batchs of 32 images, or 200000 images
x_test : 2599 images
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 3 - Get VAE model ## Step 3 - Get VAE model
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
tag = f'CelebA.006-S.{os.getenv("SLURM_JOB_ID","unknown")}' tag = f'CelebA.001-S.{os.getenv("SLURM_JOB_ID","unknown")}'
input_shape = (128, 128, 3) input_shape = (128, 128, 3)
z_dim = 200 z_dim = 200
verbose = 1 verbose = 1
encoder= [ {'type':'Conv2D', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, encoder= [ {'type':'Conv2D', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
] ]
decoder= [ {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, decoder= [ {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2DTranspose', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':3, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'sigmoid'} {'type':'Conv2DTranspose', 'filters':3, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'sigmoid'}
] ]
vae = modules.vae.VariationalAutoencoder(input_shape = input_shape, vae = modules.vae.VariationalAutoencoder(input_shape = input_shape,
encoder_layers = encoder, encoder_layers = encoder,
decoder_layers = decoder, decoder_layers = decoder,
z_dim = z_dim, z_dim = z_dim,
verbose = verbose, verbose = verbose,
run_tag = tag) run_tag = tag)
vae.save(model=None) vae.save(model=None)
``` ```
%% Output
Model initialized.
Outputs will be in : ./run/CelebA.001-S.265973
---------- Encoder --------------------------------------------------
Model: "model_1"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
encoder_input (InputLayer) [(None, 128, 128, 3) 0
__________________________________________________________________________________________________
conv2d (Conv2D) (None, 64, 64, 32) 896 encoder_input[0][0]
__________________________________________________________________________________________________
dropout (Dropout) (None, 64, 64, 32) 0 conv2d[0][0]
__________________________________________________________________________________________________
conv2d_1 (Conv2D) (None, 32, 32, 64) 18496 dropout[0][0]
__________________________________________________________________________________________________
dropout_1 (Dropout) (None, 32, 32, 64) 0 conv2d_1[0][0]
__________________________________________________________________________________________________
conv2d_2 (Conv2D) (None, 16, 16, 64) 36928 dropout_1[0][0]
__________________________________________________________________________________________________
dropout_2 (Dropout) (None, 16, 16, 64) 0 conv2d_2[0][0]
__________________________________________________________________________________________________
conv2d_3 (Conv2D) (None, 8, 8, 64) 36928 dropout_2[0][0]
__________________________________________________________________________________________________
dropout_3 (Dropout) (None, 8, 8, 64) 0 conv2d_3[0][0]
__________________________________________________________________________________________________
flatten (Flatten) (None, 4096) 0 dropout_3[0][0]
__________________________________________________________________________________________________
mu (Dense) (None, 200) 819400 flatten[0][0]
__________________________________________________________________________________________________
log_var (Dense) (None, 200) 819400 flatten[0][0]
__________________________________________________________________________________________________
encoder_output (Lambda) (None, 200) 0 mu[0][0]
log_var[0][0]
==================================================================================================
Total params: 1,732,048
Trainable params: 1,732,048
Non-trainable params: 0
__________________________________________________________________________________________________
---------- Encoder --------------------------------------------------
Model: "model_2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
decoder_input (InputLayer) [(None, 200)] 0
_________________________________________________________________
dense (Dense) (None, 4096) 823296
_________________________________________________________________
reshape (Reshape) (None, 8, 8, 64) 0
_________________________________________________________________
conv2d_transpose (Conv2DTran (None, 16, 16, 64) 36928
_________________________________________________________________
dropout_4 (Dropout) (None, 16, 16, 64) 0
_________________________________________________________________
conv2d_transpose_1 (Conv2DTr (None, 32, 32, 64) 36928
_________________________________________________________________
dropout_5 (Dropout) (None, 32, 32, 64) 0
_________________________________________________________________
conv2d_transpose_2 (Conv2DTr (None, 64, 64, 32) 18464
_________________________________________________________________
dropout_6 (Dropout) (None, 64, 64, 32) 0
_________________________________________________________________
conv2d_transpose_3 (Conv2DTr (None, 128, 128, 3) 867
=================================================================
Total params: 916,483
Trainable params: 916,483
Non-trainable params: 0
_________________________________________________________________
Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.
Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.
Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.
Config saved in : ./run/CelebA.001-S.265973/models/vae_config.json
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 4 - Compile it ## Step 4 - Compile it
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
optimizer = tf.keras.optimizers.Adam(1e-4) optimizer = tf.keras.optimizers.Adam(1e-4)
# optimizer = 'adam' # optimizer = 'adam'
r_loss_factor = 10000 r_loss_factor = 10000
vae.compile(optimizer, r_loss_factor) vae.compile(optimizer, r_loss_factor)
``` ```
%% Output
Compiled.
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 5 - Train ## Step 5 - Train
For 10 epochs, adam optimizer : For 10 epochs, adam optimizer :
- Run time at IDRIS : 1299.77 sec. - 0:21:39 - Run time at IDRIS : 1299.77 sec. - 0:21:39
- Run time at GRICAD : 2092.77 sec. - 0:34:52 - Run time at GRICAD : 2092.77 sec. - 0:34:52
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
epochs = 10 epochs = 10
initial_epoch = 0 initial_epoch = 0
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
vae.train(data_generator = data_gen, vae.train(data_generator = data_gen,
x_test = x_test, x_test = x_test,
epochs = epochs, epochs = epochs,
initial_epoch = initial_epoch initial_epoch = initial_epoch
) )
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
--- ---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img> <img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
VAE/06-VAE-with-CelebA-s.nbconvert.ipynb 0 → 100644
+ 431
− 0
View file @ e39ca41e
This diff is collapsed.
VAE/07-VAE-with-CelebA-m.ipynb
+ 118
− 22
View file @ e39ca41e
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img> <img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
# <!-- TITLE --> [VAE7] - Variational AutoEncoder (VAE) with CelebA (medium) # <!-- TITLE --> [VAE7] - Variational AutoEncoder (VAE) with CelebA (medium)
<!-- DESC --> Episode 7 : Variational AutoEncoder (VAE) with CelebA (medium res.) <!-- DESC --> Episode 7 : Variational AutoEncoder (VAE) with CelebA (medium res.)
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) --> <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives : ## Objectives :
- Build and train a VAE model with a large dataset in **medium resolution(>140 GB)** - Build and train a VAE model with a large dataset in **medium resolution(>140 GB)**
- Understanding a more advanced programming model with **data generator** - Understanding a more advanced programming model with **data generator**
The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3). The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3).
## What we're going to do : ## What we're going to do :
- Defining a VAE model - Defining a VAE model
- Build the model - Build the model
- Train it - Train it
- Follow the learning process with Tensorboard - Follow the learning process with Tensorboard
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 1 - Setup environment ## Step 1 - Setup environment
### 1.1 - Python stuff ### 1.1 - Python stuff
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import tensorflow as tf import tensorflow as tf
import numpy as np import numpy as np
import os,sys import os,sys
from importlib import reload from importlib import reload
import modules.vae import modules.vae
import modules.data_generator import modules.data_generator
reload(modules.data_generator) reload(modules.data_generator)
reload(modules.vae) reload(modules.vae)
from modules.vae import VariationalAutoencoder from modules.vae import VariationalAutoencoder
from modules.data_generator import DataGenerator from modules.data_generator import DataGenerator
sys.path.append('..') sys.path.append('..')
import fidle.pwk as ooo import fidle.pwk_ns as ooo
reload(ooo)
ooo.init() place, datasets_dir = ooo.init()
VariationalAutoencoder.about() VariationalAutoencoder.about()
DataGenerator.about() DataGenerator.about()
``` ```
%% Output
FIDLE 2020 - Practical Work Module
Version : 0.57 DEV
Run time : Friday 11 September 2020, 11:55:02
TensorFlow version : 2.2.0
Keras version : 2.3.0-tf
Current place : Fidle at IDRIS
Datasets dir : /gpfswork/rech/mlh/commun/datasets
Update keras cache : Done
FIDLE 2020 - Variational AutoEncoder (VAE)
TensorFlow version : 2.2.0
VAE version : 1.28
FIDLE 2020 - DataGenerator
Version : 0.4.1
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.2 - The good place ### 1.2 - The good place
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv("SCRATCH_DIR","")}/PROJECTS/pr-fidle/datasets/celeba', train_dir = f'{datasets_dir}/celeba/clusters-m.train'
'IDRIS' : f'{os.getenv("WORK","")}/datasets/celeba', test_dir = f'{datasets_dir}/celeba/clusters-m.test'
'HOME' : f'{os.getenv("HOME","")}/datasets/celeba'} )
# ---- train/test datasets
train_dir = f'{dataset_dir}/clusters-M.train'
test_dir = f'{dataset_dir}/clusters-M.test'
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 2 - DataGenerator and validation data ## Step 2 - DataGenerator and validation data
Ok, everything's perfect, now let's instantiate our generator for the entire dataset. Ok, everything's perfect, now let's instantiate our generator for the entire dataset.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
data_gen = DataGenerator(train_dir, 32, k_size=1) data_gen = DataGenerator(train_dir, 32, k_size=1)
x_test = np.load(f'{test_dir}/images-000.npy') x_test = np.load(f'{test_dir}/images-000.npy')
print(f'Data generator : {len(data_gen)} batchs of {data_gen.batch_size} images, or {data_gen.dataset_size} images') print(f'Data generator : {len(data_gen)} batchs of {data_gen.batch_size} images, or {data_gen.dataset_size} images')
print(f'x_test : {len(x_test)} images') print(f'x_test : {len(x_test)} images')
``` ```
%% Output
Data generator : 6250 batchs of 32 images, or 200000 images
x_test : 2599 images
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 3 - Get VAE model ## Step 3 - Get VAE model
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
tag = f'CelebA.007-M.{os.getenv("SLURM_JOB_ID","unknown")}' tag = f'CelebA.002-M.{os.getenv("SLURM_JOB_ID","unknown")}'
input_shape = (192, 160, 3) input_shape = (192, 160, 3)
z_dim = 200 z_dim = 200
verbose = 0 verbose = 0
encoder= [ {'type':'Conv2D', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, encoder= [ {'type':'Conv2D', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
] ]
decoder= [ {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, decoder= [ {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2DTranspose', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':3, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'sigmoid'} {'type':'Conv2DTranspose', 'filters':3, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'sigmoid'}
] ]
vae = modules.vae.VariationalAutoencoder(input_shape = input_shape, vae = modules.vae.VariationalAutoencoder(input_shape = input_shape,
encoder_layers = encoder, encoder_layers = encoder,
decoder_layers = decoder, decoder_layers = decoder,
z_dim = z_dim, z_dim = z_dim,
verbose = verbose, verbose = verbose,
run_tag = tag) run_tag = tag)
vae.save(model=None) vae.save(model=None)
``` ```
%% Output
Model initialized.
Outputs will be in : ./run/CelebA.007-M.254865
Config saved in : ./run/CelebA.007-M.254865/models/vae_config.json
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 4 - Compile it ## Step 4 - Compile it
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
optimizer = tf.keras.optimizers.Adam(1e-4) optimizer = tf.keras.optimizers.Adam(1e-4)
r_loss_factor = 10000 r_loss_factor = 10000
vae.compile(optimizer, r_loss_factor) vae.compile(optimizer, r_loss_factor)
``` ```
%% Output
Compiled.
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 5 - Train ## Step 5 - Train
For 10 epochs, adam optimizer : For 10 epochs, adam optimizer :
- Run time at IDRIS : 1299.77 sec. - 0:21:39 - Run time at IDRIS : 1299.77 sec. - 0:21:39
- Run time at GRICAD : 2092.77 sec. - 0:34:52 - Run time at GRICAD : 2092.77 sec. - 0:34:52
- Run time at IDRIS with medium resolution : Train duration : 6638.61 sec. - 1:50:38 - Run time at IDRIS with medium resolution : Train duration : 6638.61 sec. - 1:50:38
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
epochs = 20 epochs = 20
initial_epoch = 0 initial_epoch = 0
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
vae.train(data_generator = data_gen, vae.train(data_generator = data_gen,
x_test = x_test, x_test = x_test,
epochs = epochs, epochs = epochs,
initial_epoch = initial_epoch initial_epoch = initial_epoch
) )
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
--- ---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img> <img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
VAE/07-VAE-with-CelebA-m.nbconvert-done.ipynb → VAE/07-VAE-with-CelebA-m.nbconvert.ipynb
+ 353
− 0
View file @ e39ca41e
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Variational AutoEncoder (VAE) with CelebA <img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
=========================================
--- # <!-- TITLE --> [VAE7] - Variational AutoEncoder (VAE) with CelebA (medium)
Formation Introduction au Deep Learning (FIDLE) - S. Arias, E. Maldonado, JL. Parouty - CNRS/SARI/DEVLOG - 2020 <!-- DESC --> Episode 7 : Variational AutoEncoder (VAE) with CelebA (medium res.)
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives :
- Build and train a VAE model with a large dataset in **medium resolution(>140 GB)**
- Understanding a more advanced programming model with **data generator**
## Episode 1 - Train a model The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) contains about 200,000 images (202599,218,178,3).
## What we're going to do :
- Defining a VAE model - Defining a VAE model
- Build the model - Build the model
- Train it - Train it
- Follow the learning process with Tensorboard - Follow the learning process with Tensorboard
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 1 - Setup environment ## Step 1 - Setup environment
### 1.1 - Python stuff ### 1.1 - Python stuff
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import tensorflow as tf import tensorflow as tf
import numpy as np import numpy as np
import os,sys import os,sys
from importlib import reload from importlib import reload
import modules.vae import modules.vae
import modules.data_generator import modules.data_generator
reload(modules.data_generator) reload(modules.data_generator)
reload(modules.vae) reload(modules.vae)
from modules.vae import VariationalAutoencoder from modules.vae import VariationalAutoencoder
from modules.data_generator import DataGenerator from modules.data_generator import DataGenerator
sys.path.append('..') sys.path.append('..')
import fidle.pwk as ooo import fidle.pwk_ns as ooo
reload(ooo)
ooo.init() place, datasets_dir = ooo.init()
VariationalAutoencoder.about() VariationalAutoencoder.about()
DataGenerator.about() DataGenerator.about()
``` ```
%% Output %% Output
FIDLE 2020 - Practical Work Module FIDLE 2020 - Practical Work Module
Version : 0.2.8 Version : 0.57 DEV
Run time : Friday 14 February 2020, 00:07:28 Run time : Sunday 13 September 2020, 15:27:04
TensorFlow version : 2.0.0 TensorFlow version : 2.0.0
Keras version : 2.2.4-tf Keras version : 2.2.4-tf
Current place : Fidle at IDRIS
Datasets dir : /gpfswork/rech/mlh/commun/datasets
Update keras cache : Done
FIDLE 2020 - Variational AutoEncoder (VAE) FIDLE 2020 - Variational AutoEncoder (VAE)
TensorFlow version : 2.0.0 TensorFlow version : 2.0.0
VAE version : 1.28 VAE version : 1.28
FIDLE 2020 - DataGenerator FIDLE 2020 - DataGenerator
Version : 0.4.1 Version : 0.4.1
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### 1.2 - The good place ### 1.2 - The good place
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv("SCRATCH_DIR","")}/PROJECTS/pr-fidle/datasets/celeba', train_dir = f'{datasets_dir}/celeba/clusters-m.train'
'IDRIS' : f'{os.getenv("WORK","")}/datasets/celeba', test_dir = f'{datasets_dir}/celeba/clusters-m.test'
'HOME' : f'{os.getenv("HOME","")}/datasets/celeba'} )
# ---- train/test datasets
train_dir = f'{dataset_dir}/clusters-M.train'
test_dir = f'{dataset_dir}/clusters-M.test'
``` ```
%% Output
Well, we should be at IDRIS !
We are going to use: /gpfswork/rech/mlh/uja62cb/datasets/celeba
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 2 - DataGenerator and validation data ## Step 2 - DataGenerator and validation data
Ok, everything's perfect, now let's instantiate our generator for the entire dataset. Ok, everything's perfect, now let's instantiate our generator for the entire dataset.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
data_gen = DataGenerator(train_dir, 32, k_size=1) data_gen = DataGenerator(train_dir, 32, k_size=1)
x_test = np.load(f'{test_dir}/images-000.npy') x_test = np.load(f'{test_dir}/images-000.npy')
print(f'Data generator : {len(data_gen)} batchs of {data_gen.batch_size} images, or {data_gen.dataset_size} images') print(f'Data generator : {len(data_gen)} batchs of {data_gen.batch_size} images, or {data_gen.dataset_size} images')
print(f'x_test : {len(x_test)} images') print(f'x_test : {len(x_test)} images')
``` ```
%% Output %% Output
Data generator : 6250 batchs of 32 images, or 200000 images Data generator : 6250 batchs of 32 images, or 200000 images
x_test : 2599 images x_test : 2599 images
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 3 - Get VAE model ## Step 3 - Get VAE model
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
tag = f'CelebA.052-M.{os.getenv("SLURM_JOB_ID","unknown")}' tag = f'CelebA.002-M.{os.getenv("SLURM_JOB_ID","unknown")}'
input_shape = (192, 160, 3) input_shape = (192, 160, 3)
z_dim = 200 z_dim = 200
verbose = 0 verbose = 0
encoder= [ {'type':'Conv2D', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, encoder= [ {'type':'Conv2D', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2D', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
] ]
decoder= [ {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, decoder= [ {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2DTranspose', 'filters':64, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'}, {'type':'Conv2DTranspose', 'filters':32, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'relu'},
{'type':'Dropout', 'rate':0.25}, {'type':'Dropout', 'rate':0.25},
{'type':'Conv2DTranspose', 'filters':3, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'sigmoid'} {'type':'Conv2DTranspose', 'filters':3, 'kernel_size':(3,3), 'strides':2, 'padding':'same', 'activation':'sigmoid'}
] ]
vae = modules.vae.VariationalAutoencoder(input_shape = input_shape, vae = modules.vae.VariationalAutoencoder(input_shape = input_shape,
encoder_layers = encoder, encoder_layers = encoder,
decoder_layers = decoder, decoder_layers = decoder,
z_dim = z_dim, z_dim = z_dim,
verbose = verbose, verbose = verbose,
run_tag = tag) run_tag = tag)
vae.save(model=None) vae.save(model=None)
``` ```
%% Output %% Output
Model initialized. Model initialized.
Outputs will be in : ./run/CelebA.052-M.810156 Outputs will be in : ./run/CelebA.002-M.266176
Config saved in : ./run/CelebA.052-M.810156/models/vae_config.json Config saved in : ./run/CelebA.002-M.266176/models/vae_config.json
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 4 - Compile it ## Step 4 - Compile it
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
optimizer = tf.keras.optimizers.Adam(1e-4) optimizer = tf.keras.optimizers.Adam(1e-4)
r_loss_factor = 10000 r_loss_factor = 10000
vae.compile(optimizer, r_loss_factor) vae.compile(optimizer, r_loss_factor)
``` ```
%% Output %% Output
Compiled. Compiled.
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Step 5 - Train ## Step 5 - Train
For 10 epochs, adam optimizer : For 10 epochs, adam optimizer :
- Run time at IDRIS : 1299.77 sec. - 0:21:39 - Run time at IDRIS : 1299.77 sec. - 0:21:39
- Run time at GRICAD : 2092.77 sec. - 0:34:52 - Run time at GRICAD : 2092.77 sec. - 0:34:52
- At IDRIS with medium resolution : Train duration : 6638.61 sec. - 1:50:38 - Run time at IDRIS with medium resolution : Train duration : 6638.61 sec. - 1:50:38
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
epochs = 20 epochs = 20
initial_epoch = 0 initial_epoch = 0
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
vae.train(data_generator = data_gen, vae.train(data_generator = data_gen,
x_test = x_test, x_test = x_test,
epochs = epochs, epochs = epochs,
initial_epoch = initial_epoch initial_epoch = initial_epoch
) )
``` ```
%% Output %% Output
Epoch 1/20 Epoch 1/20
6250/6250 [==============================] - 342s 55ms/step - loss: 332.4792 - vae_r_loss: 282.9655 - vae_kl_loss: 49.5134 - val_loss: 235.4288 - val_vae_r_loss: 187.4334 - val_vae_kl_loss: 48.1192 1/6250 [..............................] - ETA: 7:18:28 - loss: 902.8120 - vae_r_loss: 901.9757 - vae_kl_loss: 0.8364WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.214785). Check your callbacks.
6250/6250 [==============================] - 357s 57ms/step - loss: 334.6058 - vae_r_loss: 286.9088 - vae_kl_loss: 47.6970 - val_loss: 241.3782 - val_vae_r_loss: 194.3746 - val_vae_kl_loss: 47.1940
Epoch 2/20 Epoch 2/20
6250/6250 [==============================] - 337s 54ms/step - loss: 224.0962 - vae_r_loss: 166.4602 - vae_kl_loss: 57.6360 - val_loss: 210.9632 - val_vae_r_loss: 155.0925 - val_vae_kl_loss: 56.0114 6250/6250 [==============================] - 361s 58ms/step - loss: 233.6622 - vae_r_loss: 178.5938 - vae_kl_loss: 55.0685 - val_loss: 223.6776 - val_vae_r_loss: 171.8392 - val_vae_kl_loss: 51.9847
Epoch 3/20
6250/6250 [==============================] - 362s 58ms/step - loss: 223.8023 - vae_r_loss: 166.8294 - vae_kl_loss: 56.9731 - val_loss: 223.1821 - val_vae_r_loss: 171.0302 - val_vae_kl_loss: 52.3176
Epoch 4/20 Epoch 4/20
6250/6250 [==============================] - 333s 53ms/step - loss: 214.5463 - vae_r_loss: 155.7666 - vae_kl_loss: 58.7794 - val_loss: 203.8241 - val_vae_r_loss: 147.3248 - val_vae_kl_loss: 56.5778 6250/6250 [==============================] - 361s 58ms/step - loss: 212.7579 - vae_r_loss: 154.4291 - vae_kl_loss: 58.3290 - val_loss: 207.5755 - val_vae_r_loss: 151.5950 - val_vae_kl_loss: 56.0710
Epoch 7/20 Epoch 8/20
6250/6250 [==============================] - 327s 52ms/step - loss: 211.1459 - vae_r_loss: 152.4026 - vae_kl_loss: 58.7437 - val_loss: 201.1862 - val_vae_r_loss: 145.6906 - val_vae_kl_loss: 55.6326 5728/6250 [==========================>...] - ETA: 29s - loss: 207.6788 - vae_r_loss: 149.3138 - vae_kl_loss: 58.3651
Epoch 10/20
6250/6250 [==============================] - 335s 54ms/step - loss: 209.7628 - vae_r_loss: 151.0874 - vae_kl_loss: 58.6756 - val_loss: 202.3954 - val_vae_r_loss: 147.0956 - val_vae_kl_loss: 55.4047
Epoch 12/20
6250/6250 [==============================] - 333s 53ms/step - loss: 207.9830 - vae_r_loss: 149.3870 - vae_kl_loss: 58.5959 - val_loss: 198.5626 - val_vae_r_loss: 142.5848 - val_vae_kl_loss: 56.0871
Epoch 16/20
6250/6250 [==============================] - 330s 53ms/step - loss: 206.6382 - vae_r_loss: 148.0522 - vae_kl_loss: 58.5863 - val_loss: 197.5800 - val_vae_r_loss: 142.6799 - val_vae_kl_loss: 54.9832
Train duration : 6638.61 sec. - 1:50:38
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
---- ---
That's all folks ! <img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
VAE/08-VAE-withCelebA-post.ipynb
+ 55
− 46
View file @ e39ca41e
This diff is collapsed.
VAE/batch_slurm.sh
+ 4
− 4
View file @ e39ca41e
#!/bin/bash #!/bin/bash
#SBATCH --job-name="VAE_bizness" # nom du job #SBATCH --job-name="VAE" # nom du job
#SBATCH --ntasks=1 # nombre de tâche (un unique processus ici) #SBATCH --ntasks=1 # nombre de tâche (un unique processus ici)
#SBATCH --gres=gpu:1 # nombre de GPU à réserver (un unique GPU ici) #SBATCH --gres=gpu:1 # nombre de GPU à réserver (un unique GPU ici)
#SBATCH --cpus-per-task=10 # nombre de coeurs à réserver (un quart du noeud) #SBATCH --cpus-per-task=10 # nombre de coeurs à réserver (un quart du noeud)
#SBATCH --hint=nomultithread # on réserve des coeurs physiques et non logiques #SBATCH --hint=nomultithread # on réserve des coeurs physiques et non logiques
#SBATCH --time=00:20:00 # temps exécution maximum demande (HH:MM:SS) #SBATCH --time=05:00:00 # temps exécution maximum demande (HH:MM:SS)
#SBATCH --output="_batch/VAE_%j.out" # nom du fichier de sortie #SBATCH --output="_batch/VAE_%j.out" # nom du fichier de sortie
#SBATCH --error="_batch/VAE_%j.err" # nom du fichier d'erreur (ici commun avec la sortie) #SBATCH --error="_batch/VAE_%j.err" # nom du fichier d'erreur (ici commun avec la sortie)
#SBATCH --mail-user=Jean-Luc.Parouty@grenoble-inp.fr #SBATCH --mail-user=Jean-Luc.Parouty@grenoble-inp.fr
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
MODULE_ENV="tensorflow-gpu/py3/2.0.0" MODULE_ENV="tensorflow-gpu/py3/2.0.0"
RUN_DIR="$WORK/fidle/VAE" RUN_DIR="$WORK/fidle/VAE"
RUN_IPYNB="01-VAE-with-MNIST.ipynb" RUN_IPYNB="07-VAE-with-CelebA-m.ipynb"
# ---- Welcome... # ---- Welcome...
...@@ -46,7 +46,7 @@ echo '------------------------------------------------------------' ...@@ -46,7 +46,7 @@ echo '------------------------------------------------------------'
# ---- Module # ---- Module
module purge module purge
module load $MODULE_ENV module load "$MODULE_ENV"
# ---- Run it... # ---- Run it...
......
fidle/pwk_ns.py 0 → 100644
+ 562
− 0
View file @ e39ca41e
# ==================================================================
# ____ _ _ _ __ __ _
# | _ \ _ __ __ _ ___| |_(_) ___ __ _| | \ \ / /__ _ __| | __
# | |_) | '__/ _` |/ __| __| |/ __/ _` | | \ \ /\ / / _ \| '__| |/ /
# | __/| | | (_| | (__| |_| | (_| (_| | | \ V V / (_) | | | <
# |_| |_| \__,_|\___|\__|_|\___\__,_|_| \_/\_/ \___/|_| |_|\_\
# module pwk
# ==================================================================
# A simple module to host some common functions for practical work
# Jean-Luc Parouty 2020
import os
import glob
import shutil
from datetime import datetime
import itertools
import datetime, time
import math
import numpy as np
from collections.abc import Iterable
import tensorflow as tf
from tensorflow import keras
from sklearn.metrics import confusion_matrix
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
#import seaborn as sn #IDRIS : module en cours d'installation
from IPython.display import display,Image,Markdown,HTML
import fidle.config as config
_save_figs = False
_figs_dir = './figs'
_figs_name = 'fig_'
_figs_id = 0
# -------------------------------------------------------------
# init_all
# -------------------------------------------------------------
#
def init( mplstyle='../fidle/mplstyles/custom.mplstyle',
cssfile='../fidle/css/custom.css',
places={ 'SOMEWHERE' : '/path/to/datasets'}):
update_keras_cache=False
# ---- Predifined places
#
predefined_places = config.locations
# ---- Load matplotlib style and css
#
matplotlib.style.use(mplstyle)
load_cssfile(cssfile)
# ---- Create subdirs
#
mkdir('./run')
# ---- Try to find where we are
#
place_name, dataset_dir = where_we_are({**places, **predefined_places})
# ---- If we are at IDRIS, we need to copy datasets/keras_cache to keras cache...
#
if place_name=='Fidle at IDRIS':
from_dir = f'{dataset_dir}/keras_cache/*.*'
to_dir = os.path.expanduser('~/.keras/datasets')
mkdir(to_dir)
for pathname in glob.glob(from_dir):
filename=os.path.basename(pathname)
destname=f'{to_dir}/{filename}'
if not os.path.isfile(destname):
shutil.copy(pathname, destname)
update_keras_cache=True
# ---- Hello world
print('\nFIDLE 2020 - Practical Work Module')
print('Version :', config.VERSION)
print('Run time : {}'.format(time.strftime("%A %-d %B %Y, %H:%M:%S")))
print('TensorFlow version :',tf.__version__)
print('Keras version :',tf.keras.__version__)
print('Current place :',place_name )
print('Datasets dir :',dataset_dir)
if update_keras_cache:
print('Update keras cache : Done')
return place_name, dataset_dir
# -------------------------------------------------------------
# Folder cooking
# -------------------------------------------------------------
#
def tag_now():
return datetime.datetime.now().strftime("%Y-%m-%d_%Hh%Mm%Ss")
def mkdir(path):
os.makedirs(path, mode=0o750, exist_ok=True)
def get_directory_size(path):
"""
Return the directory size, but only 1 level
args:
path : directory path
return:
size in Mo
"""
size=0
for f in os.listdir(path):
if os.path.isfile(path+'/'+f):
size+=os.path.getsize(path+'/'+f)
return size/(1024*1024)
# ------------------------------------------------------------------
# Where we are ?
# ------------------------------------------------------------------
#
def where_we_are(places):
for place_name, place_dir in places.items():
if os.path.isdir(place_dir):
return place_name,place_dir
print('** ERROR ** : Le dossier datasets est introuvable\n')
print(' Vous devez :\n')
print(' 1/ Récupérer le dossier datasets')
print(' Une archive (datasets.tar) est disponible via le repo Fidle.\n')
print(" 2/ Préciser la localisation de ce dossier datasets")
print(" Soit dans le fichier fidle/config.py (préférable)")
print(" Soit via un paramètre à la fonction ooo.init()\n")
print(' Par exemple :')
print(" ooo.init( places={ 'Chez-moi':'/tmp/datasets', 'Sur-mon-cluster':'/tests/datasets'}')\n")
print(' Note : Vous pouvez également déposer le dossier datasets directement dans votre home : ~/datasets\n\n')
assert False, 'datasets folder not found : Abort all.'
# -------------------------------------------------------------
# shuffle_dataset
# -------------------------------------------------------------
#
def shuffle_np_dataset(x, y):
"""
Shuffle a dataset (x,y)
args:
x,y : dataset
return:
x,y mixed
"""
assert (len(x) == len(y)), "x and y must have same size"
p = np.random.permutation(len(x))
return x[p], y[p]
def update_progress(what,i,imax, redraw=False):
"""
Display a text progress bar, as :
My progress bar : ############# 34%
args:
what : Progress bas name
i : Current progress
imax : Max value for i
return:
nothing
"""
bar_length = min(40,imax)
if (i%int(imax/bar_length))!=0 and i<imax and not redraw:
return
progress = float(i/imax)
block = int(round(bar_length * progress))
endofline = '\r' if progress<1 else '\n'
text = "{:16s} [{}] {:>5.1f}% of {}".format( what, "#"*block+"-"*(bar_length-block), progress*100, imax)
print(text, end=endofline)
def rmax(l):
"""
Recursive max() for a given iterable of iterables
Should be np.array of np.array or list of list, etc.
args:
l : Iterable of iterables
return:
max value
"""
maxi = float('-inf')
for item in l:
if isinstance(item, Iterable):
t = rmax(item)
else:
t = item
if t > maxi:
maxi = t
return maxi
def rmin(l):
"""
Recursive min() for a given iterable of iterables
Should be np.array of np.array or list of list, etc.
args:
l : Iterable of iterables
return:
min value
"""
mini = float('inf')
for item in l:
if isinstance(item, Iterable):
t = rmin(item)
else:
t = item
if t < mini:
mini = t
return mini
# -------------------------------------------------------------
# show_images
# -------------------------------------------------------------
#
def plot_images(x,y=None, indices='all', columns=12, x_size=1, y_size=1,
colorbar=False, y_pred=None, cm='binary',y_padding=0.35, spines_alpha=1,
fontsize=20, save_as='auto'):
"""
Show some images in a grid, with legends
args:
x : images - Shapes must be (-1,lx,ly) (-1,lx,ly,1) or (-1,lx,ly,3)
y : real classes or labels or None (None)
indices : indices of images to show or None for all (None)
columns : number of columns (12)
x_size,y_size : figure size (1), (1)
colorbar : show colorbar (False)
y_pred : predicted classes (None)
cm : Matplotlib color map (binary)
y_padding : Padding / rows (0.35)
spines_alpha : Spines alpha (1.)
font_size : Font size in px (20)
save_as : Filename to use if save figs is enable ('auto')
returns:
nothing
"""
if indices=='all': indices=range(len(x))
draw_labels = (y is not None)
draw_pred = (y_pred is not None)
rows = math.ceil(len(indices)/columns)
fig=plt.figure(figsize=(columns*x_size, rows*(y_size+y_padding)))
n=1
for i in indices:
axs=fig.add_subplot(rows, columns, n)
n+=1
# ---- Shape is (lx,ly)
if len(x[i].shape)==2:
xx=x[i]
# ---- Shape is (lx,ly,n)
if len(x[i].shape)==3:
(lx,ly,lz)=x[i].shape
if lz==1:
xx=x[i].reshape(lx,ly)
else:
xx=x[i]
img=axs.imshow(xx, cmap = cm, interpolation='lanczos')
axs.spines['right'].set_visible(True)
axs.spines['left'].set_visible(True)
axs.spines['top'].set_visible(True)
axs.spines['bottom'].set_visible(True)
axs.spines['right'].set_alpha(spines_alpha)
axs.spines['left'].set_alpha(spines_alpha)
axs.spines['top'].set_alpha(spines_alpha)
axs.spines['bottom'].set_alpha(spines_alpha)
axs.set_yticks([])
axs.set_xticks([])
if draw_labels and not draw_pred:
axs.set_xlabel(y[i],fontsize=fontsize)
if draw_labels and draw_pred:
if y[i]!=y_pred[i]:
axs.set_xlabel(f'{y_pred[i]} ({y[i]})',fontsize=fontsize)
axs.xaxis.label.set_color('red')
else:
axs.set_xlabel(y[i],fontsize=fontsize)
if colorbar:
fig.colorbar(img,orientation="vertical", shrink=0.65)
save_fig(save_as)
plt.show()
def plot_image(x,cm='binary', figsize=(4,4),save_as='auto'):
"""
Draw a single image.
Image shape can be (lx,ly), (lx,ly,1) or (lx,ly,n)
args:
x : image as np array
cm : color map ('binary')
figsize : fig size (4,4)
"""
# ---- Shape is (lx,ly)
if len(x.shape)==2:
xx=x
# ---- Shape is (lx,ly,n)
if len(x.shape)==3:
(lx,ly,lz)=x.shape
if lz==1:
xx=x.reshape(lx,ly)
else:
xx=x
# ---- Draw it
plt.figure(figsize=figsize)
plt.imshow(xx, cmap = cm, interpolation='lanczos')
save_fig(save_as)
plt.show()
# -------------------------------------------------------------
# show_history
# -------------------------------------------------------------
#
def plot_history(history, figsize=(8,6),
plot={"Accuracy":['accuracy','val_accuracy'], 'Loss':['loss', 'val_loss']},
save_as='auto'):
"""
Show history
args:
history: history
figsize: fig size
plot: list of data to plot : {<title>:[<metrics>,...], ...}
"""
fig_id=0
for title,curves in plot.items():
plt.figure(figsize=figsize)
plt.title(title)
plt.ylabel(title)
plt.xlabel('Epoch')
for c in curves:
plt.plot(history.history[c])
plt.legend(curves, loc='upper left')
if save_as=='auto':
figname='auto'
else:
figname=f'{save_as}_{fig_id}'
fig_id+=1
save_fig(figname)
plt.show()
# -------------------------------------------------------------
# plot_confusion_matrix
# -------------------------------------------------------------
# Bug in Matplotlib 3.1.1
#
def plot_confusion_matrix(cm,
title='Confusion matrix',
figsize=(12,8),
cmap="gist_heat_r",
vmin=0,
vmax=1,
xticks=5,yticks=5,
annot=True,
save_as='auto'):
"""
given a sklearn confusion matrix (cm), make a nice plot
Note:bug in matplotlib 3.1.1
Args:
cm: confusion matrix from sklearn.metrics.confusion_matrix
title: the text to display at the top of the matrix
figsize: Figure size (12,8)
cmap: color map (gist_heat_r)
vmi,vmax: Min/max 0 and 1
annot: Annotation or just colors (True)
"""
accuracy = np.trace(cm) / float(np.sum(cm))
misclass = 1 - accuracy
plt.figure(figsize=figsize)
sn.heatmap(cm, linewidths=1, linecolor="#ffffff",square=True,
cmap=cmap, xticklabels=xticks, yticklabels=yticks,
vmin=vmin,vmax=vmax,annot=annot)
plt.ylabel('True label')
plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
save_fig(save_as)
plt.show()
def display_confusion_matrix(y_true,y_pred,labels=None,color='green',
font_size='12pt', title="#### Confusion matrix is :"):
"""
Show a confusion matrix for a predictions.
see : sklearn.metrics.confusion_matrix
Args:
y_true Real classes
y_pred Predicted classes
labels List of classes to show in the cm
color: Color for the palette (green)
font_size: Values font size
title: the text to display at the top of the matrix
"""
assert (labels!=None),"Label must be set"
if title != None : display(Markdown(title))
cm = confusion_matrix( y_true,y_pred, normalize="true", labels=labels)
df=pd.DataFrame(cm)
cmap = sn.light_palette(color, as_cmap=True)
df.style.set_properties(**{'font-size': '20pt'})
display(df.style.format('{:.2f}') \
.background_gradient(cmap=cmap)
.set_properties(**{'font-size': font_size}))
def plot_donut(values, labels, colors=["lightsteelblue","coral"], figsize=(6,6), title=None, save_as='auto'):
"""
Draw a donut
args:
values : list of values
labels : list of labels
colors : list of color (["lightsteelblue","coral"])
figsize : size of figure ( (6,6) )
return:
nothing
"""
# ---- Title or not
if title != None : display(Markdown(title))
# ---- Donut
plt.figure(figsize=figsize)
# ---- Draw a pie chart..
plt.pie(values, labels=labels,
colors = colors, autopct='%1.1f%%', startangle=70, pctdistance=0.85,
textprops={'fontsize': 18},
wedgeprops={"edgecolor":"w",'linewidth': 5, 'linestyle': 'solid', 'antialiased': True})
# ---- ..with a white circle
circle = plt.Circle((0,0),0.70,fc='white')
ax = plt.gca()
ax.add_artist(circle)
# Equal aspect ratio ensures that pie is drawn as a circle
plt.axis('equal')
plt.tight_layout()
save_fig(save_as)
plt.show()
def plot_multivariate_serie(sequence, labels=None, predictions=None, only_features=None,
columns=3, width=5,height=4,wspace=0.3,hspace=0.2,
save_as='auto', time_dt=1):
sequence_len = len(sequence)
features_len = sequence.shape[1]
if only_features is None : only_features=range(features_len)
if labels is None : labels=range(features_len)
t = np.arange(sequence_len)
if predictions is None:
dt = 0
else:
dt = len(predictions)
sequence_with_pred = sequence.copy()
sequence_with_pred[-dt:]=predictions
rows = math.ceil(features_len/columns)
fig = plt.figure(figsize=(columns*width, rows*height))
fig.subplots_adjust(wspace=0.3,hspace=0.2)
n=1
for i in only_features:
ax=fig.add_subplot(rows, columns, n)
ax.plot(t[:-dt], sequence[:-dt,i], '-', linewidth=1, color='steelblue', label=labels[i])
ax.plot(t[:-dt], sequence[:-dt,i], 'o', markersize=4, color='steelblue')
ax.plot(t[-dt-1:], sequence[-dt-1:,i],'--o', linewidth=1, fillstyle='none', markersize=6, color='steelblue')
if predictions is not None:
ax.plot(t[-dt-1:], sequence_with_pred[-dt-1:,i], '--', linewidth=1, fillstyle='full', markersize=6, color='red')
ax.plot(t[-dt:], predictions[:,i], 'o', linewidth=1, fillstyle='full', markersize=6, color='red')
ax.legend(loc="upper left")
n+=1
save_fig(save_as)
plt.show()
def set_save_fig(save=True, figs_dir='./figs', figs_name='fig_', figs_id=0):
"""
Set save_fig parameters
Default figs name is <figs_name><figs_id>.{png|svg}
args:
save : Boolean, True to save figs (True)
figs_dir : Path to save figs (./figs)
figs_name : Default basename for figs (figs_)
figs_id : Start id for figs name (0)
"""
global _save_figs, _figs_dir, _figs_name, _figs_id
_save_figs = save
_figs_dir = figs_dir
_figs_name = figs_name
_figs_id = figs_id
print(f'Save figs : {_save_figs}')
print(f'Path figs : {_figs_dir}')
def save_fig(filename='auto', png=True, svg=False):
"""
Save current figure
args:
filename : Image filename ('auto')
png : Boolean. Save as png if True (True)
svg : Boolean. Save as svg if True (False)
"""
global _save_figs, _figs_dir, _figs_name, _figs_id
if not _save_figs : return
mkdir(_figs_dir)
if filename=='auto':
path=f'{_figs_dir}/{_figs_name}{_figs_id:02d}'
else:
path=f'{_figs_dir}/{filename}'
if png : plt.savefig( f'{path}.png')
if svg : plt.savefig( f'{path}.png')
if filename=='auto': _figs_id+=1
def subtitle(t):
display(Markdown(f'<br>**{t}**'))
def display_md(md_text):
display(Markdown(md_text))
def display_img(img):
display(Image(img))
def hdelay(sec):
return str(datetime.timedelta(seconds=int(sec)))
def hsize(num, suffix='o'):
for unit in ['','K','M','G','T','P','E','Z']:
if abs(num) < 1024.0:
return f'{num:3.1f} {unit}{suffix}'
num /= 1024.0
return f'{num:.1f} Y{suffix}'
def load_cssfile(cssfile):
if cssfile is None: return
styles = open(cssfile, "r").read()
display(HTML(styles))
def np_print(*args, format={'float': '{:6.3f}'.format}):
with np.printoptions(formatter=format):
for a in args:
print(a)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment