GTSRB.Keras3/modules/ImagenetClassnames.py

+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                         Imagenet Classes
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2024
+
+
+import os
+import json
+
+class ImagenetClassnames:
+
+    classes_file = 'ImagenetClassnames.json'
+
+    def __init__(self):
+        path = os.path.abspath(__file__)
+        dir_path = os.path.dirname(path)
+        with open(f'{dir_path}/{self.classes_file}') as f:
+            self.classes = json.load(f)
+        print(f'Imagenet classes loaded ({len(self.classes)} classes)')
+
+
+    def get(self, classes_id, top_n=2):
+        top_classes = [self.classes[str(i)] for i in classes_id[-top_n:]]
+        return top_classes
\ No newline at end of file

GTSRB.Keras3/modules/my_TensorboardCallback.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                     Tensorboard callback
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+#
+# See : https://keras.io/api/callbacks/
+# See : https://keras.io/guides/writing_your_own_callbacks/
+# See : https://pytorch.org/docs/stable/tensorboard.html
+
+import keras
+from torch.utils.tensorboard import SummaryWriter
+
+
+class TensorboardCallback(keras.callbacks.Callback):
+
+    def __init__(self, log_dir=None):
+        '''
+        Init callback
+        Args:
+            log_dir : log directory
+        '''
+        self.writer = SummaryWriter(log_dir=log_dir)
+
+
+    def on_epoch_end(self, epoch, logs=None):
+        '''
+        Record logs at epoch end
+        '''
+
+        # ---- Records all metrics (very simply)
+        #
+        # for k,v in logs.items():
+        #     self.writer.add_scalar(k,v, epoch)
+
+        # ---- Records and group specific metrics
+        #
+        self.writer.add_scalars('Accuracy',
+                                {'Train':logs['accuracy'],
+                                  'Validation':logs['val_accuracy']},
+                                 epoch )
+        
+        self.writer.add_scalars('Loss',
+                                {'Train':logs['loss'],
+                                  'Validation':logs['val_loss']},
+                                 epoch )
+

GTSRB.Keras3/modules/my_loader.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                           Dataset reader
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+
+import h5py
+import os
+import fidle
+
+
+def read_dataset(enhanced_dir, dataset_name, scale=1):
+    '''
+    Reads h5 dataset
+    Args:
+        filename     : datasets filename
+        dataset_name : dataset name, without .h5
+    Returns:    
+        x_train,y_train, x_test,y_test data, x_meta,y_meta
+    '''
+
+    # ---- Read dataset
+    #
+    chrono=fidle.Chrono()
+    chrono.start()
+    filename = f'{enhanced_dir}/{dataset_name}.h5'
+    with  h5py.File(filename,'r') as f:
+        x_train = f['x_train'][:]
+        y_train = f['y_train'][:]
+        x_test  = f['x_test'][:]
+        y_test  = f['y_test'][:]
+        x_meta  = f['x_meta'][:]
+        y_meta  = f['y_meta'][:]
+
+    # ---- Rescale 
+    #
+    print('Original shape  :', x_train.shape, y_train.shape)
+    x_train,y_train, x_test,y_test = fidle.utils.rescale_dataset(x_train,y_train,x_test,y_test, scale=scale)
+    print('Rescaled shape  :', x_train.shape, y_train.shape)
+
+    # ---- Shuffle
+    #
+    x_train,y_train=fidle.utils.shuffle_np_dataset(x_train,y_train)
+
+    # ---- done
+    #
+    duration = chrono.get_delay()
+    size     = fidle.utils.hsize(os.path.getsize(filename))
+    print(f'\nDataset "{dataset_name}" is loaded and shuffled. ({size} in {duration})')
+    return x_train,y_train, x_test,y_test, x_meta,y_meta
+
+
+
+
+print('Module my_loader loaded.')
\ No newline at end of file

GTSRB.Keras3/modules/my_models.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                         Some nice models
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+
+import keras
+
+# ------------------------------------------------------------------
+# -- A simple model, for 24x24 or 48x48 images                    --
+# ------------------------------------------------------------------
+#
+def get_model_01(lx,ly,lz):
+    
+    model = keras.models.Sequential()
+
+    model.add( keras.layers.Input((lx,ly,lz)) )
+    
+    model.add( keras.layers.Conv2D(96, (3,3), activation='relu' ))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.2))
+
+    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.2))
+
+    model.add( keras.layers.Flatten()) 
+    model.add( keras.layers.Dense(1500, activation='relu'))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Dense(43, activation='softmax'))
+    return model
+    
+
+# ------------------------------------------------------------------
+# -- A more sophisticated model, for 48x48 images                 --
+# ------------------------------------------------------------------
+#
+def get_model_02(lx,ly,lz):
+    model = keras.models.Sequential()
+    
+    model.add( keras.layers.Input((lx,ly,lz)) )
+    
+    model.add( keras.layers.Conv2D(32, (3,3),   activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Flatten()) 
+    model.add( keras.layers.Dense(1152, activation='relu'))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Dense(43, activation='softmax'))
+    return model
+
+
+
+def get_model(name, lx,ly,lz):
+    '''
+    Return a model given by name
+    Args:
+        f_name : function name to retreive model
+        lxly,lz : inpuy shape 
+    Returns:
+        model
+    '''
+    if name=='model_01' : return get_model_01(lx,ly,lz)
+    if name=='model_02' : return get_model_01(lx,ly,lz)
+    print('*** Model not found : ', name)
+    return None
+
+# A More fun version ;-)
+def get_model2(name, lx,ly,lz):
+    get_model=globals()['get_'+name]
+    model=get_model(lx,ly,lz)
+    return model
+
+
+
+print('Module my_models loaded.')
\ No newline at end of file

GTSRB.Keras3/modules/my_tools.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|          A small traffic sign classifier
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+import numpy as np
+import matplotlib.pyplot as plt
+import fidle
+
+
+def show_prediction( prediction, x, y, x_meta ):
+
+    # ---- A prediction is just the output layer
+    #
+    fidle.utils.subtitle("Output layer from model is (x100) :")
+    with np.printoptions(precision=2, suppress=True, linewidth=95):
+        print(prediction*100)
+
+    # ---- Graphic visualisation
+    #
+    fidle.utils.subtitle("Graphically :")
+    plt.figure(figsize=(8,2))
+    plt.bar(range(43), prediction[0], align='center', alpha=0.5)
+    plt.ylabel('Probability')
+    plt.ylim((0,1))
+    plt.xlabel('Class')
+    plt.title('Trafic Sign prediction')
+    fidle.scrawler.save_fig('05-prediction-proba')
+    plt.show()
+
+    # ---- Predict class
+    #
+    p = np.argmax(prediction)
+
+    # ---- Show result
+    #
+    fidle.utils.subtitle('In pictures :')
+    print("\nThe image :               Prediction :            Real stuff:")
+    fidle.scrawler.images([x,x_meta[p], x_meta[y]], [p,p,y], range(3),  columns=3,  x_size=1.5, y_size=1.5, save_as='06-prediction-images')
+
+    if p==y:
+        print("YEEES ! that's right!")
+    else:
+        print("oups, that's wrong ;-(")
\ No newline at end of file

GTSRB/04-Data-augmentation.ipynb

-%% Cell type:markdown id: tags:
-
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
-
-# <!-- TITLE --> [GTS4] - CNN with GTSRB dataset - Data augmentation
-<!-- DESC --> Episode 4 : Improving the results with data augmentation
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
-  - Trying to improve training by **enhancing the data**
-  - Using Keras' **data augmentation utilities**, finding their limits...
-
-The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
-The final aim is to recognise them !
-Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
-
-
-## What we're going to do :
- - Increase and improve the training dataset
- - Identify the limits of these tools
-
-## Step 1 - Import and init
-### 1.1 - Python
-
-%% Cell type:code id: tags:
-
-``` python
-import tensorflow as tf
-from tensorflow import keras
-from tensorflow.keras.callbacks import TensorBoard
-
-import numpy as np
-import h5py
-
-from sklearn.metrics import confusion_matrix
-
-import matplotlib.pyplot as plt
-import seaborn as sn
-import os, sys, time, random
-
-from importlib import reload
-
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
-```
-
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.4.3
-    Run time             : Friday 28 February 2020, 14:23:29
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
-%% Cell type:markdown id: tags:
-
-### 1.2 - Where are we ?
-
-%% Cell type:code id: tags:
-
-``` python
-place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv("SCRATCH_DIR","")}/PROJECTS/pr-fidle/datasets/GTSRB',
-                                       'IDRIS'  : f'{os.getenv("WORK","")}/datasets/GTSRB',
-                                       'HOME'   : f'{os.getenv("HOME","")}/datasets/GTSRB'} )
-```
-
-%% Output
-
-    Well, we should be at GRICAD !
-    We are going to use: /bettik/PROJECTS/pr-fidle/datasets/GTSRB
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Dataset loader
-Dataset is one of the saved dataset: RGB25, RGB35, L25, L35, etc.
-First of all, we're going to use a smart dataset : **set-24x24-L**
-(with a GPU, it only takes 35'' compared to more than 5' with a CPU !)
-
-%% Cell type:code id: tags:
-
-``` python
-%%time
-
-def read_dataset(dataset_dir, name):
-    '''Reads h5 dataset from dataset_dir
-    Args:
-        dataset_dir : datasets dir
-        name        : dataset name, without .h5
-    Returns:    x_train,y_train,x_test,y_test data'''
-    # ---- Read dataset
-    filename=f'{dataset_dir}/{name}.h5'
-    with  h5py.File(filename,'r') as f:
-        x_train = f['x_train'][:]
-        y_train = f['y_train'][:]
-        x_test  = f['x_test'][:]
-        y_test  = f['y_test'][:]
-
-    # ---- done
-    print('Dataset "{}" is loaded. ({:.1f} Mo)\n'.format(name,os.path.getsize(filename)/(1024*1024)))
-    return x_train,y_train,x_test,y_test
-```
-
-%% Output
-
-    CPU times: user 0 ns, sys: 0 ns, total: 0 ns
-    Wall time: 5.48 µs
-
-%% Cell type:markdown id: tags:
-
-## Step 3 - Models
-We will now build a model and train it...
-
-This is my model ;-)
-
-%% Cell type:code id: tags:
-
-``` python
-# A basic model
-#
-def get_model_v1(lx,ly,lz):
-
-    model = keras.models.Sequential()
-
-    model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz)))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Flatten())
-    model.add( keras.layers.Dense(1500, activation='relu'))
-    model.add( keras.layers.Dropout(0.5))
-
-    model.add( keras.layers.Dense(43, activation='softmax'))
-    return model
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 4 - Callbacks
-We prepare 2 kind callbacks :  TensorBoard and Model backup
-
-%% Cell type:code id: tags:
-
-``` python
-ooo.mkdir('./run/models')
-ooo.mkdir('./run/logs')
-
-# ---- Callback tensorboard
-log_dir = "./run/logs/tb_" + ooo.tag_now()
-tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
-
-# ---- Callback ModelCheckpoint - Save best model
-save_dir = "./run/models/best-model.h5"
-bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True)
-
-# ---- Callback ModelCheckpoint - Save model each epochs
-save_dir = "./run/models/model-{epoch:04d}.h5"
-savemodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0)
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 5 - Load and prepare dataset
-### 5.1 - Load
-
-%% Cell type:code id: tags:
-
-``` python
-x_train,y_train,x_test,y_test = read_dataset(dataset_dir,'set-24x24-L')
-```
-
-%% Output
-
-    Dataset "set-24x24-L" is loaded. (228.8 Mo)
-    
-
-%% Cell type:markdown id: tags:
-
-### 5.2 - Data augmentation
-
-%% Cell type:code id: tags:
-
-``` python
-datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
-                             featurewise_std_normalization=False,
-                             width_shift_range=0.1,
-                             height_shift_range=0.1,
-                             zoom_range=0.2,
-                             shear_range=0.1,
-                             rotation_range=10.)
-datagen.fit(x_train)
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 6 - Train the model
-**Get the shape of my data :**
-
-%% Cell type:code id: tags:
-
-``` python
-(n,lx,ly,lz) = x_train.shape
-print("Images of the dataset have this folowing shape : ",(lx,ly,lz))
-```
-
-%% Output
-
-    Images of the dataset have this folowing shape :  (24, 24, 1)
-
-%% Cell type:markdown id: tags:
-
-**Get and compile a model, with the data shape :**
-
-%% Cell type:code id: tags:
-
-``` python
-model = get_model_v1(lx,ly,lz)
-
-# model.summary()
-
-model.compile(optimizer='adam',
-              loss='sparse_categorical_crossentropy',
-              metrics=['accuracy'])
-```
-
-%% Cell type:markdown id: tags:
-
-**Train it :**
-Note : La courbe d'apprentissage est visible en temps réel avec Tensorboard :
-`#tensorboard --logdir ./run/logs`
-
-%% Cell type:code id: tags:
-
-``` python
-%%time
-
-batch_size = 64
-epochs     = 10
-
-# ---- Shuffle train data
-#x_train,y_train=ooo.shuffle_np_dataset(x_train,y_train)
-
-# ---- Train
-#
-history = model.fit(  datagen.flow(x_train, y_train, batch_size=batch_size),
-                      steps_per_epoch = int(x_train.shape[0]/batch_size),
-                      epochs=epochs,
-                      verbose=1,
-                      validation_data=(x_test, y_test),
-                      callbacks=[tensorboard_callback, bestmodel_callback, savemodel_callback] )
-
-model.save('./run/models/last-model.h5')
-```
-
-%% Output
-
-    Train for 612 steps, validate on 12630 samples
-    Epoch 1/10
-      1/612 [..............................] - ETA: 1:57:56 - loss: 3.7702 - accuracy: 0.0156WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.238514). Check your callbacks.
-    612/612 [==============================] - 15s 24ms/step - loss: 2.1205 - accuracy: 0.4047 - val_loss: 0.8360 - val_accuracy: 0.7512
-    Epoch 2/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.9110 - accuracy: 0.7209 - val_loss: 0.4255 - val_accuracy: 0.8871
-    Epoch 3/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.6064 - accuracy: 0.8142 - val_loss: 0.3533 - val_accuracy: 0.9116
-    Epoch 4/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.4648 - accuracy: 0.8560 - val_loss: 0.2755 - val_accuracy: 0.9246
-    Epoch 5/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.3765 - accuracy: 0.8824 - val_loss: 0.2727 - val_accuracy: 0.9295
-    Epoch 6/10
-    612/612 [==============================] - 13s 20ms/step - loss: 0.3227 - accuracy: 0.8989 - val_loss: 0.2075 - val_accuracy: 0.9487
-    Epoch 7/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.2795 - accuracy: 0.9126 - val_loss: 0.2256 - val_accuracy: 0.9431
-    Epoch 8/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.2508 - accuracy: 0.9205 - val_loss: 0.2228 - val_accuracy: 0.9453
-    Epoch 9/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.2268 - accuracy: 0.9282 - val_loss: 0.2177 - val_accuracy: 0.9459
-    Epoch 10/10
-    612/612 [==============================] - 12s 20ms/step - loss: 0.2088 - accuracy: 0.9358 - val_loss: 0.1811 - val_accuracy: 0.9509
-    CPU times: user 2min 16s, sys: 4.9 s, total: 2min 20s
-    Wall time: 2min 14s
-
-%% Cell type:markdown id: tags:
-
-**Evaluate it :**
-
-%% Cell type:code id: tags:
-
-``` python
-max_val_accuracy = max(history.history["val_accuracy"])
-print("Max validation accuracy is : {:.4f}".format(max_val_accuracy))
-```
-
-%% Output
-
-    Max validation accuracy is : 0.9509
-
-%% Cell type:code id: tags:
-
-``` python
-score = model.evaluate(x_test, y_test, verbose=0)
-
-print('Test loss      : {:5.4f}'.format(score[0]))
-print('Test accuracy  : {:5.4f}'.format(score[1]))
-```
-
-%% Output
-
-    Test loss      : 0.1811
-    Test accuracy  : 0.9509
-
-%% Cell type:markdown id: tags:
-
-## Step 7 - History
-The return of model.fit() returns us the learning history
-
-%% Cell type:code id: tags:
-
-``` python
-ooo.plot_history(history)
-```
-
-%% Output
-
-
-
-
-
-%% Cell type:markdown id: tags:
-
-## Step 8 - Evaluate best model
-
-%% Cell type:markdown id: tags:
-
-### 8.1 - Restore best model :
-
-%% Cell type:code id: tags:
-
-``` python
-loaded_model = tf.keras.models.load_model('./run/models/best-model.h5')
-# best_model.summary()
-print("Loaded.")
-```
-
-%% Output
-
-    Loaded.
-
-%% Cell type:markdown id: tags:
-
-### 8.2 - Evaluate it :
-
-%% Cell type:code id: tags:
-
-``` python
-score = loaded_model.evaluate(x_test, y_test, verbose=0)
-
-print('Test loss      : {:5.4f}'.format(score[0]))
-print('Test accuracy  : {:5.4f}'.format(score[1]))
-```
-
-%% Output
-
-    Test loss      : 0.1811
-    Test accuracy  : 0.9509
-
-%% Cell type:markdown id: tags:
-
-**Plot confusion matrix**
-
-%% Cell type:code id: tags:
-
-``` python
-y_pred   = model.predict_classes(x_test)
-conf_mat = confusion_matrix(y_test,y_pred, normalize="true", labels=range(43))
-
-ooo.plot_confusion_matrix(conf_mat,annot=False)
-```
-
-%% Output
-
-
-
-%% Cell type:markdown id: tags:
-
-<div class="todo">
-     What you can do:
-    <ul>
-        <li>Try different datasets / models</li>
-        <li>Test different hyperparameters (epochs, batch size, optimization, etc.)</li>
-        <li>What's the best strategy?  How to compare?</li>
-    </ul>
-
-</div>
-
-%% Cell type:markdown id: tags:
-
----
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>

GTSRB/05-Full-convolutions.ipynb

-%% Cell type:markdown id: tags:
-
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
-
-# <!-- TITLE --> [GTS5] - CNN with GTSRB dataset - Full convolutions
-<!-- DESC --> Episode 5 : A lot of models, a lot of datasets and a lot of results.
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
-  - Try multiple solutions
-  - Design a generic and batch-usable code
-
-The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
-The final aim is to recognise them !
-Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
-
-
-## What we're going to do :
-
-Our main steps:
- - Try n models with n datasets
- - Save a Pandas/h5 report
- - Write to be run in batch mode
-
-## Step 1 - Import
-### 1.1 - Python
-
-%% Cell type:code id: tags:
-
-``` python
-import tensorflow as tf
-from tensorflow import keras
-
-import numpy as np
-import h5py
-import os,time,json
-import random
-
-from IPython.display import display
-
-VERSION='1.6'
-```
-
-%% Cell type:markdown id: tags:
-
-### 1.2 - Where are we ?
-
-%% Cell type:code id: tags:
-
-``` python
-# At GRICAD
-dataset_dir = '/bettik/PROJECTS/pr-fidle/datasets/GTSRB/'
-
-# At IDRIS
-# dataset_dir = f'{os.getenv("WORK","")}/datasets/GTSRB'
-
-# At Home
-# dataset_dir = f'{os.getenv("HOME","")}/datasets/GTSRB'
-
-print(f'We will use : dataset_dir={dataset_dir}')
-```
-
-%% Output
-
-    We will use : dataset_dir=/bettik/PROJECTS/pr-fidle/datasets/GTSRB
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Init and start
-
-%% Cell type:code id: tags:
-
-``` python
-# ---- Where I am ?
-now    = time.strftime("%A %d %B %Y - %Hh%Mm%Ss")
-here   = os.getcwd()
-random.seed(time.time())
-tag_id = '{:06}'.format(random.randint(0,99999))
-
-# ---- Who I am ?
-oar_id   = os.getenv("OAR_JOB_ID",  "??")
-slurm_id = os.getenv("SLURM_JOBID", "??")
-print('\nFull Convolutions Notebook')
-print('  Version            : {}'.format(VERSION))
-print('  Now is             : {}'.format(now))
-print('  OAR id             : {}'.format(oar_id))
-print('  SLURM id           : {}'.format(slurm_id))
-print('  Tag id             : {}'.format(tag_id))
-print('  Working directory  : {}'.format(here))
-print('  Dataset_dir        : {}'.format(dataset_dir))
-print('  TensorFlow version :',tf.__version__)
-print('  Keras version      :',tf.keras.__version__)
-print('  for tensorboard    : --logdir {}/run/logs_{}'.format(here,tag_id))
-```
-
-%% Output
-
-    
-    Full Convolutions Notebook
-      Version            : 1.6
-      Now is             : Friday 28 February 2020 - 15h06m25s
-      OAR id             : 5878410
-      SLURM id           : ??
-      Tag id             : 083052
-      Working directory  : /home/paroutyj/fidle/GTSRB
-      Dataset_dir        : /bettik/PROJECTS/pr-fidle/datasets/GTSRB
-      TensorFlow version : 2.0.0
-      Keras version      : 2.2.4-tf
-      for tensorboard    : --logdir /home/paroutyj/fidle/GTSRB/run/logs_083052
-
-%% Cell type:markdown id: tags:
-
-## Step 3 - Dataset loading
-
-%% Cell type:code id: tags:
-
-``` python
-def read_dataset(dataset_dir, name):
-    '''Reads h5 dataset from dataset_dir
-    Args:
-        dataset_dir : datasets dir
-        name        : dataset name, without .h5
-    Returns:    x_train,y_train,x_test,y_test data'''
-    # ---- Read dataset
-    filename=f'{dataset_dir}/{name}.h5'
-    with  h5py.File(filename,'r') as f:
-        x_train = f['x_train'][:]
-        y_train = f['y_train'][:]
-        x_test  = f['x_test'][:]
-        y_test  = f['y_test'][:]
-
-    # ---- done
-    return x_train,y_train,x_test,y_test
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 4 - Models collection
-
-%% Cell type:code id: tags:
-
-``` python
-
-# A basic model
-#
-def get_model_v1(lx,ly,lz):
-
-    model = keras.models.Sequential()
-
-    model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz)))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Flatten())
-    model.add( keras.layers.Dense(1500, activation='relu'))
-    model.add( keras.layers.Dropout(0.5))
-
-    model.add( keras.layers.Dense(43, activation='softmax'))
-    return model
-
-# A more sophisticated model
-#
-def get_model_v2(lx,ly,lz):
-    model = keras.models.Sequential()
-
-    model.add( keras.layers.Conv2D(64, (3, 3), padding='same', input_shape=(lx,ly,lz), activation='relu'))
-    model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(128, (3, 3), padding='same', activation='relu'))
-    model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(256, (3, 3), padding='same',activation='relu'))
-    model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Flatten())
-    model.add( keras.layers.Dense(512, activation='relu'))
-    model.add( keras.layers.Dropout(0.5))
-    model.add( keras.layers.Dense(43, activation='softmax'))
-    return model
-
-def get_model_v3(lx,ly,lz):
-    model = keras.models.Sequential()
-    model.add(tf.keras.layers.Conv2D(32, (5, 5), padding='same',  activation='relu', input_shape=(lx,ly,lz)))
-    model.add(tf.keras.layers.BatchNormalization(axis=-1))
-    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add(tf.keras.layers.Dropout(0.2))
-
-    model.add(tf.keras.layers.Conv2D(64, (5, 5), padding='same',  activation='relu'))
-    model.add(tf.keras.layers.BatchNormalization(axis=-1))
-    model.add(tf.keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu'))
-    model.add(tf.keras.layers.BatchNormalization(axis=-1))
-    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add(tf.keras.layers.Dropout(0.2))
-
-    model.add(tf.keras.layers.Flatten())
-    model.add(tf.keras.layers.Dense(512, activation='relu'))
-    model.add(tf.keras.layers.BatchNormalization())
-    model.add(tf.keras.layers.Dropout(0.4))
-
-    model.add(tf.keras.layers.Dense(43, activation='softmax'))
-    return model
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 5 - Multiple datasets, multiple models ;-)
-
-%% Cell type:code id: tags:
-
-``` python
-def multi_run(dataset_dir, datasets, models, datagen=None,
-              train_size=1, test_size=1, batch_size=64, epochs=16,
-              verbose=0, extension_dir='last'):
-    """
-    Launches a dataset-model combination
-    args:
-        dataset_dir    : Directory of the datasets
-        datasets       : List of dataset (whitout .h5)
-        models         : List of model like { "model name":get_model(), ...}
-        datagen        : Data generator or None (None)
-        train_size     : % of train dataset to use. 1 mean all. (1)
-        test_size      : % of test dataset to use.  1 mean all. (1)
-        batch_size     : Batch size (64)
-        epochs         : Number of epochs (16)
-        verbose        : Verbose level (0)
-        extension_dir  : postfix for logs and models dir (_last)
-    return:
-        report        : Report as a dict for Pandas.
-    """
-    # ---- Logs and models dir
-    #
-    os.makedirs(f'./run/logs_{extension_dir}',   mode=0o750, exist_ok=True)
-    os.makedirs(f'./run/models_{extension_dir}', mode=0o750, exist_ok=True)
-
-    # ---- Columns of output
-    #
-    output={}
-    output['Dataset'] = []
-    output['Size']    = []
-    for m in models:
-        output[m+'_Accuracy'] = []
-        output[m+'_Duration'] = []
-
-    # ---- Let's go
-    #
-    for d_name in datasets:
-        print("\nDataset : ",d_name)
-
-        # ---- Read dataset
-        x_train,y_train,x_test,y_test = read_dataset(dataset_dir, d_name)
-        d_size=os.path.getsize(f'{dataset_dir}/{d_name}.h5')/(1024*1024)
-        output['Dataset'].append(d_name)
-        output['Size'].append(d_size)
-
-        # ---- Get the shape
-        (n,lx,ly,lz) = x_train.shape
-        n_train = int( x_train.shape[0] * train_size )
-        n_test  = int( x_test.shape[0]  * test_size  )
-
-        # ---- For each model
-        for m_name,m_function in models.items():
-            print("    Run model {}  : ".format(m_name), end='')
-            # ---- get model
-            try:
-                model=m_function(lx,ly,lz)
-                # ---- Compile it
-                model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
-                # ---- Callbacks tensorboard
-                log_dir = f"./run/logs_{extension_dir}/tb_{d_name}_{m_name}"
-                tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
-                # ---- Callbacks bestmodel
-                save_dir = f"./run/models_{extension_dir}/model_{d_name}_{m_name}.h5"
-                bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True)
-                # ---- Train
-                start_time = time.time()
-                if datagen==None:
-                    # ---- No data augmentation (datagen=None) --------------------------------------
-                    history = model.fit(x_train[:n_train], y_train[:n_train],
-                                        batch_size      = batch_size,
-                                        epochs          = epochs,
-                                        verbose         = verbose,
-                                        validation_data = (x_test[:n_test], y_test[:n_test]),
-                                        callbacks       = [tensorboard_callback, bestmodel_callback])
-                else:
-                    # ---- Data augmentation (datagen given) ----------------------------------------
-                    datagen.fit(x_train)
-                    history = model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),
-                                        steps_per_epoch = int(n_train/batch_size),
-                                        epochs          = epochs,
-                                        verbose         = verbose,
-                                        validation_data = (x_test[:n_test], y_test[:n_test]),
-                                        callbacks       = [tensorboard_callback, bestmodel_callback])
-
-                # ---- Result
-                end_time = time.time()
-                duration = end_time-start_time
-                accuracy = max(history.history["val_accuracy"])*100
-                #
-                output[m_name+'_Accuracy'].append(accuracy)
-                output[m_name+'_Duration'].append(duration)
-                print(f"Accuracy={accuracy:.2f} and Duration={duration:.2f}")
-            except:
-                output[m_name+'_Accuracy'].append('0')
-                output[m_name+'_Duration'].append('999')
-                print('-')
-    return output
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 6 - Run !
-
-%% Cell type:code id: tags:
-
-``` python
-start_time = time.time()
-
-print('\n---- Run','-'*50)
-
-# --------- Datasets, models, and more.. -----------------------------------
-#
-# ---- For tests
-# datasets   = ['set-24x24-L', 'set-24x24-RGB']
-# models     = {'v1':get_model_v1, 'v4':get_model_v2}
-# batch_size = 64
-# epochs     = 2
-# train_size = 0.1
-# test_size  = 0.1
-# with_datagen = False
-# verbose      = 0
-#
-# ---- All possibilities
-# datasets     = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']
-# models       = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3}
-# batch_size   = 64
-# epochs       = 16
-# train_size   = 1
-# test_size    = 1
-# with_datagen = False
-# verbose      = 0
-#
-# ---- Data augmentation
-datasets     = ['set-48x48-RGB']
-models       = {'v2':get_model_v2}
-batch_size   = 64
-epochs       = 20
-train_size   = 1
-test_size    = 1
-with_datagen = True
-verbose      = 0
-#
-# ---------------------------------------------------------------------------
-
-# ---- Data augmentation
-#
-if with_datagen :
-    datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
-                                                           featurewise_std_normalization=False,
-                                                           width_shift_range=0.1,
-                                                           height_shift_range=0.1,
-                                                           zoom_range=0.2,
-                                                           shear_range=0.1,
-                                                           rotation_range=10.)
-else:
-    datagen=None
-
-# ---- Run
-#
-output = multi_run(dataset_dir,
-                   datasets, models,
-                   datagen=datagen,
-                   train_size=train_size, test_size=test_size,
-                   batch_size=batch_size, epochs=epochs,
-                   verbose=verbose,
-                   extension_dir=tag_id)
-
-# ---- Save report
-#
-report={}
-report['output']=output
-report['description']='train_size={} test_size={} batch_size={} epochs={} data_aug={}'.format(train_size,test_size,batch_size,epochs,with_datagen)
-
-report_name=f'./run/report_{tag_id}.json'
-
-with open(report_name, 'w') as file:
-    json.dump(report, file)
-
-print('\nReport saved as ',report_name)
-end_time = time.time()
-duration = end_time-start_time
-print(f'Duration : {duration:.2f} s')
-print('-'*59)
-```
-
-%% Output
-
-    
-    ---- Run --------------------------------------------------
-    
-    Dataset :  set-24x24-L
-        Run model v1  : Accuracy=39.98 and Duration=2.23
-        Run model v4  : Accuracy=6.18 and Duration=2.17
-    
-    Dataset :  set-24x24-RGB
-        Run model v1  : Accuracy=53.52 and Duration=2.20
-        Run model v4  : Accuracy=11.80 and Duration=2.01
-    
-    Report saved as  ./run/report_083052.json
-    Duration : 10.37 s
-    -----------------------------------------------------------
-
-%% Cell type:markdown id: tags:
-
-## Step 7 - That's all folks..
-
-%% Cell type:code id: tags:
-
-``` python
-print('\n{}'.format(time.strftime("%A %-d %B %Y, %H:%M:%S")))
-print("The work is done.\n")
-```
-
-%% Cell type:markdown id: tags:
-
----
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>

GTSRB/06-Full-convolutions-batch.ipynb

-%% Cell type:markdown id: tags:
-
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
-
-# <!-- TITLE --> [GTS6] - CNN with GTSRB dataset - Full convolutions as a batch
-<!-- DESC --> Episode 6 : Run Full convolution notebook as a batch
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
-  - Run a notebook code as a **job**
-  - Follow up with Tensorboard
-
-The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
-The final aim is to recognise them !
-Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
-
-
-## What we're going to do :
-Our main steps:
- - Run Full-convolution.ipynb as a batch :
-    - Notebook mode
-    - Script mode
- - Tensorboard follow up
-
-%% Cell type:markdown id: tags:
-
-### Step 0 - Just for convenience
-
-%% Cell type:code id: tags:
-
-``` python
-import sys
-sys.path.append('..')
-import fidle.pwk as ooo
-ooo.init()
-```
-
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.4.3
-    Run time             : Friday 28 February 2020, 17:55:56
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
-%% Cell type:markdown id: tags:
-
-## Step 1 - Run a notebook as a batch
-To run a notebook in a command line :
-```jupyter nbconvert (...) --to notebook --execute <notebook>```
-
-For example :
-```jupyter nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --output='./run/full_convolutions' --execute '05-Full-convolutions.ipynb'```
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Export as a script (What we're going to do this time)
-To export a notebook as a script :
-```jupyter nbconvert --to script <notebook>```
-To run the script :
-```ipython <script>```
-
-%% Cell type:code id: tags:
-
-``` python
-%%bash
-
-# ---- This will convert a notebook to a notebook.py script
-#
-jupyter nbconvert --to script --output='./run/full_convolutions_01' '05-Full-convolutions.ipynb'
-```
-
-%% Output
-
-    [NbConvertApp] Converting notebook 05-Full-convolutions.ipynb to script
-    [NbConvertApp] Writing 13061 bytes to ./run/full_convolutions_01.py
-
-%% Cell type:code id: tags:
-
-``` python
-!ls -l ./run/*.py
-```
-
-%% Output
-
-    -rwxr-xr-x 1 paroutyj l-simap 13061 Feb 28 17:56 ./run/full_convolutions_01.py
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Batch submission
-### 2.1 - Create batch script :
-
-%% Cell type:code id: tags:
-
-``` python
-%%writefile "./run/full_convolutions_01.sh"
-#!/bin/bash
-#OAR -n Full convolutions
-#OAR -t gpu
-#OAR -l /nodes=1/gpudevice=1,walltime=01:00:00
-#OAR --stdout full_convolutions_%jobid%.out
-#OAR --stderr full_convolutions_%jobid%.err
-#OAR --project fidle
-
-#---- With cpu
-# use :
-# OAR -l /nodes=1/core=32,walltime=02:00:00
-# and add a 2>/dev/null to ipython xxx
-
-# ----------------------------------
-#   _           _       _
-#  | |__   __ _| |_ ___| |__
-#  | '_ \ / _` | __/ __| '_ \
-#  | |_) | (_| | || (__| | | |
-#  |_.__/ \__,_|\__\___|_| |_|
-#                  Full convolutions
-# ----------------------------------
-#
-
-CONDA_ENV=fidle
-RUN_DIR=~/fidle/GTSRB
-RUN_SCRIPT=./run/full_convolutions_01.py
-
-# ---- Cuda Conda initialization
-#
-echo '------------------------------------------------------------'
-echo "Start : $0"
-echo '------------------------------------------------------------'
-#
-source /applis/environments/cuda_env.sh dahu 10.0
-source /applis/environments/conda.sh
-#
-conda activate "$CONDA_ENV"
-
-# ---- Run it...
-#
-cd $RUN_DIR
-ipython $RUN_SCRIPT
-```
-
-%% Output
-
-    Overwriting ./run/full_convolutions_01.sh
-
-%% Cell type:markdown id: tags:
-
-### 2.2 - Have a look
-
-%% Cell type:code id: tags:
-
-``` python
-%%bash
-chmod 755 ./run/*.sh
-chmod 755 ./run/*.py
-ls -l ./run/*full_convolutions*
-```
-
-%% Output
-
-    -rwxr-xr-x 1 paroutyj l-simap 13061 Feb 28 16:31 ./run/full_convolutions_01.py
-    -rwxr-xr-x 1 paroutyj l-simap  1015 Feb 28 16:31 ./run/full_convolutions_01.sh
-
-%% Cell type:markdown id: tags:
-
-### 2.3 - Job submission
-Have to be done on the frontal :
-```bash
-# hostname
-f-dahu
-
-# pwd
-/home/paroutyj
-
-# oarsub -S ~/fidle/GTSRB/run/full_convolutions_01.sh
-[GPUNODE] Adding gpu node restriction
-[ADMISSION RULE] Modify resource description with type constraints
-
-#oarstat -u
-Job id    S User     Duration   System message
---------- - -------- ---------- ------------------------------------------------
-5878410   R paroutyj    0:19:56 R=8,W=1:0:0,J=I,P=fidle,T=gpu (Karma=0.005,quota_ok)
-5896266   W paroutyj    0:00:00 R=8,W=1:0:0,J=B,N=Full convolutions,P=fidle,T=gpu
-
-# ls -l
-total 8
--rw-r--r-- 1 paroutyj l-simap    0 Feb 28 15:58 full_convolutions_5896266.err
--rw-r--r-- 1 paroutyj l-simap 5703 Feb 28 15:58 full_convolutions_5896266.out
-```
-
-%% Cell type:markdown id: tags:
-
-<div class='todo'>
- Your mission if you accept it: Run our full_convolution code in batch mode.<br>
- For that :
-  <ul>
-      <li>Validate the full_convolution notebook on short tests</li>
-      <li>Submit it in batch mode for validation</li>
-      <li>Modify the notebook for a full run and submit it :-)</li>
-  </ul>
-
-</div>
-
-%% Cell type:markdown id: tags:
-
----
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>

GTSRB/07-Full-convolutions-reports.ipynb

-%% Cell type:markdown id: tags:
-
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
-
-# <!-- TITLE --> [GTS7] - Full convolutions Report
-<!-- DESC -->  Episode 7 : Displaying the reports of the different jobs
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
- - Compare the results of different dataset-model combinations
-
-Les rapports (format json) sont générés par les jobs "Full convolution" [GTS5][GTS6]
-
-
-## What we're going to do :
-
- - Read json files and display results
-
-## 1/ Python import
-
-%% Cell type:code id: tags:
-
-``` python
-import pandas as pd
-import os,glob,json
-from pathlib import Path
-from IPython.display import display, Markdown
-```
-
-%% Cell type:markdown id: tags:
-
-## 2/ A nice function
-
-%% Cell type:code id: tags:
-
-``` python
-def highlight_max(s):
-    is_max = (s == s.max())
-    return ['background-color: yellow' if v else '' for v in is_max]
-
-def show_report(file):
-    # ---- Read json file
-    with open(file) as infile:
-        dict_report = json.load( infile )
-    output      = dict_report['output']
-    description = dict_report['description']
-    # ---- about
-    print("\n\n\nReport : ",Path(file).stem)
-    print(    "Desc.  : ",description,'\n')
-    # ---- Create a pandas
-    report       = pd.DataFrame (output)
-    col_accuracy = [ c for c in output.keys() if c.endswith('Accuracy')]
-    col_duration = [ c for c in output.keys() if c.endswith('Duration')]
-    # ---- Build formats
-    lambda_acc = lambda x : '{:.2f} %'.format(x) if (isinstance(x, float)) else '{:}'.format(x)
-    lambda_dur = lambda x : '{:.1f} s'.format(x) if (isinstance(x, float)) else '{:}'.format(x)
-    formats = {'Size':'{:.2f} Mo'}
-    for c in col_accuracy:
-        formats[c]=lambda_acc
-    for c in col_duration:
-        formats[c]=lambda_dur
-    t=report.style.highlight_max(subset=col_accuracy).format(formats).hide_index()
-    display(t)
-```
-
-%% Cell type:markdown id: tags:
-
-## 3/ Reports display
-
-%% Cell type:code id: tags:
-
-``` python
-for file in glob.glob("./run/*.json"):
-    show_report(file)
-```
-
-%% Output
-
-    
-    
-    
-    Report :  report_028304
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-    
-    
-    
-    Report :  report_029273
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-    
-    
-    
-    Report :  report_033476
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-    
-    
-    
-    Report :  report_035478
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=20 data_aug=True
-    
-
-
-    
-    
-    
-    Report :  report_049176
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=20 data_aug=True
-    
-
-
-    
-    
-    
-    Report :  report_094813
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=20 data_aug=True
-    
-
-
-%% Cell type:markdown id: tags:
-
----
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>

IMDB/01-Embedding-Keras.ipynb

-%% Cell type:markdown id: tags:
-
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
-
-# <!-- TITLE --> [IMDB1] - Text embedding with IMDB
-<!-- DESC --> A very classical example of word embedding for text classification (sentiment analysis)
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
- - The objective is to guess whether film reviews are **positive or negative** based on the analysis of the text.
- - Understand the management of **textual data** and **sentiment analysis**
-
-Original dataset can be find **[there](http://ai.stanford.edu/~amaas/data/sentiment/)**
-Note that [IMDb.com](https://imdb.com) offers several easy-to-use [datasets](https://www.imdb.com/interfaces/)
-For simplicity's sake, we'll use the dataset directly [embedded in Keras](https://www.tensorflow.org/api_docs/python/tf/keras/datasets)
-
-## What we're going to do :
-
- - Retrieve data
- - Preparing the data
- - Build a model
- - Train the model
- - Evaluate the result
-
-%% Cell type:markdown id: tags:
-
-## Step 1 - Init python stuff
-
-%% Cell type:code id: tags:
-
-``` python
-import numpy as np
-
-import tensorflow as tf
-import tensorflow.keras as keras
-import tensorflow.keras.datasets.imdb as imdb
-
-import matplotlib.pyplot as plt
-import matplotlib
-import seaborn as sns
-
-import os,sys,h5py,json
-from importlib import reload
-
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
-```
-
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.4.2
-    Run time             : Sunday 23 February 2020, 20:06:09
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
-%% Cell type:code id: tags:
-
-``` python
-place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv("SCRATCH_DIR","")}/PROJECTS/pr-fidle/datasets/IMDB',
-                                       'IDRIS'  : f'{os.getenv("WORK","")}/datasets/IMDB',
-                                       'HOME'   : f'{os.getenv("HOME","")}/datasets/IMDB'} )
-```
-
-%% Output
-
-    Well, we should be at HOME !
-    We are going to use: /home/pjluc/datasets/IMDB
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Retrieve data
-
-**From Keras :**
-This IMDb dataset can bet get directly from [Keras datasets](https://www.tensorflow.org/api_docs/python/tf/keras/datasets)
-
-Due to their nature, textual data can be somewhat complex.
-
-### 2.1 - Data structure :
-The dataset is composed of 2 parts: **reviews** and **opinions** (positive/negative),  with a **dictionary**
-
-  - dataset = (reviews, opinions)
-    - reviews = \[ review_0, review_1, ...\]
-      - review_i = [ int1, int2, ...] where int_i is the index of the word in the dictionary.
-    - opinions = \[ int0, int1, ...\] where int_j == 0 if opinion is negative or 1 if opinion is positive.
-  - dictionary = \[ mot1:int1, mot2:int2, ... ]
-
-%% Cell type:markdown id: tags:
-
-### 2.2 - Get dataset
-For simplicity, we will use a pre-formatted dataset.
-See : https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb/load_data
-
-However, Keras offers some usefull tools for formatting textual data.
-See : https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text
-
-%% Cell type:code id: tags:
-
-``` python
-vocab_size = 10000
-
-# ----- Retrieve x,y
-#
-(x_train, y_train), (x_test, y_test) = imdb.load_data( num_words  = vocab_size,
-                                                       skip_top   = 0,
-                                                       maxlen     = None,
-                                                       seed       = 42,
-                                                       start_char = 1,
-                                                       oov_char   = 2,
-                                                       index_from = 3, )
-```
-
-%% Cell type:code id: tags:
-
-``` python
-print("  Max(x_train,x_test)  : ", ooo.rmax([x_train,x_test]) )
-print("  x_train : {}  y_train : {}".format(x_train.shape, y_train.shape))
-print("  x_test  : {}  y_test  : {}".format(x_test.shape,  y_test.shape))
-
-print('\nReview example (x_train[12]) :\n\n',x_train[12])
-```
-
-%% Output
-
-      Max(x_train,x_test)  :  9999
-      x_train : (25000,)  y_train : (25000,)
-      x_test  : (25000,)  y_test  : (25000,)
-    
-    Review example (x_train[12]) :
-    
-     [1, 14, 22, 1367, 53, 206, 159, 4, 636, 898, 74, 26, 11, 436, 363, 108, 7, 14, 432, 14, 22, 9, 1055, 34, 8599, 2, 5, 381, 3705, 4509, 14, 768, 47, 839, 25, 111, 1517, 2579, 1991, 438, 2663, 587, 4, 280, 725, 6, 58, 11, 2714, 201, 4, 206, 16, 702, 5, 5176, 19, 480, 5920, 157, 13, 64, 219, 4, 2, 11, 107, 665, 1212, 39, 4, 206, 4, 65, 410, 16, 565, 5, 24, 43, 343, 17, 5602, 8, 169, 101, 85, 206, 108, 8, 3008, 14, 25, 215, 168, 18, 6, 2579, 1991, 438, 2, 11, 129, 1609, 36, 26, 66, 290, 3303, 46, 5, 633, 115, 4363]
-
-%% Cell type:markdown id: tags:
-
-### 2.3 - Have a look for humans (optional)
-When we loaded the dataset, we asked for using \<start\> as 1, \<unknown word\> as 2
-So, we shifted the dataset by 3 with the parameter index_from=3
-
-%% Cell type:code id: tags:
-
-``` python
-# ---- Retrieve dictionary {word:index}, and encode it in ascii
-#
-word_index = imdb.get_word_index()
-
-# ---- Shift the dictionary from +3
-#
-word_index = {w:(i+3) for w,i in word_index.items()}
-
-# ---- Add <pad>, <start> and unknown tags
-#
-word_index.update( {'<pad>':0, '<start>':1, '<unknown>':2} )
-
-# ---- Create a reverse dictionary : {index:word}
-#
-index_word = {index:word for word,index in word_index.items()}
-
-# ---- Add a nice function to transpose :
-#
-def dataset2text(review):
-    return ' '.join([index_word.get(i, '?') for i in review])
-```
-
-%% Cell type:code id: tags:
-
-``` python
-print('\nDictionary size     : ', len(word_index))
-for k in range(440,455):print(f'{k:2d} : {index_word[k]}' )
-print('\nReview example (x_train[12]) :\n\n',x_train[12])
-print('\nIn real words :\n\n', dataset2text(x_train[12]))
-```
-
-%% Output
-
-    
-    Dictionary size     :  88587
-    440 : hope
-    441 : entertaining
-    442 : she's
-    443 : mr
-    444 : overall
-    445 : evil
-    446 : called
-    447 : loved
-    448 : based
-    449 : oh
-    450 : several
-    451 : fans
-    452 : mother
-    453 : drama
-    454 : beginning
-    
-    Review example (x_train[12]) :
-    
-     [1, 14, 22, 1367, 53, 206, 159, 4, 636, 898, 74, 26, 11, 436, 363, 108, 7, 14, 432, 14, 22, 9, 1055, 34, 8599, 2, 5, 381, 3705, 4509, 14, 768, 47, 839, 25, 111, 1517, 2579, 1991, 438, 2663, 587, 4, 280, 725, 6, 58, 11, 2714, 201, 4, 206, 16, 702, 5, 5176, 19, 480, 5920, 157, 13, 64, 219, 4, 2, 11, 107, 665, 1212, 39, 4, 206, 4, 65, 410, 16, 565, 5, 24, 43, 343, 17, 5602, 8, 169, 101, 85, 206, 108, 8, 3008, 14, 25, 215, 168, 18, 6, 2579, 1991, 438, 2, 11, 129, 1609, 36, 26, 66, 290, 3303, 46, 5, 633, 115, 4363]
-    
-    In real words :
-    
-     <start> this film contains more action before the opening credits than are in entire hollywood films of this sort this film is produced by tsui <unknown> and stars jet li this team has brought you many worthy hong kong cinema productions including the once upon a time in china series the action was fast and furious with amazing wire work i only saw the <unknown> in two shots aside from the action the story itself was strong and not just used as filler to find any other action films to rival this you must look for a hong kong cinema <unknown> in your area they are really worth checking out and usually never disappoint
-
-%% Cell type:markdown id: tags:
-
-### 2.4 - Have a look for neurons
-
-%% Cell type:code id: tags:
-
-``` python
-plt.figure(figsize=(12, 6))
-ax=sns.distplot([len(i) for i in x_train],bins=60)
-ax.set_title('Distribution of reviews by size')
-plt.xlabel("Review's sizes")
-plt.ylabel('Density')
-ax.set_xlim(0, 1500)
-plt.show()
-```
-
-%% Output
-
-
-
-%% Cell type:markdown id: tags:
-
-## Step 3 - Preprocess the data
-In order to be processed by an NN, all entries must have the same length.
-We chose a review length of **review_len**
-We will therefore complete them with a padding (of \<pad\>\)
-
-%% Cell type:code id: tags:
-
-``` python
-review_len = 256
-
-x_train = keras.preprocessing.sequence.pad_sequences(x_train,
-                                                     value   = 0,
-                                                     padding = 'post',
-                                                     maxlen  = review_len)
-
-x_test  = keras.preprocessing.sequence.pad_sequences(x_test,
-                                                     value   = 0 ,
-                                                     padding = 'post',
-                                                     maxlen  = review_len)
-
-print('\nReview example (x_train[12]) :\n\n',x_train[12])
-print('\nIn real words :\n\n', dataset2text(x_train[12]))
-```
-
-%% Output
-
-    
-    Review example (x_train[12]) :
-    
-     [   1   14   22 1367   53  206  159    4  636  898   74   26   11  436
-      363  108    7   14  432   14   22    9 1055   34 8599    2    5  381
-     3705 4509   14  768   47  839   25  111 1517 2579 1991  438 2663  587
-        4  280  725    6   58   11 2714  201    4  206   16  702    5 5176
-       19  480 5920  157   13   64  219    4    2   11  107  665 1212   39
-        4  206    4   65  410   16  565    5   24   43  343   17 5602    8
-      169  101   85  206  108    8 3008   14   25  215  168   18    6 2579
-     1991  438    2   11  129 1609   36   26   66  290 3303   46    5  633
-      115 4363    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0]
-    
-    In real words :
-    
-     <start> this film contains more action before the opening credits than are in entire hollywood films of this sort this film is produced by tsui <unknown> and stars jet li this team has brought you many worthy hong kong cinema productions including the once upon a time in china series the action was fast and furious with amazing wire work i only saw the <unknown> in two shots aside from the action the story itself was strong and not just used as filler to find any other action films to rival this you must look for a hong kong cinema <unknown> in your area they are really worth checking out and usually never disappoint <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>
-
-%% Cell type:markdown id: tags:
-
-### Save dataset and dictionary (For future use)
-
-%% Cell type:code id: tags:
-
-``` python
-# ---- To write dataset in the project place
-#
-output_dir = dataset_dir
-
-# ---- To write h5 dataset in a test place
-#      For small tests only !
-#
-# output_dir = './data'
-# ooo.mkdir(output_dir)
-
-
-with h5py.File(f'{output_dir}/dataset_imdb.h5', 'w') as f:
-    f.create_dataset("x_train",    data=x_train)
-    f.create_dataset("y_train",    data=y_train)
-    f.create_dataset("x_test",     data=x_test)
-    f.create_dataset("y_test",     data=y_test)
-
-with open(f'{output_dir}/word_index.json', 'w') as fp:
-    json.dump(word_index, fp)
-
-with open(f'{output_dir}/index_word.json', 'w') as fp:
-    json.dump(index_word, fp)
-
-print('Saved.')
-```
-
-%% Output
-
-    Saved.
-
-%% Cell type:markdown id: tags:
-
-## Step 4 - Build the model
-Few remarks :
-1. We'll choose a dense vector size for the embedding output with **dense_vector_size**
-2. **GlobalAveragePooling1D** do a pooling on the last dimension : (None, lx, ly) -> (None, ly)
-In other words: we average the set of vectors/words of a sentence
-3. L'embedding de Keras fonctionne de manière supervisée. Il s'agit d'une couche de *vocab_size* neurones vers *n_neurons* permettant de maintenir une table de vecteurs (les poids constituent les vecteurs). Cette couche ne calcule pas de sortie a la façon des couches normales, mais renvois la valeur des vecteurs. n mots => n vecteurs (ensuite empilés par le pooling)
-Voir : https://stats.stackexchange.com/questions/324992/how-the-embedding-layer-is-trained-in-keras-embedding-layer
-
-A SUIVRE : https://www.liip.ch/en/blog/sentiment-detection-with-keras-word-embeddings-and-lstm-deep-learning-networks
-### 4.1 - Build
-More documentation about :
- - [Embedding](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Embedding)
- - [GlobalAveragePooling1D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/GlobalAveragePooling1D)
-
-%% Cell type:code id: tags:
-
-``` python
-def get_model(dense_vector_size=32):
-
-    model = keras.Sequential()
-    model.add(keras.layers.Embedding(input_dim    = vocab_size,
-                                     output_dim   = dense_vector_size,
-                                     input_length = review_len))
-    model.add(keras.layers.GlobalAveragePooling1D())
-    model.add(keras.layers.Dense(dense_vector_size, activation='relu'))
-    model.add(keras.layers.Dense(1,                 activation='sigmoid'))
-
-    model.compile(optimizer = 'adam',
-                  loss      = 'binary_crossentropy',
-                  metrics   = ['accuracy'])
-    return model
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 5 - Train the model
-### 5.1 - Get it
-
-%% Cell type:code id: tags:
-
-``` python
-model = get_model(32)
-
-model.summary()
-```
-
-%% Output
-
-    Model: "sequential"
-    _________________________________________________________________
-    Layer (type)                 Output Shape              Param #
-    =================================================================
-    embedding (Embedding)        (None, 256, 32)           320000
-    _________________________________________________________________
-    global_average_pooling1d (Gl (None, 32)                0
-    _________________________________________________________________
-    dense (Dense)                (None, 32)                1056
-    _________________________________________________________________
-    dense_1 (Dense)              (None, 1)                 33
-    =================================================================
-    Total params: 321,089
-    Trainable params: 321,089
-    Non-trainable params: 0
-    _________________________________________________________________
-
-%% Cell type:markdown id: tags:
-
-### 5.2 - Add callback
-
-%% Cell type:code id: tags:
-
-``` python
-os.makedirs('./run/models',   mode=0o750, exist_ok=True)
-save_dir = "./run/models/best_model.h5"
-savemodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, save_best_only=True)
-```
-
-%% Cell type:markdown id: tags:
-
-### 5.1 - Train it
-
-%% Cell type:code id: tags:
-
-``` python
-%%time
-
-n_epochs   = 30
-batch_size = 512
-
-history = model.fit(x_train,
-                    y_train,
-                    epochs          = n_epochs,
-                    batch_size      = batch_size,
-                    validation_data = (x_test, y_test),
-                    verbose         = 1,
-                    callbacks       = [savemodel_callback])
-```
-
-%% Output
-
-    Train on 25000 samples, validate on 25000 samples
-    Epoch 1/30
-    25000/25000 [==============================] - 2s 63us/sample - loss: 0.6872 - accuracy: 0.6386 - val_loss: 0.6756 - val_accuracy: 0.7404
-    Epoch 2/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.6445 - accuracy: 0.7591 - val_loss: 0.6068 - val_accuracy: 0.7795
-    Epoch 3/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.5446 - accuracy: 0.8122 - val_loss: 0.5002 - val_accuracy: 0.8214
-    Epoch 4/30
-    25000/25000 [==============================] - 1s 33us/sample - loss: 0.4322 - accuracy: 0.8539 - val_loss: 0.4097 - val_accuracy: 0.8488
-    Epoch 5/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.3517 - accuracy: 0.8778 - val_loss: 0.3574 - val_accuracy: 0.8615
-    Epoch 6/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.3012 - accuracy: 0.8926 - val_loss: 0.3281 - val_accuracy: 0.8691
-    Epoch 7/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.2668 - accuracy: 0.9040 - val_loss: 0.3108 - val_accuracy: 0.8737
-    Epoch 8/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.2420 - accuracy: 0.9116 - val_loss: 0.2996 - val_accuracy: 0.8772
-    Epoch 9/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.2218 - accuracy: 0.9199 - val_loss: 0.2921 - val_accuracy: 0.8796
-    Epoch 10/30
-    25000/25000 [==============================] - 1s 33us/sample - loss: 0.2048 - accuracy: 0.9266 - val_loss: 0.2890 - val_accuracy: 0.8813
-    Epoch 11/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1908 - accuracy: 0.9331 - val_loss: 0.2873 - val_accuracy: 0.8828
-    Epoch 12/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1790 - accuracy: 0.9368 - val_loss: 0.2868 - val_accuracy: 0.8825
-    Epoch 13/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.1672 - accuracy: 0.9419 - val_loss: 0.2887 - val_accuracy: 0.8831
-    Epoch 14/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1582 - accuracy: 0.9466 - val_loss: 0.2973 - val_accuracy: 0.8790
-    Epoch 15/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.1494 - accuracy: 0.9498 - val_loss: 0.2953 - val_accuracy: 0.8822
-    Epoch 16/30
-    25000/25000 [==============================] - 1s 33us/sample - loss: 0.1408 - accuracy: 0.9532 - val_loss: 0.3004 - val_accuracy: 0.8817
-    Epoch 17/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1335 - accuracy: 0.9554 - val_loss: 0.3063 - val_accuracy: 0.8803
-    Epoch 18/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1267 - accuracy: 0.9589 - val_loss: 0.3131 - val_accuracy: 0.8778
-    Epoch 19/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1210 - accuracy: 0.9609 - val_loss: 0.3205 - val_accuracy: 0.8760
-    Epoch 20/30
-    25000/25000 [==============================] - 1s 33us/sample - loss: 0.1145 - accuracy: 0.9638 - val_loss: 0.3283 - val_accuracy: 0.8743
-    Epoch 21/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1085 - accuracy: 0.9670 - val_loss: 0.3345 - val_accuracy: 0.8751
-    Epoch 22/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1033 - accuracy: 0.9691 - val_loss: 0.3436 - val_accuracy: 0.8738
-    Epoch 23/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.0992 - accuracy: 0.9704 - val_loss: 0.3523 - val_accuracy: 0.8721
-    Epoch 24/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.0940 - accuracy: 0.9733 - val_loss: 0.3613 - val_accuracy: 0.8707
-    Epoch 25/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.0892 - accuracy: 0.9746 - val_loss: 0.3775 - val_accuracy: 0.8659
-    Epoch 26/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.0852 - accuracy: 0.9766 - val_loss: 0.3809 - val_accuracy: 0.8687
-    Epoch 27/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.0814 - accuracy: 0.9776 - val_loss: 0.3921 - val_accuracy: 0.8666
-    Epoch 28/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.0776 - accuracy: 0.9794 - val_loss: 0.4015 - val_accuracy: 0.8657
-    Epoch 29/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.0733 - accuracy: 0.9810 - val_loss: 0.4121 - val_accuracy: 0.8642
-    Epoch 30/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.0704 - accuracy: 0.9823 - val_loss: 0.4246 - val_accuracy: 0.8627
-    CPU times: user 1min 30s, sys: 4.75 s, total: 1min 35s
-    Wall time: 24.7 s
-
-%% Cell type:markdown id: tags:
-
-## Step 6 - Evaluate
-### 6.1 - Training history
-
-%% Cell type:code id: tags:
-
-``` python
-ooo.plot_history(history)
-```
-
-%% Output
-
-
-
-
-
-%% Cell type:markdown id: tags:
-
-### 6.2 - Reload and evaluate best model
-
-%% Cell type:code id: tags:
-
-``` python
-model = keras.models.load_model('./run/models/best_model.h5')
-
-# ---- Evaluate
-reload(ooo)
-score  = model.evaluate(x_test, y_test, verbose=0)
-
-print('x_test / loss      : {:5.4f}'.format(score[0]))
-print('x_test / accuracy  : {:5.4f}'.format(score[1]))
-
-values=[score[1], 1-score[1]]
-ooo.plot_donut(values,["Accuracy","Errors"], title="#### Accuracy donut is :")
-
-# ---- Confusion matrix
-
-y_pred   = model.predict_classes(x_test)
-
-# ooo.display_confusion_matrix(y_test,y_pred,labels=range(2),color='orange',font_size='20pt')
-ooo.display_confusion_matrix(y_test,y_pred,labels=range(2))
-```
-
-%% Output
-
-    x_test / loss      : 0.2868
-    x_test / accuracy  : 0.8825
-
-    #### Accuracy donut is :
-
-
-
-    #### Confusion matrix is :
-
-
-%% Cell type:markdown id: tags:
-
----
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>

IRIS/01-Simple-Perceptron.ipynb

-%% Cell type:markdown id: tags:
-
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
-
-# <!-- TITLE --> [PER57] - Perceptron Model 1957
-<!-- DESC --> A simple perceptron, with the IRIS dataset.
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
- - Implement a historical linear classifier with a historical dataset !
- - The objective is to predict the type of Iris from the size of the leaves.
- - Identifying its limitations
-
-The [IRIS dataset](https://archive.ics.uci.edu/ml/datasets/Iris) is probably one of the oldest datasets, dating back to 1936 .
-
-## What we're going to do :
- - Retrieve the dataset, via scikit learn
- - training and classifying
-
-## Step 1 - Import and init
-
-%% Cell type:code id: tags:
-
-``` python
-import numpy as np
-from sklearn.datasets     import load_iris
-from sklearn.linear_model import Perceptron
-
-import matplotlib.pyplot as plt
-import matplotlib
-
-import os,sys
-
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
-```
-
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.4.0
-    Run time             : Saturday 22 February 2020, 21:46:23
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - IRIS Data Recovery
-
-Retrieve a dataset : http://scikit-learn.org/stable/modules/classes.html#module-sklearn.datasets
-About the datesets : http://scikit-learn.org/stable/datasets/index.html
-
-Data fields (X) :
-- 0 : sepal length in cm
-- 1 : sepal width in cm
-- 2 : petal length in cm
-- 3 : petal width in cm
-
-Class (y) :
-- 0 : class 0=Iris-Setosa, 1=Iris-Versicolour, 2=Iris-Virginica
-
-%% Cell type:code id: tags:
-
-``` python
-X0,y0 = load_iris(return_X_y=True)
-
-X = X0[:, (2,3)]     # We only keep fields 2 and 3
-y = y0.copy()
-
-y[ y0==0 ] = 1       # 1 = Iris setosa
-y[ y0>=1 ] = 0       # 0 = not iris setosa
-
-print('Examples :')
-print('Length   Width   Iris Setosa (0/1)')
-print('  x1       x2          y')
-for i in range(45,55):
-    print(f" {X[i][0]:3.1f}      {X[i][0]:3.1f}          {y[i]}")
-print(f'\nX shape : {X.shape}')
-print(f'y shape : {y.shape}')
-```
-
-%% Output
-
-    Examples :
-    Length   Width   Iris Setosa (0/1)
-      x1       x2          y
-     1.4      1.4          1
-     1.6      1.6          1
-     1.4      1.4          1
-     1.5      1.5          1
-     1.4      1.4          1
-     4.7      4.7          0
-     4.5      4.5          0
-     4.9      4.9          0
-     4.0      4.0          0
-     4.6      4.6          0
-    
-    X shape : (150, 2)
-    y shape : (150,)
-
-%% Cell type:markdown id: tags:
-
-## Step 3 - Get a perceptron
-
-%% Cell type:code id: tags:
-
-``` python
-pct = Perceptron(max_iter=100, random_state=82, tol=0.01)
-pct.fit(X, y)
-```
-
-%% Output
-
-    Perceptron(alpha=0.0001, class_weight=None, early_stopping=False, eta0=1.0,
-               fit_intercept=True, max_iter=100, n_iter_no_change=5, n_jobs=None,
-               penalty=None, random_state=82, shuffle=True, tol=0.01,
-               validation_fraction=0.1, verbose=0, warm_start=False)
-
-%% Cell type:markdown id: tags:
-
-## Step 4 - Prédictions
-
-%% Cell type:code id: tags:
-
-``` python
-X_pred = np.array( [ [2.0, 0.5], [5.0, 1.5] ] )
-
-y_pred = pct.predict(X_pred)
-
-for i,x in enumerate(X_pred):
-    print(f'Petal : length={x[0]}, width={x[0]}  => is setosa : {y_pred[i]}')
-```
-
-%% Output
-
-    Petal : length=2.0, width=2.0  => is setosa : 1
-    Petal : length=5.0, width=5.0  => is setosa : 0
-
-%% Cell type:markdown id: tags:
-
-## Step 5 - Visualisation
-
-%% Cell type:code id: tags:
-
-``` python
-a = -pct.coef_[0][0] / pct.coef_[0][1]
-b = -pct.intercept_ / pct.coef_[0][1]
-box=[X.min(axis=0)[0],X.max(axis=0)[0],X.min(axis=0)[1],X.max(axis=0)[1]]
-mx=(box[1]-box[0])/20
-my=(box[3]-box[2])/20
-box=[box[0]-mx,box[1]+mx,box[2]-my,box[3]+my]
-
-fig, axs = plt.subplots(1, 1)
-fig.set_size_inches(10,6)
-axs.plot(X[y==1, 0], X[y==1, 1], "o", label="Iris-Setosa")
-axs.plot(X[y==0, 0], X[y==0, 1], "o", label="Autres")
-axs.plot([box[0], box[1]], [a*box[0]+b, a*box[1]+b], "k--", linewidth=2)
-axs.set_xlabel("Petal length (cm)", labelpad=15) #, fontsize=14)
-axs.set_ylabel("Petal width (cm)",  labelpad=15) #, fontsize=14)
-axs.legend(loc="lower right", fontsize=14)
-axs.set_xlim(box[0],box[1])
-axs.set_ylim(box[2],box[3])
-
-plt.show()
-```
-
-%% Output
-
-
-
-%% Cell type:markdown id: tags:
-
----
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>

LinearReg/01-Linear-Regression.ipynb

 %% Cell type:markdown id: tags:
 
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
+<img width="800px" src="../fidle/img/header.svg"></img>
 
 # <!-- TITLE --> [LINR1] - Linear regression with direct resolution
-<!-- DESC --> Direct determination of linear regression
+<!-- DESC --> Low-level implementation, using numpy, of a direct resolution for a linear regression
 <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
 
 ## Objectives :
  - Just one, the illustration of a direct resolution :-)
 
 ## What we're going to do :
 
-Equation : $ Y = X.\theta + N$
+Equation : $$Y = X.\theta + N$$
 Where N is a noise vector
 and $\theta = (a,b)$ a vector as y = a.x + b
 
 %% Cell type:markdown id: tags:
 
 ## Step 1 - Import and init
 
 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 import math
 import matplotlib
 import matplotlib.pyplot as plt
 import sys
+import fidle
 
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('LINR1')
 ```
 
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.4.0
-    Run time             : Saturday 22 February 2020, 15:41:32
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
 %% Cell type:markdown id: tags:
 
 ## Step 2 - Retrieve a set of points
 
 %% Cell type:code id: tags:
 
 ``` python
 # ---- Paramètres
 nb    = 100     # Nombre de points
 xmin  = 0       # Distribution / x
 xmax  = 10
 a     = 4       # Distribution / y
 b     = 2       # y= a.x + b (+ bruit)
 noise = 7       # bruit
 
 theta = np.array([[a],[b]])
 
 # ---- Vecteur X  (1,x) x nb
 #      la premiere colonne est a 1 afin que X.theta <=> 1.b + x.a
 
 Xc1 = np.ones((nb,1))
 Xc2 = np.random.uniform(xmin,xmax,(nb,1))
 X = np.c_[ Xc1, Xc2 ]
 
 # ---- Noise
 # N = np.random.uniform(-noise,noise,(nb,1))
 N = noise * np.random.normal(0,1,(nb,1))
 
 # ---- Vecteur Y
 Y = (X @ theta) + N
 
 # print("X:\n",X,"\nY:\n ",Y)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### Show it
 
 %% Cell type:code id: tags:
 
 ``` python
 width = 12
 height = 6
 
 fig, ax = plt.subplots()
 fig.set_size_inches(width,height)
 ax.plot(X[:,1], Y, ".")
 ax.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
 ax.set_xlabel('x axis')
 ax.set_ylabel('y axis')
+fidle.scrawler.save_fig('01-set_of_points')
 plt.show()
 ```
 
-%% Output
-
-
-
 %% Cell type:markdown id: tags:
 
 ## Step 3 - Direct calculation of the normal equation
 
 
 We'll try to find an optimal value of $\theta$, minimizing a cost function.
 The cost function, classically used in the case of linear regressions, is the **root mean square error** (racine carré de l'erreur quadratique moyenne):
 
-$RMSE(X,h_\theta)=\sqrt{\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2}$
+$$RMSE(X,h_\theta)=\sqrt{\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2}$$
 
-With the simplified variant : $MSE(X,h_\theta)=\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2$
+With the simplified variant : $$MSE(X,h_\theta)=\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2$$
 
-The optimal value of regression is : $ \hat{ \theta } =( X^{-T} .X)^{-1}.X^{-T}.Y$
+The optimal value of regression is : $$\hat{ \theta } =( X^{T} .X)^{-1}.X^{T}.Y$$
 
 Démontstration : https://eli.thegreenplace.net/2014/derivation-of-the-normal-equation-for-linear-regression
 
 %% Cell type:code id: tags:
 
 ``` python
 theta_hat = np.linalg.inv(X.T @ X) @ X.T @ Y
 
 print("Theta :\n",theta,"\n\ntheta hat :\n",theta_hat)
 ```
 
-%% Output
-
-    Theta :
-     [[4]
-     [2]]
-    
-    theta hat :
-     [[6.68975287]
-     [1.6787656 ]]
-
 %% Cell type:markdown id: tags:
 
 ### Show it
 
 %% Cell type:code id: tags:
 
 ``` python
 Xd = np.array([[1,xmin], [1,xmax]])
 Yd = Xd @ theta_hat
 
 fig, ax = plt.subplots()
 fig.set_size_inches(width,height)
 ax.plot(X[:,1], Y, ".")
 ax.plot(Xd[:,1], Yd, "-")
 ax.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
 ax.set_xlabel('x axis')
 ax.set_ylabel('y axis')
+fidle.scrawler.save_fig('02-regression-line')
 plt.show()
 ```
 
-%% Output
+%% Cell type:code id: tags:
 
-
+``` python
+fidle.end()
+```
 
 %% Cell type:markdown id: tags:
 
 ---
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/03-Polynomial-Regression.ipynb

 %% Cell type:markdown id: tags:
 
-<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
+<img width="800px" src="../fidle/img/header.svg"></img>
 
 # <!-- TITLE --> [POLR1] - Complexity Syndrome
 <!-- DESC --> Illustration of the problem of complexity with the polynomial regression
 <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
 
 ## Objectives :
  - Visualizing and understanding under and overfitting
 
 ## What we're going to do :
 
 We are looking for a polynomial function to approximate the observed series :
 $ y = a_n\cdot x^n + \dots + a_i\cdot x^i + \dots + a_1\cdot x + b $
 
 
 ## Step 1 - Import and init
 
 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 import math
 import random
 import matplotlib
 import matplotlib.pyplot as plt
 import sys
+import fidle
 
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('POLR1')
 ```
 
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.4.0
-    Run time             : Saturday 22 February 2020, 15:45:15
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
 %% Cell type:markdown id: tags:
 
-## Step 2 - Preparation of learning data :
+## Step 2 - Dataset generation
 
 %% Cell type:code id: tags:
 
 ``` python
 # ---- Parameters
 
 n         = 100
 
 xob_min   = -5
 xob_max   = 5
 
 deg       =  7
 a_min     = -2
 a_max     =  2
 
 noise     =  2000
 
 # ---- Train data
 #      X,Y              : data
 #      X_norm,Y_norm    : normalized data
 
 X = np.random.uniform(xob_min,xob_max,(n,1))
 # N = np.random.uniform(-noise,noise,(n,1))
 N = noise * np.random.normal(0,1,(n,1))
 
 a = np.random.uniform(a_min,a_max, (deg,))
 fy = np.poly1d( a )
 
 Y = fy(X) + N
 
 # ---- Data normalization
 #
 X_norm = (X - X.mean(axis=0)) / X.std(axis=0)
 Y_norm = (Y - Y.mean(axis=0)) / Y.std(axis=0)
 
 # ---- Data visualization
 
 width = 12
 height = 6
 nb_viz = min(2000,n)
 
 def vector_infos(name,V):
     m=V.mean(axis=0).item()
     s=V.std(axis=0).item()
     print("{:8} :      mean={:+12.4f}  std={:+12.4f}    min={:+12.4f}    max={:+12.4f}".format(name,m,s,V.min(),V.max()))
 
 
-print("Nombre de points : {}  a={} deg={} bruit={}".format(n,a,deg,noise))
+fidle.utils.display_md('#### Generator :')
+print(f"Nomber of points={n}  deg={deg} bruit={noise}")
 
-ooo.display_md('#### Before normalization :')
-print("\nDonnées d'aprentissage brute :")
-print("({} points visibles sur {})".format(nb_viz,n))
+fidle.utils.display_md('#### Datasets :')
+print(f"{nb_viz} points visibles sur {n})")
 plt.figure(figsize=(width, height))
 plt.plot(X[:nb_viz], Y[:nb_viz], '.')
 plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
 plt.xlabel('x axis')
 plt.ylabel('y axis')
+fidle.scrawler.save_fig("01-dataset")
 plt.show()
+
+fidle.utils.display_md('#### Before normalization :')
 vector_infos('X',X)
 vector_infos('Y',Y)
 
-ooo.display_md('#### After normalization :')
-print("\nDonnées d'aprentissage normalisées :")
-print("({} points visibles sur {})".format(nb_viz,n))
-plt.figure(figsize=(width, height))
-plt.plot(X_norm[:nb_viz], Y_norm[:nb_viz], '.')
-plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
-plt.xlabel('x axis')
-plt.ylabel('y axis')
-plt.show()
+fidle.utils.display_md('#### After normalization :')
 vector_infos('X_norm',X_norm)
 vector_infos('Y_norm',Y_norm)
 ```
 
-%% Output
-
-    Nombre de points : 100  a=[-0.87167753  1.76610157 -1.5797434   0.77278901 -1.03581256  1.20870761
-      1.12549131] deg=7 bruit=2000
-
-    #### Before normalization :
-
-    
-    Données d'aprentissage brute :
-    (100 points visibles sur 100)
-
-
-
-    X        :      mean=     +0.4444  std=     +2.8560    min=     -4.8506    max=     +4.9948
-    Y        :      mean=  -1918.5970  std=  +4018.7742    min= -15663.0729    max=  +7697.5273
-
-    #### After normalization :
-
-    
-    Données d'aprentissage normalisées :
-    (100 points visibles sur 100)
-
-
-
-    X_norm   :      mean=     -0.0000  std=     +1.0000    min=     -1.8540    max=     +1.5933
-    Y_norm   :      mean=     -0.0000  std=     +1.0000    min=     -3.4201    max=     +2.3928
-
 %% Cell type:markdown id: tags:
 
 ## Step 3 - Polynomial regression with NumPy
 ### 3.1 - Underfitting
 
 %% Cell type:code id: tags:
 
 ``` python
-def draw_reg(X_norm, Y_norm, x_hat,fy_hat, size):
+def draw_reg(X_norm, Y_norm, x_hat,fy_hat, size, save_as):
     plt.figure(figsize=size)
     plt.plot(X_norm, Y_norm, '.')
 
     x_hat = np.linspace(X_norm.min(), X_norm.max(), 100)
 
     plt.plot(x_hat, fy_hat(x_hat))
     plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
     plt.xlabel('x axis')
     plt.ylabel('y axis')
+    fidle.scrawler.save_fig(save_as)
     plt.show()
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 reg_deg=1
 
 a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
 fy_hat  = np.poly1d( a_hat )
 
-print("Nombre de degrés : {} a_hat={}".format(reg_deg, a_hat))
-draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height))
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='02-underfitting')
 ```
 
-%% Output
-
-    Nombre de degrés : 1 a_hat=[-2.90278334e-02 -6.29052465e-17]
-
-
-
 %% Cell type:markdown id: tags:
 
 ### 3.2 - Good fitting
 
 %% Cell type:code id: tags:
 
 ``` python
 reg_deg=5
 
 a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
 fy_hat  = np.poly1d( a_hat )
 
-print("Nombre de degrés : {} a_hat={}".format(reg_deg, a_hat))
-draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height))
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='03-good_fitting')
 ```
 
-%% Output
-
-    Nombre de degrés : 5 a_hat=[-0.00633816 -0.55313883 -0.12623462  0.67829195  0.12629161  0.36691953]
-
-
-
 %% Cell type:markdown id: tags:
 
 ### 3.3 - Overfitting
 
 %% Cell type:code id: tags:
 
 ``` python
 reg_deg=24
 
 a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
 fy_hat  = np.poly1d( a_hat )
 
-print("Nombre de degrés : {} a_hat={}".format(reg_deg, a_hat))
-draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height))
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='04-over_fitting')
 ```
 
-%% Output
-
-    Nombre de degrés : 24 a_hat=[-2.12316577e+00 -6.91597507e+00  2.77684272e+01  1.03893090e+02
-     -1.50202123e+02 -6.79701345e+02  4.21895164e+02  2.53839748e+03
-     -5.95294613e+02 -5.96004033e+03  1.53557264e+02  9.13691399e+03
-      8.49288955e+02 -9.20441697e+03 -1.46513846e+03  5.98771133e+03
-      1.14599636e+03 -2.40885163e+03 -4.79737229e+02  5.50768446e+02
-      1.03516653e+02 -5.93457926e+01 -9.83903899e+00  1.62330659e+00
-      6.80542884e-01]
+%% Cell type:code id: tags:
 
-
+``` python
+fidle.end()
+```
 
 %% Cell type:markdown id: tags:
 
 ---
-<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/modules/RegressionCooker.py

@@ -14,7 +14,7 @@
 import numpy as np
 import math
 import random
-import datetime, time
+import datetime, time, sys
 
 import matplotlib
 import matplotlib.pyplot as plt
@@ -23,14 +23,14 @@ from IPython.display import display,Markdown,HTML
 
 class RegressionCooker():
     
-    __version__ = '0.1'
+    fidle   = None
+    version = '0.1'
     
-    def __init__(self, mplstyle='../fidle/mplstyles/custom.mplstyle'):
-        print('\nFIDLE 2020 - Regression Cooker')
-        print('Version      :', self.__version__)
-        print('Run time     : {}'.format(time.strftime("%A %-d %B %Y, %H:%M:%S")))
-        if mplstyle is not None:
-            matplotlib.style.use(mplstyle)
+    def __init__(self, fidle):
+        self.fidle = fidle
+        fidle.utils.subtitle('FIDLE 2020 - Regression Cooker')
+        print('Version      :', self.version)
+        print('Run time     : {}'.format(time.strftime("%A %d %B %Y, %H:%M:%S")))
         
 
     @classmethod
@@ -99,6 +99,7 @@ class RegressionCooker():
         print(f"X shape : {X.shape}  Y shape : {Y.shape}  plot : {nb_viz} points")
         plt.figure(figsize=(width, height))
         plt.plot(X[:nb_viz], Y[:nb_viz], '.')
+        self.fidle.scrawler.save_fig('01-dataset')
         plt.show()
         self.vector_infos('X',X)
         self.vector_infos('Y',Y)
@@ -111,7 +112,9 @@ class RegressionCooker():
         if i<0:
             print( "    #i   Loss       Gradient         Theta")
         else:
-            print("  {:3d}  {:+7.3f}  {:+7.3f} {:+7.3f}  {:+7.3f} {:+7.3f}".format(i,loss,gradient.item(0),gradient.item(1),theta.item(0),theta.item(1)))
+            print("  {:3d}  {:+7.3f}  {:+7.3f} {:+7.3f}  {:+7.3f} {:+7.3f}".format(i,loss,gradient.item(0),
+                                                                                   gradient.item(1),theta.item(0),
+                                                                                   theta.item(1)))
 
             
     def __plot_XY(self, X,Y,width=12,height=6):
@@ -184,10 +187,13 @@ class RegressionCooker():
 
         # ---- Visualization
 
-        display(Markdown(f'**Visualization :**  '))
+        self.fidle.utils.subtitle('Visualization :')
+        self.fidle.scrawler.save_fig('02-basic_descent')
         plt.show()
-        display(Markdown(f'**Loss :**  '))
+
+        self.fidle.utils.subtitle('Loss :')
         self.__plot_loss(loss)
+        self.fidle.scrawler.save_fig('03-basic_descent_loss')
         plt.show()
         
         return theta
@@ -257,13 +263,17 @@ class RegressionCooker():
 #             draw_theta(epoch,mse,gradients, theta,0.1+epoch/(n_epochs+1))
 
 #         draw_theta(epoch,mse,gradients,theta,1)
-
+        
         # ---- Visualization
 
-        display(Markdown(f'**Visualization :**  '))
+        self.fidle.utils.subtitle('Visualization :')
+        self.fidle.scrawler.save_fig('04-minibatch_descent')
         plt.show()
-        display(Markdown(f'**Loss :**  '))
+
+        self.fidle.utils.subtitle('Loss :')
         self.__plot_loss(loss)
+        self.fidle.scrawler.save_fig('05-minibatch_descent_loss')
         plt.show()
         
-        return theta
\ No newline at end of file
+        
+        return theta

MNIST.Keras3/01-DNN-MNIST.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [K3MNIST1] - Simple classification with DNN
+<!-- DESC --> An example of classification using a dense neural network for the famous MNIST dataset
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with Keras
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id: tags:
+
+``` python
+import os
+os.environ['KERAS_BACKEND'] = 'torch'
+
+import keras
+
+import numpy as np
+import matplotlib.pyplot as plt
+import sys,os
+from importlib import reload
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('K3MNIST1')
+```
+
+%% Cell type:markdown id: tags:
+
+Verbosity during training : 0 = silent, 1 = progress bar, 2 = one line per epoch
+
+%% Cell type:code id: tags:
+
+``` python
+fit_verbosity = 1
+```
+
+%% Cell type:markdown id: tags:
+
+Override parameters (batch mode) - Just forget this cell
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.override('fit_verbosity')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Retrieve data
+MNIST is one of the most famous historic dataset.
+Include in [Keras datasets](https://keras.io/datasets)
+
+%% Cell type:code id: tags:
+
+``` python
+(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
+
+print("x_train : ",x_train.shape)
+print("y_train : ",y_train.shape)
+print("x_test  : ",x_test.shape)
+print("y_test  : ",y_test.shape)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Preparing the data
+
+%% Cell type:code id: tags:
+
+``` python
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+xmax=x_train.max()
+x_train = x_train / xmax
+x_test  = x_test  / xmax
+
+print('After normalization  : Min={}, max={}'.format(x_train.min(),x_train.max()))
+```
+
+%% Cell type:markdown id: tags:
+
+### Have a look
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Create model
+About informations about :
+ - [Optimizer](https://keras.io/api/optimizers)
+ - [Activation](https://keras.io/api/layers/activations)
+ - [Loss](https://keras.io/api/losses)
+ - [Metrics](https://keras.io/api/metrics)
+
+%% Cell type:code id: tags:
+
+``` python
+hidden1     = 100
+hidden2     = 100
+
+model = keras.Sequential([
+    keras.layers.Input((28,28)),
+    keras.layers.Flatten(),
+    keras.layers.Dense( hidden1, activation='relu'),
+    keras.layers.Dense( hidden2, activation='relu'),
+    keras.layers.Dense( 10,      activation='softmax')
+])
+
+model.compile(optimizer='adam',
+              loss='sparse_categorical_crossentropy',
+              metrics=['accuracy'])
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 5 - Train the model
+
+%% Cell type:code id: tags:
+
+``` python
+batch_size  = 512
+epochs      =  16
+
+history = model.fit(  x_train, y_train,
+                      batch_size      = batch_size,
+                      epochs          = epochs,
+                      verbose         = fit_verbosity,
+                      validation_data = (x_test, y_test))
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+
+%% Cell type:code id: tags:
+
+``` python
+score = model.evaluate(x_test, y_test, verbose=0)
+
+print('Test loss     :', score[0])
+print('Test accuracy :', score[1])
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.2 - Plot history
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.history(history, figsize=(6,4), save_as='03-history')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.3 - Plot results
+
+%% Cell type:code id: tags:
+
+``` python
+#y_pred   = model.predict_classes(x_test)           Deprecated after 01/01/2021 !!
+
+y_sigmoid = model.predict(x_test, verbose=fit_verbosity)
+y_pred    = np.argmax(y_sigmoid, axis=-1)
+
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id: tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>