GTSRB.Keras3/modules/ImagenetClassnames.py

+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                         Imagenet Classes
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2024
+
+
+import os
+import json
+
+class ImagenetClassnames:
+
+    classes_file = 'ImagenetClassnames.json'
+
+    def __init__(self):
+        path = os.path.abspath(__file__)
+        dir_path = os.path.dirname(path)
+        with open(f'{dir_path}/{self.classes_file}') as f:
+            self.classes = json.load(f)
+        print(f'Imagenet classes loaded ({len(self.classes)} classes)')
+
+
+    def get(self, classes_id, top_n=2):
+        top_classes = [self.classes[str(i)] for i in classes_id[-top_n:]]
+        return top_classes
\ No newline at end of file

GTSRB.Keras3/modules/my_TensorboardCallback.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                     Tensorboard callback
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+#
+# See : https://keras.io/api/callbacks/
+# See : https://keras.io/guides/writing_your_own_callbacks/
+# See : https://pytorch.org/docs/stable/tensorboard.html
+
+import keras
+from torch.utils.tensorboard import SummaryWriter
+
+
+class TensorboardCallback(keras.callbacks.Callback):
+
+    def __init__(self, log_dir=None):
+        '''
+        Init callback
+        Args:
+            log_dir : log directory
+        '''
+        self.writer = SummaryWriter(log_dir=log_dir)
+
+
+    def on_epoch_end(self, epoch, logs=None):
+        '''
+        Record logs at epoch end
+        '''
+
+        # ---- Records all metrics (very simply)
+        #
+        # for k,v in logs.items():
+        #     self.writer.add_scalar(k,v, epoch)
+
+        # ---- Records and group specific metrics
+        #
+        self.writer.add_scalars('Accuracy',
+                                {'Train':logs['accuracy'],
+                                  'Validation':logs['val_accuracy']},
+                                 epoch )
+        
+        self.writer.add_scalars('Loss',
+                                {'Train':logs['loss'],
+                                  'Validation':logs['val_loss']},
+                                 epoch )
+

GTSRB.Keras3/modules/my_loader.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                           Dataset reader
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+
+import h5py
+import os
+import fidle
+
+
+def read_dataset(enhanced_dir, dataset_name, scale=1):
+    '''
+    Reads h5 dataset
+    Args:
+        filename     : datasets filename
+        dataset_name : dataset name, without .h5
+    Returns:    
+        x_train,y_train, x_test,y_test data, x_meta,y_meta
+    '''
+
+    # ---- Read dataset
+    #
+    chrono=fidle.Chrono()
+    chrono.start()
+    filename = f'{enhanced_dir}/{dataset_name}.h5'
+    with  h5py.File(filename,'r') as f:
+        x_train = f['x_train'][:]
+        y_train = f['y_train'][:]
+        x_test  = f['x_test'][:]
+        y_test  = f['y_test'][:]
+        x_meta  = f['x_meta'][:]
+        y_meta  = f['y_meta'][:]
+
+    # ---- Rescale 
+    #
+    print('Original shape  :', x_train.shape, y_train.shape)
+    x_train,y_train, x_test,y_test = fidle.utils.rescale_dataset(x_train,y_train,x_test,y_test, scale=scale)
+    print('Rescaled shape  :', x_train.shape, y_train.shape)
+
+    # ---- Shuffle
+    #
+    x_train,y_train=fidle.utils.shuffle_np_dataset(x_train,y_train)
+
+    # ---- done
+    #
+    duration = chrono.get_delay()
+    size     = fidle.utils.hsize(os.path.getsize(filename))
+    print(f'\nDataset "{dataset_name}" is loaded and shuffled. ({size} in {duration})')
+    return x_train,y_train, x_test,y_test, x_meta,y_meta
+
+
+
+
+print('Module my_loader loaded.')
\ No newline at end of file

GTSRB.Keras3/modules/my_models.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                         Some nice models
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+
+import keras
+
+# ------------------------------------------------------------------
+# -- A simple model, for 24x24 or 48x48 images                    --
+# ------------------------------------------------------------------
+#
+def get_model_01(lx,ly,lz):
+    
+    model = keras.models.Sequential()
+
+    model.add( keras.layers.Input((lx,ly,lz)) )
+    
+    model.add( keras.layers.Conv2D(96, (3,3), activation='relu' ))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.2))
+
+    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.2))
+
+    model.add( keras.layers.Flatten()) 
+    model.add( keras.layers.Dense(1500, activation='relu'))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Dense(43, activation='softmax'))
+    return model
+    
+
+# ------------------------------------------------------------------
+# -- A more sophisticated model, for 48x48 images                 --
+# ------------------------------------------------------------------
+#
+def get_model_02(lx,ly,lz):
+    model = keras.models.Sequential()
+    
+    model.add( keras.layers.Input((lx,ly,lz)) )
+    
+    model.add( keras.layers.Conv2D(32, (3,3),   activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Flatten()) 
+    model.add( keras.layers.Dense(1152, activation='relu'))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Dense(43, activation='softmax'))
+    return model
+
+
+
+def get_model(name, lx,ly,lz):
+    '''
+    Return a model given by name
+    Args:
+        f_name : function name to retreive model
+        lxly,lz : inpuy shape 
+    Returns:
+        model
+    '''
+    if name=='model_01' : return get_model_01(lx,ly,lz)
+    if name=='model_02' : return get_model_01(lx,ly,lz)
+    print('*** Model not found : ', name)
+    return None
+
+# A More fun version ;-)
+def get_model2(name, lx,ly,lz):
+    get_model=globals()['get_'+name]
+    model=get_model(lx,ly,lz)
+    return model
+
+
+
+print('Module my_models loaded.')
\ No newline at end of file

GTSRB.Keras3/modules/my_tools.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|          A small traffic sign classifier
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+import numpy as np
+import matplotlib.pyplot as plt
+import fidle
+
+
+def show_prediction( prediction, x, y, x_meta ):
+
+    # ---- A prediction is just the output layer
+    #
+    fidle.utils.subtitle("Output layer from model is (x100) :")
+    with np.printoptions(precision=2, suppress=True, linewidth=95):
+        print(prediction*100)
+
+    # ---- Graphic visualisation
+    #
+    fidle.utils.subtitle("Graphically :")
+    plt.figure(figsize=(8,2))
+    plt.bar(range(43), prediction[0], align='center', alpha=0.5)
+    plt.ylabel('Probability')
+    plt.ylim((0,1))
+    plt.xlabel('Class')
+    plt.title('Trafic Sign prediction')
+    fidle.scrawler.save_fig('05-prediction-proba')
+    plt.show()
+
+    # ---- Predict class
+    #
+    p = np.argmax(prediction)
+
+    # ---- Show result
+    #
+    fidle.utils.subtitle('In pictures :')
+    print("\nThe image :               Prediction :            Real stuff:")
+    fidle.scrawler.images([x,x_meta[p], x_meta[y]], [p,p,y], range(3),  columns=3,  x_size=1.5, y_size=1.5, save_as='06-prediction-images')
+
+    if p==y:
+        print("YEEES ! that's right!")
+    else:
+        print("oups, that's wrong ;-(")
\ No newline at end of file

GTSRB/05-Full-convolutions.ipynb

-%% Cell type:markdown id: tags:
-
-![Fidle](../fidle/img/00-Fidle-header-01.png)
-
-# <!-- TITLE --> CNN with GTSRB dataset - Full convolutions
-<!-- DESC --> Episode 5: A lot of models, a lot of datasets and a lot of results.
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
-  - Try multiple solutions
-  - Design a generic and batch-usable code
-
-The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
-The final aim is to recognise them !
-Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
-
-
-## What we're going to do :
-
-Our main steps:
- - Try n models with n datasets
- - Save a Pandas/h5 report
- - Write to be run in batch mode
-
-## Step 1 - Import
-
-%% Cell type:code id: tags:
-
-``` python
-import tensorflow as tf
-from tensorflow import keras
-
-import numpy as np
-import h5py
-import os,time,json
-import random
-
-from IPython.display import display
-
-VERSION='1.6'
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Init and start
-
-%% Cell type:code id: tags:
-
-``` python
-# ---- Where I am ?
-now    = time.strftime("%A %d %B %Y - %Hh%Mm%Ss")
-here   = os.getcwd()
-random.seed(time.time())
-tag_id = '{:06}'.format(random.randint(0,99999))
-
-# ---- Who I am ?
-if 'OAR_JOB_ID' in os.environ:
-    oar_id=os.environ['OAR_JOB_ID']
-else:
-    oar_id='???'
-
-print('\nFull Convolutions Notebook')
-print('  Version            : {}'.format(VERSION))
-print('  Now is             : {}'.format(now))
-print('  OAR id             : {}'.format(oar_id))
-print('  Tag id             : {}'.format(tag_id))
-print('  Working directory  : {}'.format(here))
-print('  TensorFlow version :',tf.__version__)
-print('  Keras version      :',tf.keras.__version__)
-print('  for tensorboard    : --logdir {}/run/logs_{}'.format(here,tag_id))
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 3 - Dataset loading
-
-%% Cell type:code id: tags:
-
-``` python
-def read_dataset(name):
-    '''Reads h5 dataset from ./data
-
-    Arguments:  dataset name, without .h5
-    Returns:    x_train,y_train,x_test,y_test data'''
-    # ---- Read dataset
-    filename='./data/'+name+'.h5'
-    with  h5py.File(filename,'r') as f:
-        x_train = f['x_train'][:]
-        y_train = f['y_train'][:]
-        x_test  = f['x_test'][:]
-        y_test  = f['y_test'][:]
-
-    return x_train,y_train,x_test,y_test
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 4 - Models collection
-
-%% Cell type:code id: tags:
-
-``` python
-
-# A basic model
-#
-def get_model_v1(lx,ly,lz):
-
-    model = keras.models.Sequential()
-
-    model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz)))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Flatten())
-    model.add( keras.layers.Dense(1500, activation='relu'))
-    model.add( keras.layers.Dropout(0.5))
-
-    model.add( keras.layers.Dense(43, activation='softmax'))
-    return model
-
-# A more sophisticated model
-#
-def get_model_v2(lx,ly,lz):
-    model = keras.models.Sequential()
-
-    model.add( keras.layers.Conv2D(64, (3, 3), padding='same', input_shape=(lx,ly,lz), activation='relu'))
-    model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(128, (3, 3), padding='same', activation='relu'))
-    model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(256, (3, 3), padding='same',activation='relu'))
-    model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Flatten())
-    model.add( keras.layers.Dense(512, activation='relu'))
-    model.add( keras.layers.Dropout(0.5))
-    model.add( keras.layers.Dense(43, activation='softmax'))
-    return model
-
-def get_model_v3(lx,ly,lz):
-    model = keras.models.Sequential()
-    model.add(tf.keras.layers.Conv2D(32, (5, 5), padding='same',  activation='relu', input_shape=(lx,ly,lz)))
-    model.add(tf.keras.layers.BatchNormalization(axis=-1))
-    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add(tf.keras.layers.Dropout(0.2))
-
-    model.add(tf.keras.layers.Conv2D(64, (5, 5), padding='same',  activation='relu'))
-    model.add(tf.keras.layers.BatchNormalization(axis=-1))
-    model.add(tf.keras.layers.Conv2D(128, (5, 5), padding='same', activation='relu'))
-    model.add(tf.keras.layers.BatchNormalization(axis=-1))
-    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
-    model.add(tf.keras.layers.Dropout(0.2))
-
-    model.add(tf.keras.layers.Flatten())
-    model.add(tf.keras.layers.Dense(512, activation='relu'))
-    model.add(tf.keras.layers.BatchNormalization())
-    model.add(tf.keras.layers.Dropout(0.4))
-
-    model.add(tf.keras.layers.Dense(43, activation='softmax'))
-    return model
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 5 - Multiple datasets, multiple models ;-)
-
-%% Cell type:code id: tags:
-
-``` python
-def multi_run(datasets, models, datagen=None,
-              train_size=1, test_size=1, batch_size=64, epochs=16,
-              verbose=0, extension_dir='last'):
-
-    # ---- Logs and models dir
-    #
-    os.makedirs('./run/logs_{}'.format(extension_dir),   mode=0o750, exist_ok=True)
-    os.makedirs('./run/models_{}'.format(extension_dir), mode=0o750, exist_ok=True)
-
-    # ---- Columns of output
-    #
-    output={}
-    output['Dataset']=[]
-    output['Size']   =[]
-    for m in models:
-        output[m+'_Accuracy'] = []
-        output[m+'_Duration'] = []
-
-    # ---- Let's go
-    #
-    for d_name in datasets:
-        print("\nDataset : ",d_name)
-
-        # ---- Read dataset
-        x_train,y_train,x_test,y_test = read_dataset(d_name)
-        d_size=os.path.getsize('./data/'+d_name+'.h5')/(1024*1024)
-        output['Dataset'].append(d_name)
-        output['Size'].append(d_size)
-
-        # ---- Get the shape
-        (n,lx,ly,lz) = x_train.shape
-        n_train = int(x_train.shape[0]*train_size)
-        n_test  = int(x_test.shape[0]*test_size)
-
-        # ---- For each model
-        for m_name,m_function in models.items():
-            print("    Run model {}  : ".format(m_name), end='')
-            # ---- get model
-            try:
-                model=m_function(lx,ly,lz)
-                # ---- Compile it
-                model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
-                # ---- Callbacks tensorboard
-                log_dir = "./run/logs_{}/tb_{}_{}".format(extension_dir, d_name, m_name)
-                tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
-                # ---- Callbacks bestmodel
-                save_dir = "./run/models_{}/model_{}_{}.h5".format(extension_dir, d_name, m_name)
-                bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True)
-                # ---- Train
-                start_time = time.time()
-                if datagen==None:
-                    # ---- No data augmentation (datagen=None) --------------------------------------
-                    history = model.fit(x_train[:n_train], y_train[:n_train],
-                                        batch_size      = batch_size,
-                                        epochs          = epochs,
-                                        verbose         = verbose,
-                                        validation_data = (x_test[:n_test], y_test[:n_test]),
-                                        callbacks       = [tensorboard_callback, bestmodel_callback])
-                else:
-                    # ---- Data augmentation (datagen given) ----------------------------------------
-                    datagen.fit(x_train)
-                    history = model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),
-                                        steps_per_epoch = int(n_train/batch_size),
-                                        epochs          = epochs,
-                                        verbose         = verbose,
-                                        validation_data = (x_test[:n_test], y_test[:n_test]),
-                                        callbacks       = [tensorboard_callback, bestmodel_callback])
-
-                # ---- Result
-                end_time = time.time()
-                duration = end_time-start_time
-                accuracy = max(history.history["val_accuracy"])*100
-                #
-                output[m_name+'_Accuracy'].append(accuracy)
-                output[m_name+'_Duration'].append(duration)
-                print("Accuracy={:.2f} and Duration={:.2f})".format(accuracy,duration))
-            except:
-                output[m_name+'_Accuracy'].append('0')
-                output[m_name+'_Duration'].append('999')
-                print('-')
-    return output
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 6 - Run !
-
-%% Cell type:code id: tags:
-
-``` python
-start_time = time.time()
-
-print('\n---- Run','-'*50)
-
-# --------- Datasets, models, and more.. -----------------------------------
-#
-# ---- For tests
-# datasets   = ['set-24x24-L', 'set-24x24-RGB']
-# models     = {'v1':get_model_v1, 'v4':get_model_v2}
-# batch_size = 64
-# epochs     = 2
-# train_size = 0.1
-# test_size  = 0.1
-# with_datagen = False
-# verbose      = 0
-#
-# ---- All possibilities -> Run A
-# datasets     = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']
-# models       = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3}
-# batch_size   = 64
-# epochs       = 16
-# train_size   = 1
-# test_size    = 1
-# with_datagen = False
-# verbose      = 0
-#
-# ---- Data augmentation -> Run B
-datasets     = ['set-48x48-RGB']
-models       = {'v2':get_model_v2}
-batch_size   = 64
-epochs       = 20
-train_size   = 1
-test_size    = 1
-with_datagen = True
-verbose      = 0
-#
-# ---------------------------------------------------------------------------
-
-# ---- Data augmentation
-#
-if with_datagen :
-    datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
-                                                           featurewise_std_normalization=False,
-                                                           width_shift_range=0.1,
-                                                           height_shift_range=0.1,
-                                                           zoom_range=0.2,
-                                                           shear_range=0.1,
-                                                           rotation_range=10.)
-else:
-    datagen=None
-
-# ---- Run
-#
-output = multi_run(datasets, models,
-                   datagen=datagen,
-                   train_size=train_size, test_size=test_size,
-                   batch_size=batch_size, epochs=epochs,
-                   verbose=verbose,
-                   extension_dir=tag_id)
-
-# ---- Save report
-#
-report={}
-report['output']=output
-report['description']='train_size={} test_size={} batch_size={} epochs={} data_aug={}'.format(train_size,test_size,batch_size,epochs,with_datagen)
-
-report_name='./run/report_{}.json'.format(tag_id)
-
-with open(report_name, 'w') as file:
-    json.dump(report, file)
-
-print('\nReport saved as ',report_name)
-end_time = time.time()
-duration = end_time-start_time
-print(f'Duration : {duration:.2f} s')
-print('-'*59)
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 7 - That's all folks..
-
-%% Cell type:code id: tags:
-
-``` python
-print('\n{}'.format(time.strftime("%A %-d %B %Y, %H:%M:%S")))
-print("The work is done.\n")
-```
-
-%% Cell type:markdown id: tags:
-
----
-![](../fidle/img/00-Fidle-logo-01_s.png)

GTSRB/05.2-Full-convolutions-reports.ipynb

-%% Cell type:markdown id: tags:
-
-German Traffic Sign Recognition Benchmark (GTSRB)
-=================================================
----
-Introduction au Deep Learning  (IDLE) - S. Arias, E. Maldonado, JL. Parouty - CNRS/SARI/DEVLOG - 2020
-
-## Episode 5.2 : Full Convolutions Reports
-
-Ou main steps :
- - Show reports
-
-## 1/ Import
-
-%% Cell type:code id: tags:
-
-``` python
-import pandas as pd
-import os,glob,json
-from pathlib import Path
-from IPython.display import display, Markdown
-```
-
-%% Cell type:markdown id: tags:
-
-## 2/ A nice function
-
-%% Cell type:code id: tags:
-
-``` python
-def highlight_max(s):
-    is_max = (s == s.max())
-    return ['background-color: yellow' if v else '' for v in is_max]
-
-def show_report(file):
-    # ---- Read json file
-    with open(file) as infile:
-        dict_report = json.load( infile )
-    output      = dict_report['output']
-    description = dict_report['description']
-    # ---- about
-    print("\n\n\nReport : ",Path(file).stem)
-    print(    "Desc.  : ",description,'\n')
-    # ---- Create a pandas
-    report       = pd.DataFrame (output)
-    col_accuracy = [ c for c in output.keys() if c.endswith('Accuracy')]
-    col_duration = [ c for c in output.keys() if c.endswith('Duration')]
-    # ---- Build formats
-    lambda_acc = lambda x : '{:.2f} %'.format(x) if (isinstance(x, float)) else '{:}'.format(x)
-    lambda_dur = lambda x : '{:.1f} s'.format(x) if (isinstance(x, float)) else '{:}'.format(x)
-    formats = {'Size':'{:.2f} Mo'}
-    for c in col_accuracy:
-        formats[c]=lambda_acc
-    for c in col_duration:
-        formats[c]=lambda_dur
-    t=report.style.highlight_max(subset=col_accuracy).format(formats).hide_index()
-    display(t)
-```
-
-%% Cell type:markdown id: tags:
-
-## 3/ Reports display
-
-%% Cell type:code id: tags:
-
-``` python
-for file in glob.glob("./run/*.json"):
-    show_report(file)
-```
-
-%% Output
-
-    
-    
-    
-    Report :  report_009557
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-    
-    
-    
-    Report :  report_020341
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-    
-    
-    
-    Report :  report_041040
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=20 data_aug=True
-    
-
-
-    
-    
-    
-    Report :  report_088809
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=20 data_aug=True
-    
-
-
-    
-    
-    
-    Report :  report_093384
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-    
-    
-    
-    Report :  report_094801
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=20 data_aug=True
-    
-
-
-    
-    
-    
-    Report :  report_2020_01_20_17h22m23s
-    Desc.  :  train_size=1 test_size=1 batch_size=64 epochs=16 data_aug=False
-    
-
-
-%% Cell type:code id: tags:
-
-``` python
-```
-
-%% Cell type:code id: tags:
-
-``` python
-```

GTSRB/06-Full-convolutions-batch.ipynb

-%% Cell type:markdown id: tags:
-
-![Fidle](../fidle/img/00-Fidle-header-01.png)
-
-# <!-- TITLE --> CNN with GTSRB dataset - Full convolutions as a batch
-<!-- DESC --> Episode 6 : Run Full convolution notebook as a batch
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
-  - Run a notebook code as a **job**
-  - Follow up with Tensorboard
-
-The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
-The final aim is to recognise them !
-Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
-
-
-## What we're going to do :
-Our main steps:
- - Run Full-convolution.ipynb as a batch :
-    - Notebook mode
-    - Script mode
- - Tensorboard follow up
-
-## Step 1 - Run a notebook as a batch
-To run a notebook in a command line :
-```jupyter nbconvert (...) --to notebook --execute <notebook>```
-
-%% Cell type:raw id: tags:
-
-%%bash
-
-# ---- This will execute and save a notebook
-#
-jupyter nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --output='./run/full_convolutions' --execute '05-Full-convolutions.ipynb'
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Export as a script (What we're going to do !)
-To export a notebook as a script :
-```jupyter nbconvert --to script <notebook>```
-To run the script :
-```ipython <script>```
-
-%% Cell type:code id: tags:
-
-``` python
-%%bash
-
-# ---- This will convert a notebook to a notebook.py script
-#
-jupyter nbconvert --to script --output='./run/full_convolutions_B' '05-Full-convolutions.ipynb'
-```
-
-%% Output
-
-    [NbConvertApp] Converting notebook 05-Full-convolutions.ipynb to script
-    [NbConvertApp] Writing 11305 bytes to ./run/full_convolutions_B.py
-
-%% Cell type:code id: tags:
-
-``` python
-!ls -l ./run/*.py
-```
-
-%% Output
-
-    -rw-r--r-- 1 pjluc pjluc 11305 Jan 21 00:13 ./run/full_convolutions_B.py
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Batch submission
-Create batch script :
-
-%% Cell type:code id: tags:
-
-``` python
-%%writefile "./run/batch_full_convolutions_B.sh"
-#!/bin/bash
-#OAR -n Full convolutions
-#OAR -t gpu
-#OAR -l /nodes=1/gpudevice=1,walltime=01:00:00
-#OAR --stdout full_convolutions_%jobid%.out
-#OAR --stderr full_convolutions_%jobid%.err
-#OAR --project fidle
-
-#---- With cpu
-# use :
-# OAR -l /nodes=1/core=32,walltime=01:00:00
-# and add a 2>/dev/null to ipython xxx
-
-# ----------------------------------
-#   _           _       _
-#  | |__   __ _| |_ ___| |__
-#  | '_ \ / _` | __/ __| '_ \
-#  | |_) | (_| | || (__| | | |
-#  |_.__/ \__,_|\__\___|_| |_|
-#                  Full convolutions
-# ----------------------------------
-#
-
-CONDA_ENV=deeplearning2
-RUN_DIR=~/fidle/GTSRB
-RUN_SCRIPT=./run/full_convolutions_B.py
-
-# ---- Cuda Conda initialization
-#
-echo '------------------------------------------------------------'
-echo "Start : $0"
-echo '------------------------------------------------------------'
-#
-source /applis/environments/cuda_env.sh dahu 10.0
-source /applis/environments/conda.sh
-#
-conda activate "$CONDA_ENV"
-
-# ---- Run it...
-#
-cd $RUN_DIR
-ipython $RUN_SCRIPT
-```
-
-%% Output
-
-    Writing ./run/batch_full_convolutions_B.sh
-
-%% Cell type:code id: tags:
-
-``` python
-%%bash
-chmod 755 ./run/*.sh
-chmod 755 ./run/*.py
-ls -l ./run/*full_convolutions*
-```
-
-%% Output
-
-    -rwxr-xr-x 1 pjluc pjluc  1045 Jan 21 00:15 ./run/batch_full_convolutions_B.sh
-    -rwxr-xr-x 1 pjluc pjluc   611 Jan 19 15:53 ./run/batch_full_convolutions.sh
-    -rwxr-xr-x 1 pjluc pjluc 11305 Jan 21 00:13 ./run/full_convolutions_B.py
-
-%% Cell type:raw id: tags:
-
-%%bash
-./run/batch_full_convolutions.sh
-oarsub (...)
-
-%% Cell type:markdown id: tags:
-
----
-![](../fidle/img/00-Fidle-logo-01_s.png)

IMDB/01-Embedding-Keras.ipynb

-%% Cell type:markdown id: tags:
-
-![Fidle](../fidle/img/00-Fidle-header-01.png)
-
-# <!-- TITLE --> Text embedding with IMDB
-<!-- DESC --> A very classical example of word embedding for text classification (sentiment analysis)
-<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
-
-## Objectives :
- - The objective is to guess whether film reviews are **positive or negative** based on the analysis of the text.
- - Understand the management of **textual data** and **sentiment analysis**
-
-Original dataset can be find **[there](http://ai.stanford.edu/~amaas/data/sentiment/)**
-Note that [IMDb.com](https://imdb.com) offers several easy-to-use [datasets](https://www.imdb.com/interfaces/)
-For simplicity's sake, we'll use the dataset directly [embedded in Keras](https://www.tensorflow.org/api_docs/python/tf/keras/datasets)
-
-## What we're going to do :
-
- - Retrieve data
- - Preparing the data
- - Build a model
- - Train the model
- - Evaluate the result
-
-%% Cell type:markdown id: tags:
-
-## Step 1 - Init python stuff
-
-%% Cell type:code id: tags:
-
-``` python
-import numpy as np
-
-import tensorflow as tf
-import tensorflow.keras as keras
-import tensorflow.keras.datasets.imdb as imdb
-
-import matplotlib.pyplot as plt
-import matplotlib
-import seaborn as sns
-
-import os,sys,h5py,json
-
-from importlib import reload
-
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
-```
-
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.2.9
-    Run time             : Wednesday 19 February 2020, 22:04:33
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
-%% Cell type:markdown id: tags:
-
-## Step 2 - Retrieve data
-
-**From Keras :**
-This IMDb dataset can bet get directly from [Keras datasets](https://www.tensorflow.org/api_docs/python/tf/keras/datasets)
-
-Due to their nature, textual data can be somewhat complex.
-
-### 2.1 - Data structure :
-The dataset is composed of 2 parts: **reviews** and **opinions** (positive/negative),  with a **dictionary**
-
-  - dataset = (reviews, opinions)
-    - reviews = \[ review_0, review_1, ...\]
-      - review_i = [ int1, int2, ...] where int_i is the index of the word in the dictionary.
-    - opinions = \[ int0, int1, ...\] where int_j == 0 if opinion is negative or 1 if opinion is positive.
-  - dictionary = \[ mot1:int1, mot2:int2, ... ]
-
-%% Cell type:markdown id: tags:
-
-### 2.2 - Get dataset
-For simplicity, we will use a pre-formatted dataset.
-See : https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb/load_data
-
-However, Keras offers some usefull tools for formatting textual data.
-See : https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text
-
-%% Cell type:code id: tags:
-
-``` python
-vocab_size = 10000
-
-# ----- Retrieve x,y
-#
-(x_train, y_train), (x_test, y_test) = imdb.load_data( num_words  = vocab_size,
-                                                       skip_top   = 0,
-                                                       maxlen     = None,
-                                                       seed       = 42,
-                                                       start_char = 1,
-                                                       oov_char   = 2,
-                                                       index_from = 3, )
-```
-
-%% Cell type:code id: tags:
-
-``` python
-print("  Max(x_train,x_test)  : ", ooo.rmax([x_train,x_test]) )
-print("  x_train : {}  y_train : {}".format(x_train.shape, y_train.shape))
-print("  x_test  : {}  y_test  : {}".format(x_test.shape,  y_test.shape))
-
-print('\nReview example (x_train[12]) :\n\n',x_train[12])
-```
-
-%% Output
-
-      Max(x_train,x_test)  :  9999
-      x_train : (25000,)  y_train : (25000,)
-      x_test  : (25000,)  y_test  : (25000,)
-    
-    Review example (x_train[12]) :
-    
-     [1, 14, 22, 1367, 53, 206, 159, 4, 636, 898, 74, 26, 11, 436, 363, 108, 7, 14, 432, 14, 22, 9, 1055, 34, 8599, 2, 5, 381, 3705, 4509, 14, 768, 47, 839, 25, 111, 1517, 2579, 1991, 438, 2663, 587, 4, 280, 725, 6, 58, 11, 2714, 201, 4, 206, 16, 702, 5, 5176, 19, 480, 5920, 157, 13, 64, 219, 4, 2, 11, 107, 665, 1212, 39, 4, 206, 4, 65, 410, 16, 565, 5, 24, 43, 343, 17, 5602, 8, 169, 101, 85, 206, 108, 8, 3008, 14, 25, 215, 168, 18, 6, 2579, 1991, 438, 2, 11, 129, 1609, 36, 26, 66, 290, 3303, 46, 5, 633, 115, 4363]
-
-%% Cell type:markdown id: tags:
-
-### 2.3 - Have a look for humans (optional)
-When we loaded the dataset, we asked for using \<start\> as 1, \<unknown word\> as 2
-So, we shifted the dataset by 3 with the parameter index_from=3
-
-%% Cell type:code id: tags:
-
-``` python
-# ---- Retrieve dictionary {word:index}, and encode it in ascii
-
-word_index = imdb.get_word_index()
-
-# ---- Shift the dictionary from +3
-
-word_index = {w:(i+3) for w,i in word_index.items()}
-
-# ---- Add <pad>, <start> and unknown tags
-
-word_index.update( {'<pad>':0, '<start>':1, '<unknown>':2} )
-
-# ---- Create a reverse dictionary : {index:word}
-
-index_word = {index:word for word,index in word_index.items()}
-
-# ---- Add a nice function to transpose :
-#
-def dataset2text(review):
-    return ' '.join([index_word.get(i, '?') for i in review])
-```
-
-%% Cell type:code id: tags:
-
-``` python
-print('\nDictionary size     : ', len(word_index))
-print('\nReview example (x_train[12]) :\n\n',x_train[12])
-print('\nIn real words :\n\n', dataset2text(x_train[12]))
-```
-
-%% Output
-
-    
-    Dictionary size     :  88587
-    
-    Review example (x_train[12]) :
-    
-     [1, 14, 22, 1367, 53, 206, 159, 4, 636, 898, 74, 26, 11, 436, 363, 108, 7, 14, 432, 14, 22, 9, 1055, 34, 8599, 2, 5, 381, 3705, 4509, 14, 768, 47, 839, 25, 111, 1517, 2579, 1991, 438, 2663, 587, 4, 280, 725, 6, 58, 11, 2714, 201, 4, 206, 16, 702, 5, 5176, 19, 480, 5920, 157, 13, 64, 219, 4, 2, 11, 107, 665, 1212, 39, 4, 206, 4, 65, 410, 16, 565, 5, 24, 43, 343, 17, 5602, 8, 169, 101, 85, 206, 108, 8, 3008, 14, 25, 215, 168, 18, 6, 2579, 1991, 438, 2, 11, 129, 1609, 36, 26, 66, 290, 3303, 46, 5, 633, 115, 4363]
-    
-    In real words :
-    
-     <start> this film contains more action before the opening credits than are in entire hollywood films of this sort this film is produced by tsui <unknown> and stars jet li this team has brought you many worthy hong kong cinema productions including the once upon a time in china series the action was fast and furious with amazing wire work i only saw the <unknown> in two shots aside from the action the story itself was strong and not just used as filler to find any other action films to rival this you must look for a hong kong cinema <unknown> in your area they are really worth checking out and usually never disappoint
-
-%% Cell type:markdown id: tags:
-
-### 2.4 - Have a look for neurons
-
-%% Cell type:code id: tags:
-
-``` python
-plt.figure(figsize=(12, 6))
-ax=sns.distplot([len(i) for i in x_train],bins=60)
-ax.set_title('Distribution of reviews by size')
-plt.xlabel("Review's sizes")
-plt.ylabel('Density')
-ax.set_xlim(0, 1500)
-plt.show()
-```
-
-%% Output
-
-
-
-%% Cell type:markdown id: tags:
-
-## Step 3 - Preprocess the data
-In order to be processed by an NN, all entries must have the same length.
-We chose a review length of **review_len**
-We will therefore complete them with a padding (of \<pad\>\)
-
-%% Cell type:code id: tags:
-
-``` python
-review_len = 256
-
-x_train = keras.preprocessing.sequence.pad_sequences(x_train,
-                                                     value   = 0,
-                                                     padding = 'post',
-                                                     maxlen  = review_len)
-
-x_test  = keras.preprocessing.sequence.pad_sequences(x_test,
-                                                     value   = 0 ,
-                                                     padding = 'post',
-                                                     maxlen  = review_len)
-
-print('\nReview example (x_train[12]) :\n\n',x_train[12])
-print('\nIn real words :\n\n', dataset2text(x_train[12]))
-```
-
-%% Output
-
-    
-    Review example (x_train[12]) :
-    
-     [   1   14   22 1367   53  206  159    4  636  898   74   26   11  436
-      363  108    7   14  432   14   22    9 1055   34 8599    2    5  381
-     3705 4509   14  768   47  839   25  111 1517 2579 1991  438 2663  587
-        4  280  725    6   58   11 2714  201    4  206   16  702    5 5176
-       19  480 5920  157   13   64  219    4    2   11  107  665 1212   39
-        4  206    4   65  410   16  565    5   24   43  343   17 5602    8
-      169  101   85  206  108    8 3008   14   25  215  168   18    6 2579
-     1991  438    2   11  129 1609   36   26   66  290 3303   46    5  633
-      115 4363    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0    0    0    0    0    0    0    0    0    0    0
-        0    0    0    0]
-    
-    In real words :
-    
-     <start> this film contains more action before the opening credits than are in entire hollywood films of this sort this film is produced by tsui <unknown> and stars jet li this team has brought you many worthy hong kong cinema productions including the once upon a time in china series the action was fast and furious with amazing wire work i only saw the <unknown> in two shots aside from the action the story itself was strong and not just used as filler to find any other action films to rival this you must look for a hong kong cinema <unknown> in your area they are really worth checking out and usually never disappoint <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>
-
-%% Cell type:markdown id: tags:
-
-### Save dataset and dictionary (can be usefull)
-
-%% Cell type:code id: tags:
-
-``` python
-os.makedirs('./data',   mode=0o750, exist_ok=True)
-
-with h5py.File('./data/dataset_imdb.h5', 'w') as f:
-    f.create_dataset("x_train",    data=x_train)
-    f.create_dataset("y_train",    data=y_train)
-    f.create_dataset("x_test",     data=x_test)
-    f.create_dataset("y_test",     data=y_test)
-
-with open('./data/word_index.json', 'w') as fp:
-    json.dump(word_index, fp)
-
-with open('./data/index_word.json', 'w') as fp:
-    json.dump(index_word, fp)
-
-print('Saved.')
-```
-
-%% Output
-
-    Saved.
-
-%% Cell type:markdown id: tags:
-
-## Step 4 - Build the model
-Few remarks :
-1. We'll choose a dense vector size for the embedding output with **dense_vector_size**
-2. **GlobalAveragePooling1D** do a pooling on the last dimension : (None, lx, ly) -> (None, ly)
-In other words: we average the set of vectors/words of a sentence
-3. L'embedding de Keras fonctionne de manière supervisée. Il s'agit d'une couche de *vocab_size* neurones vers *n_neurons* permettant de maintenir une table de vecteurs (les poids constituent les vecteurs). Cette couche ne calcule pas de sortie a la façon des couches normales, mais renvois la valeur des vecteurs. n mots => n vecteurs (ensuite empilés par le pooling)
-Voir : https://stats.stackexchange.com/questions/324992/how-the-embedding-layer-is-trained-in-keras-embedding-layer
-
-A SUIVRE : https://www.liip.ch/en/blog/sentiment-detection-with-keras-word-embeddings-and-lstm-deep-learning-networks
-### 4.1 - Build
-More documentation about :
- - [Embedding](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Embedding)
- - [GlobalAveragePooling1D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/GlobalAveragePooling1D)
-
-%% Cell type:code id: tags:
-
-``` python
-def get_model(dense_vector_size=32):
-
-    model = keras.Sequential()
-    model.add(keras.layers.Embedding(input_dim    = vocab_size,
-                                     output_dim   = dense_vector_size,
-                                     input_length = review_len))
-    model.add(keras.layers.GlobalAveragePooling1D())
-    model.add(keras.layers.Dense(dense_vector_size, activation='relu'))
-    model.add(keras.layers.Dense(1,                 activation='sigmoid'))
-
-    model.compile(optimizer = 'adam',
-                  loss      = 'binary_crossentropy',
-                  metrics   = ['accuracy'])
-    return model
-```
-
-%% Cell type:markdown id: tags:
-
-## Step 5 - Train the model
-### 5.1 - Get it
-
-%% Cell type:code id: tags:
-
-``` python
-model = get_model(32)
-
-model.summary()
-```
-
-%% Output
-
-    Model: "sequential"
-    _________________________________________________________________
-    Layer (type)                 Output Shape              Param #
-    =================================================================
-    embedding (Embedding)        (None, 256, 32)           320000
-    _________________________________________________________________
-    global_average_pooling1d (Gl (None, 32)                0
-    _________________________________________________________________
-    dense (Dense)                (None, 32)                1056
-    _________________________________________________________________
-    dense_1 (Dense)              (None, 1)                 33
-    =================================================================
-    Total params: 321,089
-    Trainable params: 321,089
-    Non-trainable params: 0
-    _________________________________________________________________
-
-%% Cell type:markdown id: tags:
-
-### 5.2 - Add callback
-
-%% Cell type:code id: tags:
-
-``` python
-os.makedirs('./run/models',   mode=0o750, exist_ok=True)
-save_dir = "./run/models/best_model.h5"
-savemodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, save_best_only=True)
-```
-
-%% Cell type:markdown id: tags:
-
-### 5.1 - Train it
-
-%% Cell type:code id: tags:
-
-``` python
-%%time
-
-n_epochs   = 30
-batch_size = 512
-
-history = model.fit(x_train,
-                    y_train,
-                    epochs          = n_epochs,
-                    batch_size      = batch_size,
-                    validation_data = (x_test, y_test),
-                    verbose         = 1,
-                    callbacks       = [savemodel_callback])
-```
-
-%% Output
-
-    Train on 25000 samples, validate on 25000 samples
-    Epoch 1/30
-    25000/25000 [==============================] - 2s 60us/sample - loss: 0.6883 - accuracy: 0.6220 - val_loss: 0.6783 - val_accuracy: 0.7303
-    Epoch 2/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.6511 - accuracy: 0.7672 - val_loss: 0.6162 - val_accuracy: 0.7666
-    Epoch 3/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.5571 - accuracy: 0.8088 - val_loss: 0.5094 - val_accuracy: 0.8194
-    Epoch 4/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.4412 - accuracy: 0.8528 - val_loss: 0.4150 - val_accuracy: 0.8494
-    Epoch 5/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.3553 - accuracy: 0.8767 - val_loss: 0.3595 - val_accuracy: 0.8604
-    Epoch 6/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.3036 - accuracy: 0.8907 - val_loss: 0.3316 - val_accuracy: 0.8660
-    Epoch 7/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.2684 - accuracy: 0.9020 - val_loss: 0.3108 - val_accuracy: 0.8733
-    Epoch 8/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.2427 - accuracy: 0.9120 - val_loss: 0.2999 - val_accuracy: 0.8774
-    Epoch 9/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.2222 - accuracy: 0.9196 - val_loss: 0.2923 - val_accuracy: 0.8798
-    Epoch 10/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.2055 - accuracy: 0.9262 - val_loss: 0.2885 - val_accuracy: 0.8817
-    Epoch 11/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.1915 - accuracy: 0.9321 - val_loss: 0.2871 - val_accuracy: 0.8819
-    Epoch 12/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.1795 - accuracy: 0.9364 - val_loss: 0.2869 - val_accuracy: 0.8825
-    Epoch 13/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.1680 - accuracy: 0.9418 - val_loss: 0.2893 - val_accuracy: 0.8824
-    Epoch 14/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.1581 - accuracy: 0.9454 - val_loss: 0.2915 - val_accuracy: 0.8830
-    Epoch 15/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.1490 - accuracy: 0.9498 - val_loss: 0.2970 - val_accuracy: 0.8810
-    Epoch 16/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.1411 - accuracy: 0.9530 - val_loss: 0.3006 - val_accuracy: 0.8815
-    Epoch 17/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.1341 - accuracy: 0.9556 - val_loss: 0.3075 - val_accuracy: 0.8798
-    Epoch 18/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.1273 - accuracy: 0.9588 - val_loss: 0.3131 - val_accuracy: 0.8793
-    Epoch 19/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.1206 - accuracy: 0.9608 - val_loss: 0.3199 - val_accuracy: 0.8774
-    Epoch 20/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.1151 - accuracy: 0.9630 - val_loss: 0.3319 - val_accuracy: 0.8722
-    Epoch 21/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.1097 - accuracy: 0.9658 - val_loss: 0.3357 - val_accuracy: 0.8744
-    Epoch 22/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.1043 - accuracy: 0.9688 - val_loss: 0.3439 - val_accuracy: 0.8734
-    Epoch 23/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.0986 - accuracy: 0.9708 - val_loss: 0.3530 - val_accuracy: 0.8728
-    Epoch 24/30
-    25000/25000 [==============================] - 1s 31us/sample - loss: 0.0941 - accuracy: 0.9735 - val_loss: 0.3614 - val_accuracy: 0.8696
-    Epoch 25/30
-    25000/25000 [==============================] - 1s 32us/sample - loss: 0.0897 - accuracy: 0.9749 - val_loss: 0.3718 - val_accuracy: 0.8703
-    Epoch 26/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.0854 - accuracy: 0.9768 - val_loss: 0.3822 - val_accuracy: 0.8676
-    Epoch 27/30
-    25000/25000 [==============================] - 1s 29us/sample - loss: 0.0811 - accuracy: 0.9785 - val_loss: 0.3919 - val_accuracy: 0.8668
-    Epoch 28/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.0779 - accuracy: 0.9789 - val_loss: 0.4036 - val_accuracy: 0.8651
-    Epoch 29/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.0747 - accuracy: 0.9803 - val_loss: 0.4138 - val_accuracy: 0.8640
-    Epoch 30/30
-    25000/25000 [==============================] - 1s 30us/sample - loss: 0.0714 - accuracy: 0.9819 - val_loss: 0.4262 - val_accuracy: 0.8629
-    CPU times: user 1min 35s, sys: 4.59 s, total: 1min 40s
-    Wall time: 23.6 s
-
-%% Cell type:markdown id: tags:
-
-## Step 6 - Evaluate
-### 6.1 - Training history
-
-%% Cell type:code id: tags:
-
-``` python
-ooo.plot_history(history)
-```
-
-%% Output
-
-
-
-
-
-%% Cell type:markdown id: tags:
-
-### 6.2 - Reload and evaluate best model
-
-%% Cell type:code id: tags:
-
-``` python
-model = keras.models.load_model('./run/models/best_model.h5')
-
-# ---- Evaluate
-reload(ooo)
-score  = model.evaluate(x_test, y_test, verbose=0)
-
-print('x_test / loss      : {:5.4f}'.format(score[0]))
-print('x_test / accuracy  : {:5.4f}'.format(score[1]))
-
-values=[score[1], 1-score[1]]
-ooo.plot_donut(values,["Accuracy","Errors"], title="#### Accuracy donut is :")
-
-# ---- Confusion matrix
-
-y_pred   = model.predict_classes(x_test)
-
-# ooo.display_confusion_matrix(y_test,y_pred,labels=range(2),color='orange',font_size='20pt')
-ooo.display_confusion_matrix(y_test,y_pred,labels=range(2))
-```
-
-%% Output
-
-    x_test / loss      : 0.2869
-    x_test / accuracy  : 0.8825
-
-    #### Accuracy donut is :
-
-
-
-    #### Confusion matrix is :
-
-
-%% Cell type:markdown id: tags:
-
----
-![](../fidle/img/00-Fidle-logo-01_s.png)

LinearReg/01-Linear-Regression.ipynb

 %% Cell type:markdown id: tags:
 
-![header1](../fidle/img/00-Fidle-header-01.png)
+<img width="800px" src="../fidle/img/header.svg"></img>
 
-# <!-- TITLE --> Linear regression with direct resolution
-<!-- DESC --> Direct determination of linear regression
+# <!-- TITLE --> [LINR1] - Linear regression with direct resolution
+<!-- DESC --> Low-level implementation, using numpy, of a direct resolution for a linear regression
 <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
 
 ## Objectives :
  - Just one, the illustration of a direct resolution :-)
 
 ## What we're going to do :
 
-Equation : $ Y = X.\theta + N$
+Equation : $$Y = X.\theta + N$$
 Where N is a noise vector
 and $\theta = (a,b)$ a vector as y = a.x + b
 
 %% Cell type:markdown id: tags:
 
 ## Step 1 - Import and init
 
 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 import math
 import matplotlib
 import matplotlib.pyplot as plt
 import sys
+import fidle
 
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('LINR1')
 ```
 
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.2.9
-    Run time             : Tuesday 18 February 2020, 16:02:39
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
 %% Cell type:markdown id: tags:
 
 ## Step 2 - Retrieve a set of points
 
 %% Cell type:code id: tags:
 
 ``` python
 # ---- Paramètres
 nb    = 100     # Nombre de points
 xmin  = 0       # Distribution / x
 xmax  = 10
 a     = 4       # Distribution / y
 b     = 2       # y= a.x + b (+ bruit)
 noise = 7       # bruit
 
 theta = np.array([[a],[b]])
 
 # ---- Vecteur X  (1,x) x nb
 #      la premiere colonne est a 1 afin que X.theta <=> 1.b + x.a
 
 Xc1 = np.ones((nb,1))
 Xc2 = np.random.uniform(xmin,xmax,(nb,1))
 X = np.c_[ Xc1, Xc2 ]
 
 # ---- Noise
 # N = np.random.uniform(-noise,noise,(nb,1))
 N = noise * np.random.normal(0,1,(nb,1))
 
 # ---- Vecteur Y
 Y = (X @ theta) + N
 
 # print("X:\n",X,"\nY:\n ",Y)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### Show it
 
 %% Cell type:code id: tags:
 
 ``` python
 width = 12
 height = 6
 
 fig, ax = plt.subplots()
 fig.set_size_inches(width,height)
 ax.plot(X[:,1], Y, ".")
 ax.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
 ax.set_xlabel('x axis')
 ax.set_ylabel('y axis')
+fidle.scrawler.save_fig('01-set_of_points')
 plt.show()
 ```
 
-%% Output
-
-
-
 %% Cell type:markdown id: tags:
 
 ## Step 3 - Direct calculation of the normal equation
 
 
 We'll try to find an optimal value of $\theta$, minimizing a cost function.
 The cost function, classically used in the case of linear regressions, is the **root mean square error** (racine carré de l'erreur quadratique moyenne):
 
-$RMSE(X,h_\theta)=\sqrt{\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2}$
+$$RMSE(X,h_\theta)=\sqrt{\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2}$$
 
-With the simplified variant : $MSE(X,h_\theta)=\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2$
+With the simplified variant : $$MSE(X,h_\theta)=\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2$$
 
-The optimal value of regression is : $ \hat{ \theta } =( X^{-T} .X)^{-1}.X^{-T}.Y$
+The optimal value of regression is : $$\hat{ \theta } =( X^{T} .X)^{-1}.X^{T}.Y$$
 
 Démontstration : https://eli.thegreenplace.net/2014/derivation-of-the-normal-equation-for-linear-regression
 
 %% Cell type:code id: tags:
 
 ``` python
 theta_hat = np.linalg.inv(X.T @ X) @ X.T @ Y
 
 print("Theta :\n",theta,"\n\ntheta hat :\n",theta_hat)
 ```
 
-%% Output
-
-    Theta :
-     [[4]
-     [2]]
-    
-    theta hat :
-     [[6.81242007]
-     [1.56836316]]
-
 %% Cell type:markdown id: tags:
 
 ### Show it
 
 %% Cell type:code id: tags:
 
 ``` python
 Xd = np.array([[1,xmin], [1,xmax]])
 Yd = Xd @ theta_hat
 
 fig, ax = plt.subplots()
 fig.set_size_inches(width,height)
 ax.plot(X[:,1], Y, ".")
 ax.plot(Xd[:,1], Yd, "-")
 ax.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
 ax.set_xlabel('x axis')
 ax.set_ylabel('y axis')
+fidle.scrawler.save_fig('02-regression-line')
 plt.show()
 ```
 
-%% Output
+%% Cell type:code id: tags:
 
-
+``` python
+fidle.end()
+```
 
 %% Cell type:markdown id: tags:
 
 ---
-![](../fidle/img/00-Fidle-logo-01_s.png)
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/03-Polynomial-Regression.ipynb

 %% Cell type:markdown id: tags:
 
-![header1](../fidle/img/00-Fidle-header-01.png)
+<img width="800px" src="../fidle/img/header.svg"></img>
 
-# <!-- TITLE --> Complexity Syndrome
+# <!-- TITLE --> [POLR1] - Complexity Syndrome
 <!-- DESC --> Illustration of the problem of complexity with the polynomial regression
 <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
 
 ## Objectives :
  - Visualizing and understanding under and overfitting
 
 ## What we're going to do :
 
 We are looking for a polynomial function to approximate the observed series :
 $ y = a_n\cdot x^n + \dots + a_i\cdot x^i + \dots + a_1\cdot x + b $
 
 
 ## Step 1 - Import and init
 
 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 import math
 import random
 import matplotlib
 import matplotlib.pyplot as plt
 import sys
+import fidle
 
-sys.path.append('..')
-import fidle.pwk as ooo
-
-ooo.init()
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('POLR1')
 ```
 
-%% Output
-
-
-    
-    FIDLE 2020 - Practical Work Module
-    Version              : 0.2.9
-    Run time             : Tuesday 18 February 2020, 17:23:05
-    TensorFlow version   : 2.0.0
-    Keras version        : 2.2.4-tf
-
 %% Cell type:markdown id: tags:
 
-## Step 2 - Preparation of learning data :
+## Step 2 - Dataset generation
 
 %% Cell type:code id: tags:
 
 ``` python
 # ---- Parameters
 
 n         = 100
 
 xob_min   = -5
 xob_max   = 5
 
 deg       =  7
 a_min     = -2
 a_max     =  2
 
 noise     =  2000
 
 # ---- Train data
 #      X,Y              : data
 #      X_norm,Y_norm    : normalized data
 
 X = np.random.uniform(xob_min,xob_max,(n,1))
 # N = np.random.uniform(-noise,noise,(n,1))
 N = noise * np.random.normal(0,1,(n,1))
 
 a = np.random.uniform(a_min,a_max, (deg,))
 fy = np.poly1d( a )
 
 Y = fy(X) + N
 
 # ---- Data normalization
 #
 X_norm = (X - X.mean(axis=0)) / X.std(axis=0)
 Y_norm = (Y - Y.mean(axis=0)) / Y.std(axis=0)
 
 # ---- Data visualization
 
 width = 12
 height = 6
 nb_viz = min(2000,n)
 
 def vector_infos(name,V):
     m=V.mean(axis=0).item()
     s=V.std(axis=0).item()
     print("{:8} :      mean={:+12.4f}  std={:+12.4f}    min={:+12.4f}    max={:+12.4f}".format(name,m,s,V.min(),V.max()))
 
 
-print("Nombre de points : {}  a={} deg={} bruit={}".format(n,a,deg,noise))
+fidle.utils.display_md('#### Generator :')
+print(f"Nomber of points={n}  deg={deg} bruit={noise}")
 
-ooo.display_md('#### Before normalization :')
-print("\nDonnées d'aprentissage brute :")
-print("({} points visibles sur {})".format(nb_viz,n))
+fidle.utils.display_md('#### Datasets :')
+print(f"{nb_viz} points visibles sur {n})")
 plt.figure(figsize=(width, height))
 plt.plot(X[:nb_viz], Y[:nb_viz], '.')
 plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
 plt.xlabel('x axis')
 plt.ylabel('y axis')
+fidle.scrawler.save_fig("01-dataset")
 plt.show()
+
+fidle.utils.display_md('#### Before normalization :')
 vector_infos('X',X)
 vector_infos('Y',Y)
 
-ooo.display_md('#### After normalization :')
-print("\nDonnées d'aprentissage normalisées :")
-print("({} points visibles sur {})".format(nb_viz,n))
-plt.figure(figsize=(width, height))
-plt.plot(X_norm[:nb_viz], Y_norm[:nb_viz], '.')
-plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
-plt.xlabel('x axis')
-plt.ylabel('y axis')
-plt.show()
+fidle.utils.display_md('#### After normalization :')
 vector_infos('X_norm',X_norm)
 vector_infos('Y_norm',Y_norm)
 ```
 
-%% Output
-
-    Nombre de points : 100  a=[-1.40023862  1.64009905  1.89987647  1.24972783  1.17765272  1.90935391
-      1.11259327] deg=7 bruit=2000
-
-    #### Before normalization :
-
-    
-    Données d'aprentissage brute :
-    (100 points visibles sur 100)
-
-
-
-    X        :      mean=     +0.2539  std=     +2.9283    min=     -4.9332    max=     +4.9177
-    Y        :      mean=  -2914.8537  std=  +5532.3607    min= -23848.6013    max=  +5139.0627
-
-    #### After normalization :
-
-    
-    Données d'aprentissage normalisées :
-    (100 points visibles sur 100)
-
-
-
-    X_norm   :      mean=     +0.0000  std=     +1.0000    min=     -1.7714    max=     +1.5927
-    Y_norm   :      mean=     -0.0000  std=     +1.0000    min=     -3.7839    max=     +1.4558
-
 %% Cell type:markdown id: tags:
 
 ## Step 3 - Polynomial regression with NumPy
 ### 3.1 - Underfitting
 
 %% Cell type:code id: tags:
 
 ``` python
-def draw_reg(X_norm, Y_norm, x_hat,fy_hat, size):
+def draw_reg(X_norm, Y_norm, x_hat,fy_hat, size, save_as):
     plt.figure(figsize=size)
     plt.plot(X_norm, Y_norm, '.')
 
     x_hat = np.linspace(X_norm.min(), X_norm.max(), 100)
 
     plt.plot(x_hat, fy_hat(x_hat))
     plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
     plt.xlabel('x axis')
     plt.ylabel('y axis')
+    fidle.scrawler.save_fig(save_as)
     plt.show()
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 reg_deg=1
 
 a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
 fy_hat  = np.poly1d( a_hat )
 
-print("Nombre de degrés : {} a_hat={}".format(reg_deg, a_hat))
-draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], x_hat,fy_hat, (width,height))
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='02-underfitting')
 ```
 
-%% Output
-
-    Nombre de degrés : 1 a_hat=[ 2.15635737e-01 -1.26046371e-16]
-
-
-
 %% Cell type:markdown id: tags:
 
 ### 3.2 - Good fitting
 
 %% Cell type:code id: tags:
 
 ``` python
 reg_deg=5
 
 a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
 fy_hat  = np.poly1d( a_hat )
 
-print("Nombre de degrés : {} a_hat={}".format(reg_deg, a_hat))
-draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], x_hat,fy_hat, (width,height))
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='03-good_fitting')
 ```
 
-%% Output
-
-    Nombre de degrés : 5 a_hat=[ 0.09676506 -0.49102546 -0.19674074  0.41539174 -0.00888844  0.51187173]
-
-
-
 %% Cell type:markdown id: tags:
 
 ### 3.3 - Overfitting
 
 %% Cell type:code id: tags:
 
 ``` python
 reg_deg=24
 
 a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
 fy_hat  = np.poly1d( a_hat )
 
-print("Nombre de degrés : {} a_hat={}".format(reg_deg, a_hat))
-draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], x_hat,fy_hat, (width,height))
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='04-over_fitting')
 ```
 
-%% Output
-
-    Nombre de degrés : 24 a_hat=[-3.39583761e-01  3.66524802e+00  1.35968152e+01 -5.33709389e+01
-     -1.54708597e+02  3.43661072e+02  8.76139324e+02 -1.29075459e+03
-     -2.93637308e+03  3.13537269e+03  6.25956959e+03 -5.14495063e+03
-     -8.72124478e+03  5.75688179e+03  7.93008239e+03 -4.30734159e+03
-     -4.57884398e+03  2.04404969e+03  1.58025869e+03 -5.55090657e+02
-     -2.89374022e+02  7.05282858e+01  2.12619645e+01 -2.67799255e+00
-      3.95718335e-01]
+%% Cell type:code id: tags:
 
-
+``` python
+fidle.end()
+```
 
 %% Cell type:markdown id: tags:
 
 ---
-![](../fidle/img/00-Fidle-logo-01_s.png)
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/modules/RegressionCooker.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|  Regression cooker
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning  (FIDLE)
+# CNRS/SARI/DEVLOG 2020 - S. Arias, E. Maldonado, JL. Parouty
+# ------------------------------------------------------------------
+# Initial version by JL Parouty, feb 2020
+
+import numpy as np
+import math
+import random
+import datetime, time, sys
+
+import matplotlib
+import matplotlib.pyplot as plt
+from IPython.display import display,Markdown,HTML
+
+
+class RegressionCooker():
+    
+    fidle   = None
+    version = '0.1'
+    
+    def __init__(self, fidle):
+        self.fidle = fidle
+        fidle.utils.subtitle('FIDLE 2020 - Regression Cooker')
+        print('Version      :', self.version)
+        print('Run time     : {}'.format(time.strftime("%A %d %B %Y, %H:%M:%S")))
+        
+
+    @classmethod
+    def about(cls):
+        print('\nFIDLE 2020 - Regression Cooker)')
+        print('Version       :', cls.version)
+    
+    
+    @classmethod
+    def vector_infos(cls,name,V):
+        """
+        Show some nice infos about a vector
+        args:
+            name : vector name
+            V    : vector
+        """
+        m=V.mean(axis=0).item()
+        s=V.std(axis=0).item()
+        print("{:16} :      mean={:8.3f}  std={:8.3f}    min={:8.3f}    max={:8.3f}".format(name,m,s,V.min(),V.max()))
+    
+    
+    def get_dataset(self,n):
+        """
+        Return a dataset of n observation
+        args:
+            n       : dataset size
+        return:
+            X,Y : with X shapes = (n,1) Y shape = (n,)
+        """
+        
+        xob_min   = 0       # x min and max
+        xob_max   = 10
+
+        a_min     = -30     # a min and max
+        a_max     =  30
+        b_min     = -10     # b min and max
+        b_max     =  10
+
+        noise_min =  10     # noise min and max
+        noise_max =  50
+
+        a0    = random.randint(a_min,a_max)       
+        b0    = random.randint(b_min,b_max)       
+        noise = random.randint(noise_min,noise_max)
+
+        # ---- Construction du jeu d'apprentissage ---------------
+        #      X,Y              : données brutes
+
+        X = np.random.uniform(xob_min,xob_max,(n,1))
+        N = noise * np.random.normal(0,1,(n,1))
+        Y = a0*X + b0 + N
+
+        return X,Y
+    
+
+    
+    def plot_dataset(self,X,Y,title='Dataset :',width=12,height=6):
+        """
+        Plot dataset X,Y
+        args:
+            X : Observations
+            Y : Values
+        """
+        nb_viz = min(1000,len(X))
+        display(Markdown(f'### {title}'))
+        print(f"X shape : {X.shape}  Y shape : {Y.shape}  plot : {nb_viz} points")
+        plt.figure(figsize=(width, height))
+        plt.plot(X[:nb_viz], Y[:nb_viz], '.')
+        self.fidle.scrawler.save_fig('01-dataset')
+        plt.show()
+        self.vector_infos('X',X)
+        self.vector_infos('Y',Y)
+
+        
+    def __plot_theta(self, i, theta,x_min,x_max, loss,gradient,alpha):
+        Xd = np.array([[x_min], [x_max]])
+        Yd = Xd * theta.item(1) + theta.item(0)
+        plt.plot(Xd, Yd, color=(1.,0.4,0.3,alpha))
+        if i<0:
+            print( "    #i   Loss       Gradient         Theta")
+        else:
+            print("  {:3d}  {:+7.3f}  {:+7.3f} {:+7.3f}  {:+7.3f} {:+7.3f}".format(i,loss,gradient.item(0),
+                                                                                   gradient.item(1),theta.item(0),
+                                                                                   theta.item(1)))
+
+            
+    def __plot_XY(self, X,Y,width=12,height=6):
+        nb_viz = min(1000,len(X))
+        plt.figure(figsize=(width, height))
+        plt.plot(X[:nb_viz], Y[:nb_viz], '.') 
+        plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+        plt.xlabel('x axis')
+        plt.ylabel('y axis')
+        
+    def __plot_loss(self,loss, width=8,height=4):
+        plt.figure(figsize=(width, height))
+        plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+        plt.ylim(0, 20)
+        plt.plot(range(len(loss)), loss)
+        plt.xlabel('Iterations')
+        plt.ylabel('Loss')
+
+        
+    def basic_descent(self, X, Y, epochs=200, eta=0.01,width=12,height=6):
+        """
+        Performs a gradient descent where the gradient is updated at the end
+        of each iteration for all observations.
+        args:
+            X,Y          : Observations
+            epochs       : Number of epochs (200)
+            eta          : learning rate
+            width,height : graphic size
+        return:
+            theta        : theta
+        """
+
+        display(Markdown(f'### Basic gradient descent :'))
+
+        display(Markdown(f'**With :**  '))
+        print('with :')
+        print(f'    epochs = {epochs}')
+        print(f'    eta    = {eta}')
+
+        display(Markdown(f'**epochs :**  '))
+        x_min = X.min()
+        x_max = X.max()
+        y_min = Y.min()
+        y_max = Y.max()
+        n     = len(X)
+        
+        # ---- Initialization
+
+        theta = np.array([[y_min],[0]])
+        X_b = np.c_[np.ones((n, 1)), X]
+
+        # ---- Visualization
+        
+        self.__plot_XY(X,Y,width,height)
+        self.__plot_theta( -1, theta,x_min,x_max, None,None,0.1)
+        
+        # ---- Training
+        
+        loss=[]
+        for i in range(epochs+1):
+
+            gradient = (2/n) * X_b.T @ ( X_b @ theta - Y)
+            mse = ((X_b @ theta - Y)**2).mean(axis=None)
+
+            theta = theta - eta * gradient
+            
+            loss.append(mse)
+            if (i % (epochs/10))==0:
+                self.__plot_theta( i, theta,x_min,x_max, mse,gradient,i/epochs)
+
+        # ---- Visualization
+
+        self.fidle.utils.subtitle('Visualization :')
+        self.fidle.scrawler.save_fig('02-basic_descent')
+        plt.show()
+
+        self.fidle.utils.subtitle('Loss :')
+        self.__plot_loss(loss)
+        self.fidle.scrawler.save_fig('03-basic_descent_loss')
+        plt.show()
+        
+        return theta
+    
+    
+    def minibatch_descent(self, X, Y, epochs=200, batchs=5, batch_size=10, eta=0.01,width=12,height=6):
+        """
+        Performs a gradient descent where the gradient is updated at the end
+        of each iteration for all observations.
+        args:
+            X,Y          : Observations
+            epochs       : Number of epochs (200)
+            eta          : learning rate
+            width,height : graphic size
+        return:
+            theta        : theta
+        """
+
+        display(Markdown(f'### Mini batch gradient descent :'))
+
+        display(Markdown(f'**With :**  '))
+        print('with :')
+        print(f'    epochs     = {epochs}')
+        print(f'    batchs     = {batchs}')
+        print(f'    batch size = {batch_size}')
+        print(f'    eta        = {eta}')
+
+        display(Markdown(f'**epochs :**  '))
+        x_min = X.min()
+        x_max = X.max()
+        y_min = Y.min()
+        y_max = Y.max()
+        n     = len(X)
+        
+        # ---- Initialization
+
+        theta = np.array([[y_min],[0]])
+        X_b = np.c_[np.ones((n, 1)), X]
+
+        # ---- Visualization
+        
+        self.__plot_XY(X,Y,width,height)
+        self.__plot_theta( -1, theta,x_min,x_max, None,None,0.1)
+
+        # ---- Training
+
+        def learning_schedule(t):
+            return 1 / (t + 100)
+
+        loss=[]
+        for epoch in range(epochs):
+            for i in range(batchs):
+
+                random_index = np.random.randint(n-batch_size)
+
+                xi = X_b[random_index:random_index+batch_size]
+                yi = Y[random_index:random_index+batch_size]
+
+                mse = ((xi @ theta - yi)**2).mean(axis=None)
+                gradient = 2 * xi.T.dot(xi.dot(theta) - yi)
+
+                eta = learning_schedule(epoch*150)
+                theta = theta - eta * gradient
+
+            loss.append(mse)
+            self.__plot_theta( epoch, theta,x_min,x_max, mse,gradient,epoch/epochs)
+#             draw_theta(epoch,mse,gradients, theta,0.1+epoch/(n_epochs+1))
+
+#         draw_theta(epoch,mse,gradients,theta,1)
+        
+        # ---- Visualization
+
+        self.fidle.utils.subtitle('Visualization :')
+        self.fidle.scrawler.save_fig('04-minibatch_descent')
+        plt.show()
+
+        self.fidle.utils.subtitle('Loss :')
+        self.__plot_loss(loss)
+        self.fidle.scrawler.save_fig('05-minibatch_descent_loss')
+        plt.show()
+        
+        
+        return theta

MNIST.Keras3/01-DNN-MNIST.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [K3MNIST1] - Simple classification with DNN
+<!-- DESC --> An example of classification using a dense neural network for the famous MNIST dataset
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with Keras
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id: tags:
+
+``` python
+import os
+os.environ['KERAS_BACKEND'] = 'torch'
+
+import keras
+
+import numpy as np
+import matplotlib.pyplot as plt
+import sys,os
+from importlib import reload
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('K3MNIST1')
+```
+
+%% Cell type:markdown id: tags:
+
+Verbosity during training : 0 = silent, 1 = progress bar, 2 = one line per epoch
+
+%% Cell type:code id: tags:
+
+``` python
+fit_verbosity = 1
+```
+
+%% Cell type:markdown id: tags:
+
+Override parameters (batch mode) - Just forget this cell
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.override('fit_verbosity')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Retrieve data
+MNIST is one of the most famous historic dataset.
+Include in [Keras datasets](https://keras.io/datasets)
+
+%% Cell type:code id: tags:
+
+``` python
+(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
+
+print("x_train : ",x_train.shape)
+print("y_train : ",y_train.shape)
+print("x_test  : ",x_test.shape)
+print("y_test  : ",y_test.shape)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Preparing the data
+
+%% Cell type:code id: tags:
+
+``` python
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+xmax=x_train.max()
+x_train = x_train / xmax
+x_test  = x_test  / xmax
+
+print('After normalization  : Min={}, max={}'.format(x_train.min(),x_train.max()))
+```
+
+%% Cell type:markdown id: tags:
+
+### Have a look
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Create model
+About informations about :
+ - [Optimizer](https://keras.io/api/optimizers)
+ - [Activation](https://keras.io/api/layers/activations)
+ - [Loss](https://keras.io/api/losses)
+ - [Metrics](https://keras.io/api/metrics)
+
+%% Cell type:code id: tags:
+
+``` python
+hidden1     = 100
+hidden2     = 100
+
+model = keras.Sequential([
+    keras.layers.Input((28,28)),
+    keras.layers.Flatten(),
+    keras.layers.Dense( hidden1, activation='relu'),
+    keras.layers.Dense( hidden2, activation='relu'),
+    keras.layers.Dense( 10,      activation='softmax')
+])
+
+model.compile(optimizer='adam',
+              loss='sparse_categorical_crossentropy',
+              metrics=['accuracy'])
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 5 - Train the model
+
+%% Cell type:code id: tags:
+
+``` python
+batch_size  = 512
+epochs      =  16
+
+history = model.fit(  x_train, y_train,
+                      batch_size      = batch_size,
+                      epochs          = epochs,
+                      verbose         = fit_verbosity,
+                      validation_data = (x_test, y_test))
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+
+%% Cell type:code id: tags:
+
+``` python
+score = model.evaluate(x_test, y_test, verbose=0)
+
+print('Test loss     :', score[0])
+print('Test accuracy :', score[1])
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.2 - Plot history
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.history(history, figsize=(6,4), save_as='03-history')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.3 - Plot results
+
+%% Cell type:code id: tags:
+
+``` python
+#y_pred   = model.predict_classes(x_test)           Deprecated after 01/01/2021 !!
+
+y_sigmoid = model.predict(x_test, verbose=fit_verbosity)
+y_pred    = np.argmax(y_sigmoid, axis=-1)
+
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id: tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

GTSRB/04-Data-augmentation.ipynb → MNIST.Keras3/02-CNN-MNIST.ipynb

 %% Cell type:markdown id: tags:
 
-![Fidle](../fidle/img/00-Fidle-header-01.png)
+<img width="800px" src="../fidle/img/header.svg"></img>
 
-# <!-- TITLE --> CNN with GTSRB dataset - Data augmentation
-<!-- DESC --> Episode 4: Improving the results with data augmentation
+# <!-- TITLE --> [K3MNIST2] - Simple classification with CNN
+<!-- DESC --> An example of classification using a convolutional neural network for the famous MNIST dataset
 <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
 
 ## Objectives :
-  - Trying to improve training by **enhancing the data**
-  - Using Keras' **data augmentation utilities**, finding their limits...
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with Keras
 
-The German Traffic Sign Recognition Benchmark (GTSRB) is a dataset with more than 50,000 photos of road signs from about 40 classes.
-The final aim is to recognise them !
-Description is available there : http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
 
 
 ## What we're going to do :
- - Increase and improve the training dataset
- - Identify the limits of these tools
 
-## Step 1 - Import and init
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Init python stuff
 
 %% Cell type:code id: tags:
 
 ``` python
-import tensorflow as tf
-from tensorflow import keras
-from tensorflow.keras.callbacks import TensorBoard
+import os
+os.environ['KERAS_BACKEND'] = 'torch'
 
-import numpy as np
-import h5py
-
-from sklearn.metrics import confusion_matrix
+import keras
 
+import numpy as np
 import matplotlib.pyplot as plt
-import seaborn as sn
-import os, sys, time, random
-
+import sys,os
 from importlib import reload
 
-sys.path.append('..')
-import fidle.pwk as ooo
+# Init Fidle environment
+import fidle
 
-ooo.init()
+run_id, run_dir, datasets_dir = fidle.init('K3MNIST2')
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 2 - Dataset loader
-Dataset is one of the saved dataset: RGB25, RGB35, L25, L35, etc.
-First of all, we're going to use a smart dataset : **set-24x24-L**
-(with a GPU, it only takes 35'' compared to more than 5' with a CPU !)
+Verbosity during training : 0 = silent, 1 = progress bar, 2 = one line per epoch
 
 %% Cell type:code id: tags:
 
 ``` python
-%%time
-
-def read_dataset(name):
-    '''Reads h5 dataset from ./data
-
-    Arguments:  dataset name, without .h5
-    Returns:    x_train,y_train,x_test,y_test data'''
-    # ---- Read dataset
-    filename='./data/'+name+'.h5'
-    with  h5py.File(filename) as f:
-        x_train = f['x_train'][:]
-        y_train = f['y_train'][:]
-        x_test  = f['x_test'][:]
-        y_test  = f['y_test'][:]
-
-    # ---- done
-    print('Dataset "{}" is loaded. ({:.1f} Mo)\n'.format(name,os.path.getsize(filename)/(1024*1024)))
-    return x_train,y_train,x_test,y_test
+fit_verbosity = 1
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 3 - Models
-We will now build a model and train it...
-
-This is my model ;-)
+Override parameters (batch mode) - Just forget this cell
 
 %% Cell type:code id: tags:
 
 ``` python
-# A basic model
-#
-def get_model_v1(lx,ly,lz):
-
-    model = keras.models.Sequential()
-
-    model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz)))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
-    model.add( keras.layers.MaxPooling2D((2, 2)))
-    model.add( keras.layers.Dropout(0.2))
-
-    model.add( keras.layers.Flatten())
-    model.add( keras.layers.Dense(1500, activation='relu'))
-    model.add( keras.layers.Dropout(0.5))
-
-    model.add( keras.layers.Dense(43, activation='softmax'))
-    return model
+fidle.override('fit_verbosity')
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 4 - Callbacks
-We prepare 2 kind callbacks :  TensorBoard and Model backup
+## Step 2 - Retrieve data
+MNIST is one of the most famous historic dataset.
+Include in [Keras datasets](https://keras.io/datasets)
 
 %% Cell type:code id: tags:
 
 ``` python
-%%bash
-# To clean old logs and saved model, run this cell
-#
-/bin/rm -r ./run/logs   2>/dev/null
-/bin/rm -r ./run/models 2>/dev/null
-/bin/mkdir -p -m 755 ./run/logs
-/bin/mkdir -p -m 755 ./run/models
-echo -e "Reset directories : ./run/logs and ./run/models ."
-```
+(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
 
-%% Cell type:code id: tags:
-
-``` python
-# ---- Callback tensorboard
-log_dir = "./run/logs/tb_" + ooo.tag_now()
-tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
+x_train = x_train.reshape(-1,28,28,1)
+x_test  = x_test.reshape(-1,28,28,1)
 
-# ---- Callback ModelCheckpoint - Save best model
-save_dir = "./run/models/best-model.h5"
-bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True)
-
-# ---- Callback ModelCheckpoint - Save model each epochs
-save_dir = "./run/models/model-{epoch:04d}.h5"
-savemodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, save_freq=2000*5)
+print("x_train : ",x_train.shape)
+print("y_train : ",y_train.shape)
+print("x_test  : ",x_test.shape)
+print("y_test  : ",y_test.shape)
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 5 - Load and prepare dataset
-### 5.1 - Load
+## Step 3 - Preparing the data
 
 %% Cell type:code id: tags:
 
 ``` python
-x_train,y_train,x_test,y_test = read_dataset('set-48x48-L-LHE')
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+xmax=x_train.max()
+x_train = x_train / xmax
+x_test  = x_test  / xmax
+
+print('After normalization  : Min={}, max={}'.format(x_train.min(),x_train.max()))
 ```
 
 %% Cell type:markdown id: tags:
 
-### 5.2 - Data augmentation
+### Have a look
 
 %% Cell type:code id: tags:
 
 ``` python
-datagen = keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
-                             featurewise_std_normalization=False,
-                             width_shift_range=0.1,
-                             height_shift_range=0.1,
-                             zoom_range=0.2,
-                             shear_range=0.1,
-                             rotation_range=10.)
-datagen.fit(x_train)
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 6 - Train the model
-**Get the shape of my data :**
+## Step 4 - Create model
+About informations about :
+ - [Optimizer](https://keras.io/api/optimizers)
+ - [Activation](https://keras.io/api/layers/activations)
+ - [Loss](https://keras.io/api/losses)
+ - [Metrics](https://keras.io/api/metrics)
 
 %% Cell type:code id: tags:
 
 ``` python
-(n,lx,ly,lz) = x_train.shape
-print("Images of the dataset have this folowing shape : ",(lx,ly,lz))
-```
+model = keras.models.Sequential()
 
-%% Cell type:markdown id: tags:
+model.add( keras.layers.Input((28,28,1)) )
+
+model.add( keras.layers.Conv2D(8, (3,3),  activation='relu') )
+model.add( keras.layers.MaxPooling2D((2,2)))
+model.add( keras.layers.Dropout(0.2))
 
-**Get and compile a model, with the data shape :**
+model.add( keras.layers.Conv2D(16, (3,3), activation='relu') )
+model.add( keras.layers.MaxPooling2D((2,2)))
+model.add( keras.layers.Dropout(0.2))
+
+model.add( keras.layers.Flatten())
+model.add( keras.layers.Dense(100, activation='relu'))
+model.add( keras.layers.Dropout(0.5))
+
+model.add( keras.layers.Dense(10, activation='softmax'))
+```
 
 %% Cell type:code id: tags:
 
 ``` python
-model = get_model_v3(lx,ly,lz)
-
-# model.summary()
+model.summary()
 
 model.compile(optimizer='adam',
               loss='sparse_categorical_crossentropy',
               metrics=['accuracy'])
 ```
 
 %% Cell type:markdown id: tags:
 
-**Train it :**
-Note : La courbe d'apprentissage est visible en temps réel avec Tensorboard :
-`#tensorboard --logdir ./run/logs`
+## Step 5 - Train the model
 
 %% Cell type:code id: tags:
 
 ``` python
-%%time
-
-batch_size = 64
-epochs     = 30
-
-# ---- Shuffle train data
-#x_train,y_train=ooo.shuffle_np_dataset(x_train,y_train)
-
-# ---- Train
-#
-history = model.fit(  datagen.flow(x_train, y_train, batch_size=batch_size),
-                      steps_per_epoch = int(x_train.shape[0]/batch_size),
-                      epochs=epochs,
-                      verbose=1,
-                      validation_data=(x_test, y_test),
-                      callbacks=[tensorboard_callback, bestmodel_callback, savemodel_callback] )
+batch_size  = 512
+epochs      =  16
 
-model.save('./run/models/last-model.h5')
+history = model.fit(  x_train, y_train,
+                      batch_size      = batch_size,
+                      epochs          = epochs,
+                      verbose         = fit_verbosity,
+                      validation_data = (x_test, y_test))
 ```
 
 %% Cell type:markdown id: tags:
 
-**Evaluate it :**
-
-%% Cell type:code id: tags:
-
-``` python
-max_val_accuracy = max(history.history["val_accuracy"])
-print("Max validation accuracy is : {:.4f}".format(max_val_accuracy))
-```
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+Note : With a DNN, we had a precision of the order of : 97.7%
 
 %% Cell type:code id: tags:
 
 ``` python
 score = model.evaluate(x_test, y_test, verbose=0)
 
-print('Test loss      : {:5.4f}'.format(score[0]))
-print('Test accuracy  : {:5.4f}'.format(score[1]))
+print(f'Test loss     : {score[0]:4.4f}')
+print(f'Test accuracy : {score[1]:4.4f}')
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 7 - History
-The return of model.fit() returns us the learning history
+### 6.2 - Plot history
 
 %% Cell type:code id: tags:
 
 ``` python
-ooo.plot_history(history)
+fidle.scrawler.history(history, figsize=(6,4), save_as='03-history')
 ```
 
 %% Cell type:markdown id: tags:
 
-## Step 8 - Evaluate best model
-
-%% Cell type:markdown id: tags:
-
-### 8.1 - Restore best model :
+### 6.3 - Plot results
 
 %% Cell type:code id: tags:
 
 ``` python
-loaded_model = tf.keras.models.load_model('./run/models/best-model.h5')
-# best_model.summary()
-print("Loaded.")
+#y_pred   = model.predict_classes(x_test)           Deprecated after 01/01/2021 !!
+
+y_sigmoid = model.predict(x_test, verbose=fit_verbosity)
+y_pred    = np.argmax(y_sigmoid, axis=-1)
+
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
 ```
 
 %% Cell type:markdown id: tags:
 
-### 8.2 - Evaluate it :
+### 6.4 - Plot some errors
 
 %% Cell type:code id: tags:
 
 ``` python
-score = loaded_model.evaluate(x_test, y_test, verbose=0)
-
-print('Test loss      : {:5.4f}'.format(score[0]))
-print('Test accuracy  : {:5.4f}'.format(score[1]))
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
 ```
 
-%% Cell type:markdown id: tags:
+%% Cell type:code id: tags:
 
-**Plot confusion matrix**
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
 
 %% Cell type:code id: tags:
 
 ``` python
-y_pred   = model.predict_classes(x_test)
-conf_mat = confusion_matrix(y_test,y_pred, normalize="true", labels=range(43))
-
-ooo.plot_confusion_matrix(conf_mat)
+fidle.end()
 ```
 
 %% Cell type:markdown id: tags:
 
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id: tags:
+
 ---
-![](../fidle/img/00-Fidle-logo-01_s.png)
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.Lightning/01-DNN-MNIST_Lightning.ipynb

+%% Cell type:markdown id:86fe2213-fb44-4bd4-a371-a541cba6a744 tags:
+
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [LMNIST1] - Simple classification with DNN
+<!-- DESC --> An example of classification using a dense neural network for the famous MNIST dataset, using PyTorch Lightning
+<!-- AUTHOR : MBOGOL Touye Achille (AI/ML Engineer EFELIA-MIAI/SIMAP Lab) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with pytorch lightning
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id:7f16101a-6612-4e02-93e9-c45ce1ac911d tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id:743c77d3-0983-491c-90be-ef2219861a47 tags:
+
+``` python
+import pandas as pd
+import numpy as np
+
+import torch
+import torch.nn as nn
+import lightning.pytorch as pl
+import torch.nn.functional as F
+import torchvision.transforms as T
+
+import sys,os
+import multiprocessing
+
+from torchvision import datasets
+from torchmetrics.functional import accuracy
+from torch.utils.data import Dataset, DataLoader
+from lightning.pytorch import loggers as pl_loggers
+from modules.progressbar import CustomTrainProgressBar
+from lightning.pytorch.loggers.tensorboard import TensorBoardLogger
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('LMNIST1')
+```
+
+%% Cell type:markdown id:df10dcda-aa63-476b-8665-9b1610fe51c6 tags:
+
+## Step 2 Retrieve data
+
+MNIST is one of the most famous historic dataset include in torchvision Datasets. `torchvision` provides many built-in datasets in the `torchvision.datasets`.
+
+%% Cell type:code id:6668e50c-f0c6-43cf-b733-9ac29d6a3900 tags:
+
+``` python
+# Load data sets
+train_dataset = datasets.MNIST(root="data", train=True,  download=True, transform=None)
+
+test_dataset  = datasets.MNIST(root="data", train=False, download=True, transform=None)
+```
+
+%% Cell type:code id:b543b885-6336-461d-abbe-6d3171405771 tags:
+
+``` python
+# print info for train data
+print(train_dataset)
+
+print()
+
+# print info for test data
+print(test_dataset)
+```
+
+%% Cell type:code id:44a489f5-3e53-4a2b-8069-f265b2814dc0 tags:
+
+``` python
+# See the shape of train data and test data
+print("x_train : ",train_dataset.data.shape)
+print("y_train : ",train_dataset.targets.shape)
+
+print()
+
+print("x_test  : ",test_dataset.data.shape)
+print("y_test  : ",test_dataset.targets.shape)
+
+print()
+
+# print number of labels or class
+print("Number of Targets :",len(np.unique(train_dataset.targets)))
+print("Targets Values :",    np.unique(train_dataset.targets))
+
+
+print("\nRemark that we work with torch tensors and not numpy array, not tensorflow tensor")
+print(" -> x_train.dtype = ",train_dataset.data.dtype)
+print(" -> y_train.dtype = ",train_dataset.targets.dtype)
+```
+
+%% Cell type:markdown id:b418adb7-33ea-450c-9793-3cdce5d5fa8c tags:
+
+## Step 3 -  Preparing your data for training with DataLoaders
+The Dataset retrieves our dataset’s features and labels one sample at a time. While training a model, we typically want to pass samples in `minibatches`, reshuffle the data at every epoch to reduce model overfitting, and use Python’s multiprocessing to speed up data retrieval. DataLoader is an iterable that abstracts this complexity for us in an easy API.
+
+%% Cell type:code id:8af0bc4c-acb3-46d9-aae2-143b0327d970 tags:
+
+``` python
+# Before normalization:
+x_train=train_dataset.data
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+#  After normalization:
+## T.Compose creates a pipeline where the provided transformations are run in sequence
+transforms = T.Compose(
+    [
+        # This transforms takes a np.array or a PIL image of integers
+        # in the range 0-255 and transforms it to a float tensor in the
+        # range 0.0 - 1.0
+        T.ToTensor()
+
+    ]
+)
+
+
+train_dataset = datasets.MNIST(root="data", train=True, download=True,  transform=transforms)
+test_dataset  = datasets.MNIST(root="data", train=False, download=True, transform=transforms)
+
+
+# print image and label After normalization.
+## iter() followed by next() is used to get some images and label.
+image,label=next(iter(train_dataset))
+print('After normalization  : Min={}, max={}'.format(image.min(),image.max()))
+```
+
+%% Cell type:markdown id:35d50a57-8274-4660-8765-d0f2bf7214bd tags:
+
+### Have a look
+
+%% Cell type:code id:a172ebc5-8858-4f30-8e2c-1e9c123ae0ee tags:
+
+``` python
+x_train=train_dataset.data
+y_train=train_dataset.targets
+```
+
+%% Cell type:code id:5a487760-b43a-4f7c-bfd8-1ce2c9652769 tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5, y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:code id:ca0a63ae-e6d6-4940-b8ff-9b11cb2737bb tags:
+
+``` python
+# train bacth data
+train_loader= DataLoader(
+  dataset=train_dataset,
+  shuffle=True,
+  batch_size=512,
+  num_workers=2
+)
+
+
+# test batch data
+test_loader= DataLoader(
+  dataset=test_dataset,
+  shuffle=False,
+  batch_size=512,
+  num_workers=2
+)
+
+# print image and label after normalization.
+image, label=next(iter(train_loader))
+print('Shape of first training data batch after use pytorch dataloader :\nbatch images = {} \nbatch labels = {}'.format(image.shape,label.shape))
+```
+
+%% Cell type:markdown id:51bf21ee-76ca-42fa-b67f-066dbd239a72 tags:
+
+## Step 4 - Create Model
+About informations about :
+ - [Optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)
+ - [Activation](https://www.tensorflow.org/api_docs/python/tf/keras/activations)
+ - [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses)
+ - [Metrics](https://www.tensorflow.org/api_docs/python/tf/keras/metrics)
+
+ `Note :` PyTorch provides losses such as the cross-entropy loss (`nn.CrossEntropyLoss`) usually use for classification problem. we're using the softmax function to predict class probabilities. With a softmax output, you want to use cross-entropy as the loss. To actually calculate the loss, we need to pass in the raw output of our network into the loss, not the output of the softmax function. This raw output is usually called the *logits* or *scores*. because in pytorch the cross entropy contain softmax function already.
+
+%% Cell type:code id:16701119-71eb-4f59-a50a-f153b07a74ae tags:
+
+``` python
+class MyNet(nn.Module):
+
+    def __init__(self,num_class=10):
+        super().__init__()
+        self.num_class = num_class
+        self.model     = nn.Sequential(
+
+            # Input vector:
+            nn.Flatten(),                    # convert each 2D 28x28 image into a contiguous array of 784 pixel values
+
+            # first hidden layer
+            nn.Linear(in_features=1*28*28, out_features=100),
+            nn.ReLU(),
+            nn.Dropout1d(0.1),               # Combat overfitting
+
+            # second hidden layer
+            nn.Linear(in_features=100, out_features=100),
+            nn.ReLU(),
+            nn.Dropout1d(0.1),               # Combat overfitting
+
+            # logits outpout
+            nn.Linear(100, num_class)
+        )
+
+
+
+    # forward pass
+    def forward(self, x):
+        return self.model(x)
+
+```
+
+%% Cell type:code id:37abf99b-f8ec-4048-a65d-f173ee18b234 tags:
+
+``` python
+class LitModel(pl.LightningModule):
+
+    def __init__(self, MyNet):
+        super().__init__()
+        self.MyNet = MyNet
+
+    # forward pass
+    def forward(self, x):
+        return self.MyNet(x)
+
+
+    def configure_optimizers(self):
+        # optimizer
+        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
+        return optimizer
+
+
+    def training_step(self, batch, batch_idx):
+        # defines the train loop
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        # computes the cross entropy loss between input logits and target
+        loss  = F.cross_entropy(y_hat, y)
+
+        # accuracy metrics
+        acc   = accuracy(y_hat, y,task="multiclass",num_classes=10)
+
+        metrics = {"train_loss": loss,
+                   "train_acc" : acc
+                  }
+
+        # logs metrics for each training_step
+        self.log_dict(metrics,
+                      on_step  = False,
+                      on_epoch = True,
+                      prog_bar = True,
+                      logger   = True
+                     )
+
+        return loss
+
+
+    def validation_step(self, batch, batch_idx):
+        #  defines the valid loop.
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        # computes the cross entropy loss between input logits and target
+        loss  = F.cross_entropy(y_hat, y)
+
+        # accuracy metrics
+        acc   = accuracy(y_hat, y,task="multiclass",num_classes=10)
+
+        metrics = {"test_loss": loss,
+                   "test_acc": acc
+                  }
+
+        # logs metrics for each validation_step
+        self.log_dict(metrics,
+                      on_step  = False,
+                      on_epoch = True,
+                      prog_bar = True,
+                      logger   = True
+                     )
+
+        return metrics
+
+
+    def predict_step(self, batch, batch_idx):
+        # defnie the predict loop
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        return y_hat
+
+```
+
+%% Cell type:code id:7546b27e-d492-420a-8d5d-109201b47830 tags:
+
+``` python
+# print summary model
+model=LitModel(MyNet())
+print(model)
+```
+
+%% Cell type:markdown id:fb32e85d-bd92-4ca5-a3dc-ddb5ed50ba6b tags:
+
+## Step 5 - Train Model
+
+%% Cell type:code id:96f0e087-f21a-4afc-85c5-3a3c0c353fe1 tags:
+
+``` python
+# loggers data
+os.makedirs(f'{run_dir}/logs',   mode=0o750, exist_ok=True)
+logger= TensorBoardLogger(save_dir=f'{run_dir}/logs',name="DNN_logs")
+```
+
+%% Cell type:code id:ce975c03-d05d-40c4-92ff-0cc90699c13e tags:
+
+``` python
+# train model
+# trainer= pl.Trainer(accelerator='auto',
+#                     max_epochs=20,
+#                     logger=logger,
+#                     num_sanity_val_steps=0,
+#                     callbacks=[CustomTrainProgressBar()]
+#                    )
+
+trainer= pl.Trainer(accelerator='auto',
+                    max_epochs=20,
+                    logger=logger,
+                    num_sanity_val_steps=0
+                   )
+
+trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=test_loader,)
+```
+
+%% Cell type:markdown id:a1191f05-4454-415c-a5ed-e63d9ae56651 tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+
+%% Cell type:code id:9f45316e-0d2d-4fc1-b9a8-5fb8aaf5586a tags:
+
+``` python
+score=trainer.validate(model=model,dataloaders=test_loader, verbose=False)
+
+print('x_test / acc      : {:5.4f}'.format(score[0]['test_acc']))
+print('x_test / loss     : {:5.4f}'.format(score[0]['test_loss']))
+```
+
+%% Cell type:markdown id:e352e48d-b473-4162-a1aa-72d6d4f7aa38 tags:
+
+## 6.2 - Plot history
+
+To access logs with tensorboad :
+- Under **Docker**, from a terminal launched via the jupyterlab launcher, use the following command:<br>
+```tensorboard --logdir <path-to-logs> --host 0.0.0.0```
+- If you're **not using Docker**, from a terminal :<br>
+```tensorboard --logdir <path-to-logs>```
+
+**Note:** One tensorboard instance can be used simultaneously.
+
+%% Cell type:markdown id:f00ded6b-a7db-4c5d-b1b2-72264db20bdb tags:
+
+###  6.3 - Plot results
+
+%% Cell type:code id:e387a70d-9c23-4d16-8ef7-879aec7791e2 tags:
+
+``` python
+# logits outpout by batch size
+y_logits=trainer.predict(model=model,dataloaders=test_loader)
+
+# Concat into single tensor
+y_logits=torch.cat(y_logits)
+
+# output probabilities values
+y_pred_values=F.softmax(y_logits,dim=1)
+
+# Returns the indices of the maximum output probabilities values
+y_pred=torch.argmax(y_pred_values,dim=-1).numpy()
+```
+
+%% Cell type:code id:fb2b2eeb-fcd8-453c-93ef-59a960a8bbd5 tags:
+
+``` python
+x_test=test_dataset.data
+y_test=test_dataset.targets
+```
+
+%% Cell type:code id:71187fa9-2ad3-4b23-94b9-1846045bd070 tags:
+
+``` python
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id:2fc7b2b9-9115-4848-9aae-2798bf7aa79a tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id:e55f17c4-fce7-423a-9adf-f2511c534ef5 tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id:fea1b396-70ca-4b00-851d-0538a4b347fb tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id:e982c032-cce8-4c71-8cdc-2af4b31b2914 tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id:233838c2-c97f-4489-8c79-9247d7b7456b tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id:51b87aa0-d4e9-48bb-8205-4b583f4b0b61 tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.Lightning/02-CNN-MNIST_Lightning.ipynb

+%% Cell type:markdown id:86fe2213-fb44-4bd4-a371-a541cba6a744 tags:
+
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+## <!-- TITLE --> [LMNIST2] - Simple classification with CNN
+<!-- DESC --> An example of classification using a convolutional neural network for the famous MNIST dataset, using PyTorch Lightning
+<!-- AUTHOR : MBOGOL Touye Achille (AI/ML Engineer MIAI/SIMaP) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with pytorch lightning
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id:7f16101a-6612-4e02-93e9-c45ce1ac911d tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id:743c77d3-0983-491c-90be-ef2219861a47 tags:
+
+``` python
+import pandas as pd
+import numpy as np
+
+import torch
+import torch.nn as nn
+import lightning.pytorch as pl
+import torch.nn.functional as F
+import torchvision.transforms as T
+
+import sys,os
+import multiprocessing
+import matplotlib.pyplot as plt
+
+from torchvision import datasets
+from torchmetrics.functional import accuracy
+from torch.utils.data import Dataset, DataLoader
+from modules.progressbar import CustomTrainProgressBar
+from lightning.pytorch.loggers import TensorBoardLogger
+
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('LMNIST2')
+```
+
+%% Cell type:markdown id:df10dcda-aa63-476b-8665-9b1610fe51c6 tags:
+
+## Step 2 Retrieve data
+
+MNIST is one of the most famous historic dataset include in torchvision Datasets. `torchvision` provides many built-in datasets in the `torchvision.datasets`.
+
+%% Cell type:code id:6668e50c-f0c6-43cf-b733-9ac29d6a3900 tags:
+
+``` python
+# Load data sets
+train_dataset = datasets.MNIST(root="data", train=True, download=True, transform=None)
+
+test_dataset= datasets.MNIST(root="data", train=False, download=False, transform=None)
+```
+
+%% Cell type:code id:a14d6fc2-b913-4eaa-9cde-5ca6785bfa12 tags:
+
+``` python
+# print info for train data
+print(train_dataset)
+
+print()
+
+# print info for test data
+print(test_dataset)
+```
+
+%% Cell type:code id:44a489f5-3e53-4a2b-8069-f265b2814dc0 tags:
+
+``` python
+# See the shape of train data and test data
+print("x_train : ",train_dataset.data.shape)
+print("y_train : ",train_dataset.targets.shape)
+
+print()
+
+print("x_test  : ",test_dataset.data.shape)
+print("y_test  : ",test_dataset.targets.shape)
+
+print()
+
+# print number of targets and  values targets
+print("Number of Targets :",len(np.unique(train_dataset.targets)))
+print("Targets Values    :",    np.unique(train_dataset.targets))
+
+print()
+
+print("Remark that we work with torch tensors and not numpy array, not tensorflow tensor")
+print(" -> x_train.dtype = ",train_dataset.data.dtype)
+print(" -> y_train.dtype = ",train_dataset.targets.dtype)
+```
+
+%% Cell type:markdown id:b418adb7-33ea-450c-9793-3cdce5d5fa8c tags:
+
+## Step 3 -  Preparing your data for training with DataLoaders
+The Dataset retrieves our dataset’s features and labels one sample at a time. While training a model, we typically want to pass samples in `minibatches`, reshuffle the data at every epoch to reduce model overfitting, and use Python’s multiprocessing to speed up data retrieval. DataLoader is an iterable that abstracts this complexity for us in an easy API.
+
+%% Cell type:code id:8af0bc4c-acb3-46d9-aae2-143b0327d970 tags:
+
+``` python
+# Before normalization:
+x_train=train_dataset.data
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+# After normalization:
+## T.Compose creates a pipeline where the provided transformations are run in sequence
+transforms = T.Compose(
+    [
+        # This transforms takes a np.array or a PIL image of integers
+        # in the range 0-255 and transforms it to a float tensor in the
+        # range 0.0 - 1.0
+        T.ToTensor(),
+
+    ]
+)
+
+
+train_dataset = datasets.MNIST(root="data", train=True, download=True, transform=transforms)
+test_dataset= datasets.MNIST(root="data", train=False, download=True, transform=transforms)
+
+
+# print image and label After normalization.
+# iter() followed by next() is used to get some images and label
+image,label=next(iter(train_dataset))
+print('After normalization  : Min={}, max={}'.format(image.min(),image.max()))
+```
+
+%% Cell type:markdown id:35d50a57-8274-4660-8765-d0f2bf7214bd tags:
+
+### Have a look
+
+%% Cell type:code id:a172ebc5-8858-4f30-8e2c-1e9c123ae0ee tags:
+
+``` python
+x_train=train_dataset.data
+y_train=train_dataset.targets
+```
+
+%% Cell type:code id:5a487760-b43a-4f7c-bfd8-1ce2c9652769 tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:code id:ca0a63ae-e6d6-4940-b8ff-9b11cb2737bb tags:
+
+``` python
+# train bacth data
+train_loader= DataLoader(
+  dataset=train_dataset,
+  shuffle=True,
+  batch_size=512,
+  num_workers=2
+)
+
+# test batch data
+test_loader= DataLoader(
+  dataset=test_dataset,
+  shuffle=False,
+  batch_size=512,
+  num_workers=2
+)
+
+# print image and label  After normalization and batch_size.
+image, label=next(iter(train_loader))
+print('Shape of first training data batch after use pytorch dataloader :\nbatch images = {} \nbatch labels = {}'.format(image.shape,label.shape))
+```
+
+%% Cell type:markdown id:51bf21ee-76ca-42fa-b67f-066dbd239a72 tags:
+
+## Step 4 - Create Model
+About informations about :
+ - [Optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)
+ - [Activation](https://www.tensorflow.org/api_docs/python/tf/keras/activations)
+ - [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses)
+ - [Metrics](https://www.tensorflow.org/api_docs/python/tf/keras/metrics)
+
+ `Note :` PyTorch provides losses such as the cross-entropy loss (`nn.CrossEntropyLoss`) usually use for classification problem. we're using the softmax function to predict class probabilities. With a softmax output, you want to use cross-entropy as the loss. To actually calculate the loss, we need to pass in the raw output of our network into the loss, not the output of the softmax function. This raw output is usually called the *logits* or *scores*. because in pytorch the cross entropy contain softmax function already.
+
+%% Cell type:code id:16701119-71eb-4f59-a50a-f153b07a74ae tags:
+
+``` python
+class MyNet(nn.Module):
+
+    def __init__(self,num_class=10):
+        super().__init__()
+        self.num_class=num_class
+        self.model = nn.Sequential(
+
+            # first convolution
+            nn.Conv2d(in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=0),
+            nn.ReLU(),
+            nn.MaxPool2d((2,2)),
+            nn.Dropout2d(0.1),            # Combat overfitting
+
+            # second convolution
+            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=3, stride=1, padding=0),
+            nn.ReLU(),
+            nn.MaxPool2d((2,2)),
+            nn.Dropout2d(0.1),            # Combat overfitting
+
+            nn.Flatten(),                 # convert feature map into feature vectors
+
+            # MLP network
+            nn.Linear(16*5*5,100),
+            nn.ReLU(),
+            nn.Dropout1d(0.1),            # Combat overfitting
+
+            nn.Linear(100, num_class),    # logits outpout
+        )
+
+    def forward(self, x):
+        x=self.model(x)                   # forward pass
+        return x
+```
+
+%% Cell type:code id:37abf99b-f8ec-4048-a65d-f173ee18b234 tags:
+
+``` python
+class LitModel(pl.LightningModule):
+
+    def __init__(self, MyNet):
+        super().__init__()
+        self.MyNet = MyNet
+
+    # forward pass
+    def forward(self, x):
+        return self.MyNet(x)
+
+
+     # optimizer
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
+        return optimizer
+
+
+    def training_step(self, batch, batch_idx):
+        # defines the train loop
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        # loss function
+        loss= F.cross_entropy(y_hat, y)
+
+        # metrics accuracy
+        acc=accuracy(y_hat, y,task="multiclass",num_classes=10)
+
+        metrics = {"train_loss": loss, "train_acc": acc}
+
+        # logs metrics for each training_step
+        self.log_dict(metrics,
+                      on_step=False ,
+                      on_epoch=True,
+                      prog_bar=True,
+                      logger=True)
+
+        return loss
+
+
+    def validation_step(self, batch, batch_idx):
+        # defines the valid loop.
+        x, y = batch
+
+        # forward pass
+        y_hat= self.MyNet(x)
+
+        # loss function MSE
+        loss = F.cross_entropy(y_hat, y)
+
+        # metrics accuracy
+        acc=accuracy(y_hat, y, task="multiclass", num_classes=10)
+
+
+        metrics = {"test_loss": loss, "test_acc": acc}
+
+        # logs metrics for each validation_step
+        self.log_dict(metrics,
+                      on_step  = False,
+                      on_epoch = True,
+                      prog_bar = True,
+                      logger   = True
+                     )
+
+
+        return metrics
+
+
+
+    def predict_step(self, batch, batch_idx):
+        # defnie the predict loop
+        x, y = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        return y_hat
+```
+
+%% Cell type:code id:489af62f-8f7c-4d1b-a6d0-5a0417e79869 tags:
+
+``` python
+# print summary model
+model=LitModel(MyNet())
+print(model)
+```
+
+%% Cell type:markdown id:fb32e85d-bd92-4ca5-a3dc-ddb5ed50ba6b tags:
+
+## Step 5 - Train Model
+
+%% Cell type:code id:756d5e19-6a10-42b8-8971-411389f7d19c tags:
+
+``` python
+# loggers data
+os.makedirs(f'{run_dir}/logs',   mode=0o750, exist_ok=True)
+logger= TensorBoardLogger(save_dir=f'{run_dir}/logs',name="CNN_logs")
+```
+
+%% Cell type:code id:ce975c03-d05d-40c4-92ff-0cc90699c13e tags:
+
+``` python
+# train model
+# trainer = pl.Trainer(accelerator='auto',
+#                      max_epochs=16,
+#                      logger=logger,
+#                      num_sanity_val_steps=0,
+#                      callbacks=[CustomTrainProgressBar()]
+#                     )
+
+trainer = pl.Trainer(accelerator='auto',
+                     max_epochs=16,
+                     logger=logger,
+                     num_sanity_val_steps=0
+                    )
+
+trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=test_loader)
+```
+
+%% Cell type:markdown id:a1191f05-4454-415c-a5ed-e63d9ae56651 tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+Note : With a DNN, we had a precision of the order of : 97.7%
+
+%% Cell type:code id:9f45316e-0d2d-4fc1-b9a8-5fb8aaf5586a tags:
+
+``` python
+# evaluate your model
+score=trainer.validate(model=model,dataloaders=test_loader, verbose=False)
+
+print('x_test / acc      : {:5.4f}'.format(score[0]['test_acc']))
+print('x_test / loss     : {:5.4f}'.format(score[0]['test_loss']))
+```
+
+%% Cell type:code id:5cfe9bd6-654b-42e0-b430-5f3b816526b0 tags:
+
+``` python
+score=trainer.validate(model=model,dataloaders=test_loader, verbose=False)
+```
+
+%% Cell type:markdown id:e352e48d-b473-4162-a1aa-72d6d4f7aa38 tags:
+
+## 6.2 - Plot history
+To access logs with tensorboad :
+- Under **Docker**, from a terminal launched via the jupyterlab launcher, use the following command:<br>
+```tensorboard --logdir <path-to-logs> --host 0.0.0.0```
+- If you're **not using Docker**, from a terminal :<br>
+```tensorboard --logdir <path-to-logs>```
+
+**Note:** One tensorboard instance can be used simultaneously.
+
+%% Cell type:markdown id:f00ded6b-a7db-4c5d-b1b2-72264db20bdb tags:
+
+###  6.3 - Plot results
+
+%% Cell type:code id:e387a70d-9c23-4d16-8ef7-879aec7791e2 tags:
+
+``` python
+# logits outpout by batch size
+y_logits= trainer.predict(model=model,dataloaders=test_loader)
+
+# Concat into single tensor
+y_logits= torch.cat(y_logits)
+
+# output probabilities values
+y_pred_values=F.softmax(y_logits,dim=1)
+
+# Returns the indices of the maximum output probabilities values
+y_pred=torch.argmax(y_pred_values,dim=-1)
+```
+
+%% Cell type:code id:fb2b2eeb-fcd8-453c-93ef-59a960a8bbd5 tags:
+
+``` python
+x_test=test_dataset.data
+y_test=test_dataset.targets
+```
+
+%% Cell type:code id:71187fa9-2ad3-4b23-94b9-1846045bd070 tags:
+
+``` python
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id:2fc7b2b9-9115-4848-9aae-2798bf7aa79a tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id:e55f17c4-fce7-423a-9adf-f2511c534ef5 tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id:fea1b396-70ca-4b00-851d-0538a4b347fb tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id:e982c032-cce8-4c71-8cdc-2af4b31b2914 tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id:233838c2-c97f-4489-8c79-9247d7b7456b tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id:51b87aa0-d4e9-48bb-8205-4b583f4b0b61 tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>