GTSRB.Keras3/modules/my_loader.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                           Dataset reader
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+
+import h5py
+import os
+import fidle
+
+
+def read_dataset(enhanced_dir, dataset_name, scale=1):
+    '''
+    Reads h5 dataset
+    Args:
+        filename     : datasets filename
+        dataset_name : dataset name, without .h5
+    Returns:    
+        x_train,y_train, x_test,y_test data, x_meta,y_meta
+    '''
+
+    # ---- Read dataset
+    #
+    chrono=fidle.Chrono()
+    chrono.start()
+    filename = f'{enhanced_dir}/{dataset_name}.h5'
+    with  h5py.File(filename,'r') as f:
+        x_train = f['x_train'][:]
+        y_train = f['y_train'][:]
+        x_test  = f['x_test'][:]
+        y_test  = f['y_test'][:]
+        x_meta  = f['x_meta'][:]
+        y_meta  = f['y_meta'][:]
+
+    # ---- Rescale 
+    #
+    print('Original shape  :', x_train.shape, y_train.shape)
+    x_train,y_train, x_test,y_test = fidle.utils.rescale_dataset(x_train,y_train,x_test,y_test, scale=scale)
+    print('Rescaled shape  :', x_train.shape, y_train.shape)
+
+    # ---- Shuffle
+    #
+    x_train,y_train=fidle.utils.shuffle_np_dataset(x_train,y_train)
+
+    # ---- done
+    #
+    duration = chrono.get_delay()
+    size     = fidle.utils.hsize(os.path.getsize(filename))
+    print(f'\nDataset "{dataset_name}" is loaded and shuffled. ({size} in {duration})')
+    return x_train,y_train, x_test,y_test, x_meta,y_meta
+
+
+
+
+print('Module my_loader loaded.')
\ No newline at end of file

GTSRB.Keras3/modules/my_models.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|                         Some nice models
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+
+import keras
+
+# ------------------------------------------------------------------
+# -- A simple model, for 24x24 or 48x48 images                    --
+# ------------------------------------------------------------------
+#
+def get_model_01(lx,ly,lz):
+    
+    model = keras.models.Sequential()
+
+    model.add( keras.layers.Input((lx,ly,lz)) )
+    
+    model.add( keras.layers.Conv2D(96, (3,3), activation='relu' ))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.2))
+
+    model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.2))
+
+    model.add( keras.layers.Flatten()) 
+    model.add( keras.layers.Dense(1500, activation='relu'))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Dense(43, activation='softmax'))
+    return model
+    
+
+# ------------------------------------------------------------------
+# -- A more sophisticated model, for 48x48 images                 --
+# ------------------------------------------------------------------
+#
+def get_model_02(lx,ly,lz):
+    model = keras.models.Sequential()
+    
+    model.add( keras.layers.Input((lx,ly,lz)) )
+    
+    model.add( keras.layers.Conv2D(32, (3,3),   activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
+    model.add( keras.layers.MaxPooling2D((2, 2)))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Flatten()) 
+    model.add( keras.layers.Dense(1152, activation='relu'))
+    model.add( keras.layers.Dropout(0.5))
+
+    model.add( keras.layers.Dense(43, activation='softmax'))
+    return model
+
+
+
+def get_model(name, lx,ly,lz):
+    '''
+    Return a model given by name
+    Args:
+        f_name : function name to retreive model
+        lxly,lz : inpuy shape 
+    Returns:
+        model
+    '''
+    if name=='model_01' : return get_model_01(lx,ly,lz)
+    if name=='model_02' : return get_model_01(lx,ly,lz)
+    print('*** Model not found : ', name)
+    return None
+
+# A More fun version ;-)
+def get_model2(name, lx,ly,lz):
+    get_model=globals()['get_'+name]
+    model=get_model(lx,ly,lz)
+    return model
+
+
+
+print('Module my_models loaded.')
\ No newline at end of file

GTSRB.Keras3/modules/my_tools.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|          A small traffic sign classifier
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
+# ------------------------------------------------------------------
+# JL Parouty 2023
+
+import numpy as np
+import matplotlib.pyplot as plt
+import fidle
+
+
+def show_prediction( prediction, x, y, x_meta ):
+
+    # ---- A prediction is just the output layer
+    #
+    fidle.utils.subtitle("Output layer from model is (x100) :")
+    with np.printoptions(precision=2, suppress=True, linewidth=95):
+        print(prediction*100)
+
+    # ---- Graphic visualisation
+    #
+    fidle.utils.subtitle("Graphically :")
+    plt.figure(figsize=(8,2))
+    plt.bar(range(43), prediction[0], align='center', alpha=0.5)
+    plt.ylabel('Probability')
+    plt.ylim((0,1))
+    plt.xlabel('Class')
+    plt.title('Trafic Sign prediction')
+    fidle.scrawler.save_fig('05-prediction-proba')
+    plt.show()
+
+    # ---- Predict class
+    #
+    p = np.argmax(prediction)
+
+    # ---- Show result
+    #
+    fidle.utils.subtitle('In pictures :')
+    print("\nThe image :               Prediction :            Real stuff:")
+    fidle.scrawler.images([x,x_meta[p], x_meta[y]], [p,p,y], range(3),  columns=3,  x_size=1.5, y_size=1.5, save_as='06-prediction-images')
+
+    if p==y:
+        print("YEEES ! that's right!")
+    else:
+        print("oups, that's wrong ;-(")
\ No newline at end of file

LinearReg/01-Linear-Regression.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [LINR1] - Linear regression with direct resolution
+<!-- DESC --> Low-level implementation, using numpy, of a direct resolution for a linear regression
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Just one, the illustration of a direct resolution :-)
+
+## What we're going to do :
+
+Equation : $$Y = X.\theta + N$$
+Where N is a noise vector
+and $\theta = (a,b)$ a vector as y = a.x + b
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Import and init
+
+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+import math
+import matplotlib
+import matplotlib.pyplot as plt
+import sys
+import fidle
+
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('LINR1')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Retrieve a set of points
+
+%% Cell type:code id: tags:
+
+``` python
+# ---- Paramètres
+nb    = 100     # Nombre de points
+xmin  = 0       # Distribution / x
+xmax  = 10
+a     = 4       # Distribution / y
+b     = 2       # y= a.x + b (+ bruit)
+noise = 7       # bruit
+
+theta = np.array([[a],[b]])
+
+# ---- Vecteur X  (1,x) x nb
+#      la premiere colonne est a 1 afin que X.theta <=> 1.b + x.a
+
+Xc1 = np.ones((nb,1))
+Xc2 = np.random.uniform(xmin,xmax,(nb,1))
+X = np.c_[ Xc1, Xc2 ]
+
+# ---- Noise
+# N = np.random.uniform(-noise,noise,(nb,1))
+N = noise * np.random.normal(0,1,(nb,1))
+
+# ---- Vecteur Y
+Y = (X @ theta) + N
+
+# print("X:\n",X,"\nY:\n ",Y)
+```
+
+%% Cell type:markdown id: tags:
+
+### Show it
+
+%% Cell type:code id: tags:
+
+``` python
+width = 12
+height = 6
+
+fig, ax = plt.subplots()
+fig.set_size_inches(width,height)
+ax.plot(X[:,1], Y, ".")
+ax.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+ax.set_xlabel('x axis')
+ax.set_ylabel('y axis')
+fidle.scrawler.save_fig('01-set_of_points')
+plt.show()
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Direct calculation of the normal equation
+
+
+We'll try to find an optimal value of $\theta$, minimizing a cost function.
+The cost function, classically used in the case of linear regressions, is the **root mean square error** (racine carré de l'erreur quadratique moyenne):
+
+$$RMSE(X,h_\theta)=\sqrt{\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2}$$
+
+With the simplified variant : $$MSE(X,h_\theta)=\frac1n\sum_{i=1}^n\left[h_\theta(X^{(i)})-Y^{(i)}\right]^2$$
+
+The optimal value of regression is : $$\hat{ \theta } =( X^{T} .X)^{-1}.X^{T}.Y$$
+
+Démontstration : https://eli.thegreenplace.net/2014/derivation-of-the-normal-equation-for-linear-regression
+
+%% Cell type:code id: tags:
+
+``` python
+theta_hat = np.linalg.inv(X.T @ X) @ X.T @ Y
+
+print("Theta :\n",theta,"\n\ntheta hat :\n",theta_hat)
+```
+
+%% Cell type:markdown id: tags:
+
+### Show it
+
+%% Cell type:code id: tags:
+
+``` python
+Xd = np.array([[1,xmin], [1,xmax]])
+Yd = Xd @ theta_hat
+
+fig, ax = plt.subplots()
+fig.set_size_inches(width,height)
+ax.plot(X[:,1], Y, ".")
+ax.plot(Xd[:,1], Yd, "-")
+ax.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+ax.set_xlabel('x axis')
+ax.set_ylabel('y axis')
+fidle.scrawler.save_fig('02-regression-line')
+plt.show()
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/02-Gradient-descent.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [GRAD1] - Linear regression with gradient descent
+<!-- DESC --> Low level implementation of a solution by gradient descent. Basic and stochastic approach.
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+
+## Objectives :
+ - To illustrate the iterative approach of a gradient descent
+
+## What we're going to do :
+
+Equation : $ Y = X.\Theta + N$
+Where N is a noise vector
+and $\Theta = (a,b)$ a vector as y = a.x + b
+
+We will calculate a loss function and its gradient.
+We will descend this gradient in order to find a minimum value of our loss function.
+
+$
+\triangledown_\theta MSE(\Theta)=\begin{bmatrix}
+\frac{\partial}{\partial \theta_0}MSE(\Theta)\\
+\frac{\partial}{\partial \theta_1}MSE(\Theta)\\
+\vdots\\
+\frac{\partial}{\partial \theta_n}MSE(\Theta)
+\end{bmatrix}=\frac2m X^T\cdot(X\cdot\Theta-Y)
+$
+
+and :
+
+$\Theta \leftarrow \Theta - \eta \cdot \triangledown_\theta MSE(\Theta)$
+
+where $\eta$ is the learning rate
+
+## Step 1 - Import and init
+
+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+import sys
+
+import fidle
+
+from modules.RegressionCooker import RegressionCooker
+
+# Init Fidle environment
+#
+run_id, run_dir, datasets_dir = fidle.init('GRAD1')
+
+# ---- Instanciate a Regression Cooker
+#
+cooker = RegressionCooker(fidle)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Get a dataset
+
+%% Cell type:code id: tags:
+
+``` python
+X,Y = cooker.get_dataset(1000000)
+
+cooker.plot_dataset(X,Y)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 : Data normalization
+
+%% Cell type:code id: tags:
+
+``` python
+X_norm     = ( X - X.mean() ) / X.std()
+Y_norm     = ( Y - Y.mean() ) / Y.std()
+
+cooker.vector_infos('X origine',X)
+cooker.vector_infos('X normalized',X_norm)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Basic descent
+
+%% Cell type:code id: tags:
+
+``` python
+theta = cooker.basic_descent(X_norm, Y_norm, epochs=200, eta=0.01)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 5 - Minibatch descent
+
+%% Cell type:code id: tags:
+
+``` python
+theta = cooker.minibatch_descent(X_norm, Y_norm, epochs=10, batchs=20, batch_size=10, eta=0.01)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/03-Polynomial-Regression.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [POLR1] - Complexity Syndrome
+<!-- DESC --> Illustration of the problem of complexity with the polynomial regression
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Visualizing and understanding under and overfitting
+
+## What we're going to do :
+
+We are looking for a polynomial function to approximate the observed series :
+$ y = a_n\cdot x^n + \dots + a_i\cdot x^i + \dots + a_1\cdot x + b $
+
+
+## Step 1 - Import and init
+
+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+import math
+import random
+import matplotlib
+import matplotlib.pyplot as plt
+import sys
+import fidle
+
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('POLR1')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Dataset generation
+
+%% Cell type:code id: tags:
+
+``` python
+# ---- Parameters
+
+n         = 100
+
+xob_min   = -5
+xob_max   = 5
+
+deg       =  7
+a_min     = -2
+a_max     =  2
+
+noise     =  2000
+
+# ---- Train data
+#      X,Y              : data
+#      X_norm,Y_norm    : normalized data
+
+X = np.random.uniform(xob_min,xob_max,(n,1))
+# N = np.random.uniform(-noise,noise,(n,1))
+N = noise * np.random.normal(0,1,(n,1))
+
+a = np.random.uniform(a_min,a_max, (deg,))
+fy = np.poly1d( a )
+
+Y = fy(X) + N
+
+# ---- Data normalization
+#
+X_norm = (X - X.mean(axis=0)) / X.std(axis=0)
+Y_norm = (Y - Y.mean(axis=0)) / Y.std(axis=0)
+
+# ---- Data visualization
+
+width = 12
+height = 6
+nb_viz = min(2000,n)
+
+def vector_infos(name,V):
+    m=V.mean(axis=0).item()
+    s=V.std(axis=0).item()
+    print("{:8} :      mean={:+12.4f}  std={:+12.4f}    min={:+12.4f}    max={:+12.4f}".format(name,m,s,V.min(),V.max()))
+
+
+fidle.utils.display_md('#### Generator :')
+print(f"Nomber of points={n}  deg={deg} bruit={noise}")
+
+fidle.utils.display_md('#### Datasets :')
+print(f"{nb_viz} points visibles sur {n})")
+plt.figure(figsize=(width, height))
+plt.plot(X[:nb_viz], Y[:nb_viz], '.')
+plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+plt.xlabel('x axis')
+plt.ylabel('y axis')
+fidle.scrawler.save_fig("01-dataset")
+plt.show()
+
+fidle.utils.display_md('#### Before normalization :')
+vector_infos('X',X)
+vector_infos('Y',Y)
+
+fidle.utils.display_md('#### After normalization :')
+vector_infos('X_norm',X_norm)
+vector_infos('Y_norm',Y_norm)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Polynomial regression with NumPy
+### 3.1 - Underfitting
+
+%% Cell type:code id: tags:
+
+``` python
+def draw_reg(X_norm, Y_norm, x_hat,fy_hat, size, save_as):
+    plt.figure(figsize=size)
+    plt.plot(X_norm, Y_norm, '.')
+
+    x_hat = np.linspace(X_norm.min(), X_norm.max(), 100)
+
+    plt.plot(x_hat, fy_hat(x_hat))
+    plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+    plt.xlabel('x axis')
+    plt.ylabel('y axis')
+    fidle.scrawler.save_fig(save_as)
+    plt.show()
+```
+
+%% Cell type:code id: tags:
+
+``` python
+reg_deg=1
+
+a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
+fy_hat  = np.poly1d( a_hat )
+
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='02-underfitting')
+```
+
+%% Cell type:markdown id: tags:
+
+### 3.2 - Good fitting
+
+%% Cell type:code id: tags:
+
+``` python
+reg_deg=5
+
+a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
+fy_hat  = np.poly1d( a_hat )
+
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='03-good_fitting')
+```
+
+%% Cell type:markdown id: tags:
+
+### 3.3 - Overfitting
+
+%% Cell type:code id: tags:
+
+``` python
+reg_deg=24
+
+a_hat   = np.polyfit(X_norm.reshape(-1,), Y_norm.reshape(-1,), reg_deg)
+fy_hat  = np.poly1d( a_hat )
+
+print(f'Nombre de degrés : {reg_deg}')
+draw_reg(X_norm[:nb_viz],Y_norm[:nb_viz], X_norm,fy_hat, (width,height), save_as='04-over_fitting')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/04-Logistic-Regression.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [LOGR1] - Logistic regression
+<!-- DESC --> Simple example of logistic regression with a sklearn solution
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - A logistic regression has the objective of providing a probability of belonging to a class.
+ - Découvrir une implémentation 100% Tensorflow ..et apprendre à aimer Keras
+
+## What we're going to do :
+
+X contains characteristics
+y contains the probability of membership (1 or 0)
+
+We'll look for a value of $\theta$ such that the linear regression $\theta^{T}X$ can be used to calculate our probability:
+
+$\hat{p} = h_\theta(X) = \sigma(\theta^T{X})$
+
+Where $\sigma$ is the logit function, typically a sigmoid (S) function:
+
+$
+\sigma(t) = \dfrac{1}{1 + \exp(-t)}
+$
+
+The predicted value $\hat{y}$ will then be calculated as follows:
+
+$
+\hat{y} =
+\begin{cases}
+  0 & \text{if } \hat{p} < 0.5 \\
+  1 & \text{if } \hat{p} \geq 0.5
+\end{cases}
+$
+
+**Calculation of the cost of the regression:**
+For a training observation x, the cost can be calculated as follows:
+
+$
+c(\theta) =
+\begin{cases}
+  -\log(\hat{p}) & \text{if } y = 1 \\
+  -\log(1 - \hat{p}) & \text{if } y = 0
+\end{cases}
+$
+
+The regression cost function (log loss) over the whole training set can be written as follows:
+
+$
+J(\theta) = -\dfrac{1}{m} \sum_{i=1}^{m}{\left[ y^{(i)} log\left(\hat{p}^{(i)}\right) + (1 - y^{(i)}) log\left(1 - \hat{p}^{(i)}\right)\right]}
+$
+## Step 1 - Import and init
+
+You can also adjust the verbosity by changing the value of TF_CPP_MIN_LOG_LEVEL :
+- 0 = all messages are logged (default)
+- 1 = INFO messages are not printed.
+- 2 = INFO and WARNING messages are not printed.
+- 3 = INFO , WARNING and ERROR messages are not printed.
+
+%% Cell type:code id: tags:
+
+``` python
+# import os
+# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+import numpy as np
+from sklearn import metrics
+from sklearn.linear_model import LogisticRegression
+
+import matplotlib
+import matplotlib.pyplot as plt
+# import math
+import random
+# import os
+import sys
+
+import fidle
+
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('LOGR1')
+```
+
+%% Cell type:markdown id: tags:
+
+### 1.1 - Usefull stuff (hidden)
+
+%% Cell type:code id: tags:
+
+``` python
+def vector_infos(name,V):
+    '''Displaying some information about a vector'''
+    with np.printoptions(precision=4, suppress=True):
+        print("{:16} : ndim={}  shape={:10}  Mean = {}  Std = {}".format( name,V.ndim, str(V.shape), V.mean(axis=0), V.std(axis=0)))
+
+
+def do_i_have_it(hours_of_work, hours_of_sleep):
+    '''Returns the exam result based on work and sleep hours'''
+    hours_of_sleep_min = 5
+    hours_of_work_min  = 4
+    hours_of_game_max  = 3
+    # ---- Have to sleep and work
+    if hours_of_sleep < hours_of_sleep_min: return 0
+    if hours_of_work < hours_of_work_min:   return 0
+    # ---- Gameboy is not good for you
+    hours_of_game = 24 - 10 - hours_of_sleep - hours_of_work + random.gauss(0,0.4)
+    if hours_of_game > hours_of_game_max:   return 0
+    # ---- Fine, you got it
+    return 1
+
+
+def make_students_dataset(size, noise):
+    '''Fabrique un dataset pour <size> étudiants'''
+    x = []
+    y = []
+    for i in range(size):
+        w = random.gauss(5,1)
+        s = random.gauss(7,1.5)
+        r   = do_i_have_it(w,s)
+        x.append([w,s])
+        y.append(r)
+    return (np.array(x), np.array(y))
+
+
+def plot_data(x,y, colors=('green','red'), legend=True):
+    '''Affiche un dataset'''
+    fig, ax = plt.subplots(1, 1)
+    fig.set_size_inches(10,8)
+    ax.plot(x[y==1, 0], x[y==1, 1], 'o', color=colors[0], markersize=4, label="y=1 (positive)")
+    ax.plot(x[y==0, 0], x[y==0, 1], 'o', color=colors[1], markersize=4, label="y=0 (negative)")
+    if legend : ax.legend()
+    plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+    plt.xlabel('Hours of work')
+    plt.ylabel('Hours of sleep')
+    plt.show()
+
+
+def plot_results(x_test,y_test, y_pred):
+    '''Affiche un resultat'''
+
+    precision = metrics.precision_score(y_test, y_pred)
+    recall    = metrics.recall_score(y_test, y_pred)
+
+    print("Accuracy = {:5.3f}    Recall = {:5.3f}".format(precision, recall))
+
+    x_pred_positives = x_test[ y_pred == 1 ]     # items prédits    positifs
+    x_real_positives = x_test[ y_test == 1 ]     # items réellement positifs
+    x_pred_negatives = x_test[ y_pred == 0 ]     # items prédits    négatifs
+    x_real_negatives = x_test[ y_test == 0 ]     # items réellement négatifs
+
+    fig, axs = plt.subplots(2, 2)
+    fig.subplots_adjust(wspace=.1,hspace=0.2)
+    fig.set_size_inches(14,10)
+
+    axs[0,0].plot(x_pred_positives[:,0], x_pred_positives[:,1], 'o',color='lightgreen', markersize=10, label="Prédits positifs")
+    axs[0,0].plot(x_real_positives[:,0], x_real_positives[:,1], 'o',color='green',      markersize=4,  label="Réels positifs")
+    axs[0,0].legend()
+    axs[0,0].tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+    axs[0,0].set_xlabel('$x_1$')
+    axs[0,0].set_ylabel('$x_2$')
+
+
+    axs[0,1].plot(x_pred_negatives[:,0], x_pred_negatives[:,1], 'o',color='lightsalmon', markersize=10, label="Prédits négatifs")
+    axs[0,1].plot(x_real_negatives[:,0], x_real_negatives[:,1], 'o',color='red',        markersize=4,  label="Réels négatifs")
+    axs[0,1].legend()
+    axs[0,1].tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+    axs[0,1].set_xlabel('$x_1$')
+    axs[0,1].set_ylabel('$x_2$')
+
+    axs[1,0].plot(x_pred_positives[:,0], x_pred_positives[:,1], 'o',color='lightgreen', markersize=10, label="Prédits positifs")
+    axs[1,0].plot(x_pred_negatives[:,0], x_pred_negatives[:,1], 'o',color='lightsalmon', markersize=10, label="Prédits négatifs")
+    axs[1,0].plot(x_real_positives[:,0], x_real_positives[:,1], 'o',color='green',      markersize=4,  label="Réels positifs")
+    axs[1,0].plot(x_real_negatives[:,0], x_real_negatives[:,1], 'o',color='red',        markersize=4,  label="Réels négatifs")
+    axs[1,0].tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+    axs[1,0].set_xlabel('$x_1$')
+    axs[1,0].set_ylabel('$x_2$')
+
+    axs[1,1].pie([precision,1-precision], explode=[0,0.1], labels=["","Errors"],
+                 autopct='%1.1f%%', shadow=False, startangle=70, colors=["lightsteelblue","coral"])
+    axs[1,1].axis('equal')
+
+    plt.show()
+```
+
+%% Cell type:markdown id: tags:
+
+### 1.2 - Parameters
+
+%% Cell type:code id: tags:
+
+``` python
+data_size      = 1000       # Number of observations
+data_cols      = 2          # observation size
+data_noise     = 0.2
+random_seed    = 123
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Data preparation
+### 2.1 - Get some data
+The data here are totally fabricated and represent the **examination results** (passed or failed) based on the students' **working** and **sleeping hours** .
+X=(working hours, sleeping hours) y={result} where result=0 (failed) or 1 (passed)
+
+%% Cell type:code id: tags:
+
+``` python
+x_data,y_data=make_students_dataset(data_size,data_noise)
+```
+
+%% Cell type:markdown id: tags:
+
+### 2.2 - Show it
+
+%% Cell type:code id: tags:
+
+``` python
+plot_data(x_data, y_data)
+vector_infos('Dataset X',x_data)
+vector_infos('Dataset y',y_data)
+```
+
+%% Cell type:markdown id: tags:
+
+### 2.3 - Preparation of data
+
+We're going to:
+- split the data to have : :
+  - a training set
+  - a test set
+- normalize the data
+
+%% Cell type:code id: tags:
+
+``` python
+# ---- Split data
+
+n = int(data_size * 0.8)
+x_train = x_data[:n]
+y_train = y_data[:n]
+x_test  = x_data[n:]
+y_test  = y_data[n:]
+
+# ---- Normalization
+
+mean = np.mean(x_train, axis=0)
+std  = np.std(x_train, axis=0)
+
+x_train = (x_train-mean)/std
+x_test  = (x_test-mean)/std
+
+# ---- About it
+
+vector_infos('X_train',x_train)
+vector_infos('y_train',y_train)
+vector_infos('X_test',x_test)
+vector_infos('y_test',y_test)
+
+y_train_h = y_train.reshape(-1,) # nécessaire pour la visu.
+```
+
+%% Cell type:markdown id: tags:
+
+### 2.4 - Have a look
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.utils.display_md('**This is what we know :**')
+plot_data(x_train, y_train)
+fidle.utils.display_md('**This is what we want to classify :**')
+plot_data(x_test,  y_test, colors=("gray","gray"), legend=False)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Logistic model #1
+### 3.1 - Here is the classifier
+
+%% Cell type:code id: tags:
+
+``` python
+# ---- Create an instance
+#      Use SAGA solver (Stochastic Average Gradient descent solver)
+#
+logreg = LogisticRegression(C=1e5, verbose=0, solver='saga')
+
+# ---- Fit the data.
+#
+logreg.fit(x_train, y_train)
+
+# ---- Do a prediction
+#
+y_pred = logreg.predict(x_test)
+```
+
+%% Cell type:markdown id: tags:
+
+
+### 3.3 - Evaluation
+
+Accuracy = Ability to avoid false positives = $\frac{Tp}{Tp+Fp}$
+Recall = Ability to find the right positives = $\frac{Tp}{Tp+Fn}$
+Avec :
+$T_p$ (true positive) Correct positive answer
+$F_p$ (false positive) False positive answer
+$T_n$ (true negative) Correct negative answer
+$F_n$ (false negative) Wrong negative answer
+
+%% Cell type:code id: tags:
+
+``` python
+plot_results(x_test,y_test, y_pred)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Bending the space to a model #2 ;-)
+
+We're going to increase the characteristics of our observations, with : ${x_1}^2$, ${x_2}^2$, ${x_1}^3$ et ${x_2}^3$
+
+$
+X=
+\begin{bmatrix}1 & x_{11} & x_{12} \\
+\vdots & \dots\\
+1 & x_{m1} & x_{m2}  \end{bmatrix}
+\text{et }
+X_{ng}=\begin{bmatrix}1 & x_{11} & x_{12} & x_{11}^2 & x_{12}^2& x_{11}^3 & x_{12}^3 \\
+\vdots & & & \dots \\
+1 & x_{m1} & x_{m2} & x_{m1}^2 & x_{m2}^2& x_{m1}^3 & x_{m2}^3 \end{bmatrix}
+$
+
+Note : `sklearn.preprocessing.PolynomialFeatures` can do that for us, but we'll do it ourselves:
+### 4.1 - Extend data
+
+%% Cell type:code id: tags:
+
+``` python
+x_train_enhanced = np.c_[x_train,
+                         x_train[:, 0] ** 2,
+                         x_train[:, 1] ** 2,
+                         x_train[:, 0] ** 3,
+                         x_train[:, 1] ** 3]
+x_test_enhanced = np.c_[x_test,
+                        x_test[:, 0] ** 2,
+                        x_test[:, 1] ** 2,
+                        x_test[:, 0] ** 3,
+                        x_test[:, 1] ** 3]
+```
+
+%% Cell type:markdown id: tags:
+
+### 4.2 - Run the classifier
+
+%% Cell type:code id: tags:
+
+``` python
+# ---- Create an instance
+#      Use SAGA solver (Stochastic Average Gradient descent solver)
+#
+logreg = LogisticRegression(C=1e5, verbose=0, solver='saga', max_iter=5000, n_jobs=-1)
+
+# ---- Fit the data.
+#
+logreg.fit(x_train_enhanced, y_train)
+
+# ---- Do a prediction
+#
+y_pred = logreg.predict(x_test_enhanced)
+```
+
+%% Cell type:markdown id: tags:
+
+### 4.3 - Evaluation
+
+%% Cell type:code id: tags:
+
+``` python
+plot_results(x_test_enhanced, y_test, y_pred)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

LinearReg/modules/RegressionCooker.py

+
+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|  Regression cooker
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning  (FIDLE)
+# CNRS/SARI/DEVLOG 2020 - S. Arias, E. Maldonado, JL. Parouty
+# ------------------------------------------------------------------
+# Initial version by JL Parouty, feb 2020
+
+import numpy as np
+import math
+import random
+import datetime, time, sys
+
+import matplotlib
+import matplotlib.pyplot as plt
+from IPython.display import display,Markdown,HTML
+
+
+class RegressionCooker():
+    
+    fidle   = None
+    version = '0.1'
+    
+    def __init__(self, fidle):
+        self.fidle = fidle
+        fidle.utils.subtitle('FIDLE 2020 - Regression Cooker')
+        print('Version      :', self.version)
+        print('Run time     : {}'.format(time.strftime("%A %d %B %Y, %H:%M:%S")))
+        
+
+    @classmethod
+    def about(cls):
+        print('\nFIDLE 2020 - Regression Cooker)')
+        print('Version       :', cls.version)
+    
+    
+    @classmethod
+    def vector_infos(cls,name,V):
+        """
+        Show some nice infos about a vector
+        args:
+            name : vector name
+            V    : vector
+        """
+        m=V.mean(axis=0).item()
+        s=V.std(axis=0).item()
+        print("{:16} :      mean={:8.3f}  std={:8.3f}    min={:8.3f}    max={:8.3f}".format(name,m,s,V.min(),V.max()))
+    
+    
+    def get_dataset(self,n):
+        """
+        Return a dataset of n observation
+        args:
+            n       : dataset size
+        return:
+            X,Y : with X shapes = (n,1) Y shape = (n,)
+        """
+        
+        xob_min   = 0       # x min and max
+        xob_max   = 10
+
+        a_min     = -30     # a min and max
+        a_max     =  30
+        b_min     = -10     # b min and max
+        b_max     =  10
+
+        noise_min =  10     # noise min and max
+        noise_max =  50
+
+        a0    = random.randint(a_min,a_max)       
+        b0    = random.randint(b_min,b_max)       
+        noise = random.randint(noise_min,noise_max)
+
+        # ---- Construction du jeu d'apprentissage ---------------
+        #      X,Y              : données brutes
+
+        X = np.random.uniform(xob_min,xob_max,(n,1))
+        N = noise * np.random.normal(0,1,(n,1))
+        Y = a0*X + b0 + N
+
+        return X,Y
+    
+
+    
+    def plot_dataset(self,X,Y,title='Dataset :',width=12,height=6):
+        """
+        Plot dataset X,Y
+        args:
+            X : Observations
+            Y : Values
+        """
+        nb_viz = min(1000,len(X))
+        display(Markdown(f'### {title}'))
+        print(f"X shape : {X.shape}  Y shape : {Y.shape}  plot : {nb_viz} points")
+        plt.figure(figsize=(width, height))
+        plt.plot(X[:nb_viz], Y[:nb_viz], '.')
+        self.fidle.scrawler.save_fig('01-dataset')
+        plt.show()
+        self.vector_infos('X',X)
+        self.vector_infos('Y',Y)
+
+        
+    def __plot_theta(self, i, theta,x_min,x_max, loss,gradient,alpha):
+        Xd = np.array([[x_min], [x_max]])
+        Yd = Xd * theta.item(1) + theta.item(0)
+        plt.plot(Xd, Yd, color=(1.,0.4,0.3,alpha))
+        if i<0:
+            print( "    #i   Loss       Gradient         Theta")
+        else:
+            print("  {:3d}  {:+7.3f}  {:+7.3f} {:+7.3f}  {:+7.3f} {:+7.3f}".format(i,loss,gradient.item(0),
+                                                                                   gradient.item(1),theta.item(0),
+                                                                                   theta.item(1)))
+
+            
+    def __plot_XY(self, X,Y,width=12,height=6):
+        nb_viz = min(1000,len(X))
+        plt.figure(figsize=(width, height))
+        plt.plot(X[:nb_viz], Y[:nb_viz], '.') 
+        plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+        plt.xlabel('x axis')
+        plt.ylabel('y axis')
+        
+    def __plot_loss(self,loss, width=8,height=4):
+        plt.figure(figsize=(width, height))
+        plt.tick_params(axis='both', which='both', bottom=False, left=False, labelbottom=False, labelleft=False)
+        plt.ylim(0, 20)
+        plt.plot(range(len(loss)), loss)
+        plt.xlabel('Iterations')
+        plt.ylabel('Loss')
+
+        
+    def basic_descent(self, X, Y, epochs=200, eta=0.01,width=12,height=6):
+        """
+        Performs a gradient descent where the gradient is updated at the end
+        of each iteration for all observations.
+        args:
+            X,Y          : Observations
+            epochs       : Number of epochs (200)
+            eta          : learning rate
+            width,height : graphic size
+        return:
+            theta        : theta
+        """
+
+        display(Markdown(f'### Basic gradient descent :'))
+
+        display(Markdown(f'**With :**  '))
+        print('with :')
+        print(f'    epochs = {epochs}')
+        print(f'    eta    = {eta}')
+
+        display(Markdown(f'**epochs :**  '))
+        x_min = X.min()
+        x_max = X.max()
+        y_min = Y.min()
+        y_max = Y.max()
+        n     = len(X)
+        
+        # ---- Initialization
+
+        theta = np.array([[y_min],[0]])
+        X_b = np.c_[np.ones((n, 1)), X]
+
+        # ---- Visualization
+        
+        self.__plot_XY(X,Y,width,height)
+        self.__plot_theta( -1, theta,x_min,x_max, None,None,0.1)
+        
+        # ---- Training
+        
+        loss=[]
+        for i in range(epochs+1):
+
+            gradient = (2/n) * X_b.T @ ( X_b @ theta - Y)
+            mse = ((X_b @ theta - Y)**2).mean(axis=None)
+
+            theta = theta - eta * gradient
+            
+            loss.append(mse)
+            if (i % (epochs/10))==0:
+                self.__plot_theta( i, theta,x_min,x_max, mse,gradient,i/epochs)
+
+        # ---- Visualization
+
+        self.fidle.utils.subtitle('Visualization :')
+        self.fidle.scrawler.save_fig('02-basic_descent')
+        plt.show()
+
+        self.fidle.utils.subtitle('Loss :')
+        self.__plot_loss(loss)
+        self.fidle.scrawler.save_fig('03-basic_descent_loss')
+        plt.show()
+        
+        return theta
+    
+    
+    def minibatch_descent(self, X, Y, epochs=200, batchs=5, batch_size=10, eta=0.01,width=12,height=6):
+        """
+        Performs a gradient descent where the gradient is updated at the end
+        of each iteration for all observations.
+        args:
+            X,Y          : Observations
+            epochs       : Number of epochs (200)
+            eta          : learning rate
+            width,height : graphic size
+        return:
+            theta        : theta
+        """
+
+        display(Markdown(f'### Mini batch gradient descent :'))
+
+        display(Markdown(f'**With :**  '))
+        print('with :')
+        print(f'    epochs     = {epochs}')
+        print(f'    batchs     = {batchs}')
+        print(f'    batch size = {batch_size}')
+        print(f'    eta        = {eta}')
+
+        display(Markdown(f'**epochs :**  '))
+        x_min = X.min()
+        x_max = X.max()
+        y_min = Y.min()
+        y_max = Y.max()
+        n     = len(X)
+        
+        # ---- Initialization
+
+        theta = np.array([[y_min],[0]])
+        X_b = np.c_[np.ones((n, 1)), X]
+
+        # ---- Visualization
+        
+        self.__plot_XY(X,Y,width,height)
+        self.__plot_theta( -1, theta,x_min,x_max, None,None,0.1)
+
+        # ---- Training
+
+        def learning_schedule(t):
+            return 1 / (t + 100)
+
+        loss=[]
+        for epoch in range(epochs):
+            for i in range(batchs):
+
+                random_index = np.random.randint(n-batch_size)
+
+                xi = X_b[random_index:random_index+batch_size]
+                yi = Y[random_index:random_index+batch_size]
+
+                mse = ((xi @ theta - yi)**2).mean(axis=None)
+                gradient = 2 * xi.T.dot(xi.dot(theta) - yi)
+
+                eta = learning_schedule(epoch*150)
+                theta = theta - eta * gradient
+
+            loss.append(mse)
+            self.__plot_theta( epoch, theta,x_min,x_max, mse,gradient,epoch/epochs)
+#             draw_theta(epoch,mse,gradients, theta,0.1+epoch/(n_epochs+1))
+
+#         draw_theta(epoch,mse,gradients,theta,1)
+        
+        # ---- Visualization
+
+        self.fidle.utils.subtitle('Visualization :')
+        self.fidle.scrawler.save_fig('04-minibatch_descent')
+        plt.show()
+
+        self.fidle.utils.subtitle('Loss :')
+        self.__plot_loss(loss)
+        self.fidle.scrawler.save_fig('05-minibatch_descent_loss')
+        plt.show()
+        
+        
+        return theta

MNIST.Keras3/01-DNN-MNIST.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [K3MNIST1] - Simple classification with DNN
+<!-- DESC --> An example of classification using a dense neural network for the famous MNIST dataset
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with Keras
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id: tags:
+
+``` python
+import os
+os.environ['KERAS_BACKEND'] = 'torch'
+
+import keras
+
+import numpy as np
+import matplotlib.pyplot as plt
+import sys,os
+from importlib import reload
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('K3MNIST1')
+```
+
+%% Cell type:markdown id: tags:
+
+Verbosity during training : 0 = silent, 1 = progress bar, 2 = one line per epoch
+
+%% Cell type:code id: tags:
+
+``` python
+fit_verbosity = 1
+```
+
+%% Cell type:markdown id: tags:
+
+Override parameters (batch mode) - Just forget this cell
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.override('fit_verbosity')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Retrieve data
+MNIST is one of the most famous historic dataset.
+Include in [Keras datasets](https://keras.io/datasets)
+
+%% Cell type:code id: tags:
+
+``` python
+(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
+
+print("x_train : ",x_train.shape)
+print("y_train : ",y_train.shape)
+print("x_test  : ",x_test.shape)
+print("y_test  : ",y_test.shape)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Preparing the data
+
+%% Cell type:code id: tags:
+
+``` python
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+xmax=x_train.max()
+x_train = x_train / xmax
+x_test  = x_test  / xmax
+
+print('After normalization  : Min={}, max={}'.format(x_train.min(),x_train.max()))
+```
+
+%% Cell type:markdown id: tags:
+
+### Have a look
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Create model
+About informations about :
+ - [Optimizer](https://keras.io/api/optimizers)
+ - [Activation](https://keras.io/api/layers/activations)
+ - [Loss](https://keras.io/api/losses)
+ - [Metrics](https://keras.io/api/metrics)
+
+%% Cell type:code id: tags:
+
+``` python
+hidden1     = 100
+hidden2     = 100
+
+model = keras.Sequential([
+    keras.layers.Input((28,28)),
+    keras.layers.Flatten(),
+    keras.layers.Dense( hidden1, activation='relu'),
+    keras.layers.Dense( hidden2, activation='relu'),
+    keras.layers.Dense( 10,      activation='softmax')
+])
+
+model.compile(optimizer='adam',
+              loss='sparse_categorical_crossentropy',
+              metrics=['accuracy'])
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 5 - Train the model
+
+%% Cell type:code id: tags:
+
+``` python
+batch_size  = 512
+epochs      =  16
+
+history = model.fit(  x_train, y_train,
+                      batch_size      = batch_size,
+                      epochs          = epochs,
+                      verbose         = fit_verbosity,
+                      validation_data = (x_test, y_test))
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+
+%% Cell type:code id: tags:
+
+``` python
+score = model.evaluate(x_test, y_test, verbose=0)
+
+print('Test loss     :', score[0])
+print('Test accuracy :', score[1])
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.2 - Plot history
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.history(history, figsize=(6,4), save_as='03-history')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.3 - Plot results
+
+%% Cell type:code id: tags:
+
+``` python
+#y_pred   = model.predict_classes(x_test)           Deprecated after 01/01/2021 !!
+
+y_sigmoid = model.predict(x_test, verbose=fit_verbosity)
+y_pred    = np.argmax(y_sigmoid, axis=-1)
+
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id: tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.Keras3/02-CNN-MNIST.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [K3MNIST2] - Simple classification with CNN
+<!-- DESC --> An example of classification using a convolutional neural network for the famous MNIST dataset
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with Keras
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id: tags:
+
+``` python
+import os
+os.environ['KERAS_BACKEND'] = 'torch'
+
+import keras
+
+import numpy as np
+import matplotlib.pyplot as plt
+import sys,os
+from importlib import reload
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('K3MNIST2')
+```
+
+%% Cell type:markdown id: tags:
+
+Verbosity during training : 0 = silent, 1 = progress bar, 2 = one line per epoch
+
+%% Cell type:code id: tags:
+
+``` python
+fit_verbosity = 1
+```
+
+%% Cell type:markdown id: tags:
+
+Override parameters (batch mode) - Just forget this cell
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.override('fit_verbosity')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Retrieve data
+MNIST is one of the most famous historic dataset.
+Include in [Keras datasets](https://keras.io/datasets)
+
+%% Cell type:code id: tags:
+
+``` python
+(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
+
+x_train = x_train.reshape(-1,28,28,1)
+x_test  = x_test.reshape(-1,28,28,1)
+
+print("x_train : ",x_train.shape)
+print("y_train : ",y_train.shape)
+print("x_test  : ",x_test.shape)
+print("y_test  : ",y_test.shape)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Preparing the data
+
+%% Cell type:code id: tags:
+
+``` python
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+xmax=x_train.max()
+x_train = x_train / xmax
+x_test  = x_test  / xmax
+
+print('After normalization  : Min={}, max={}'.format(x_train.min(),x_train.max()))
+```
+
+%% Cell type:markdown id: tags:
+
+### Have a look
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Create model
+About informations about :
+ - [Optimizer](https://keras.io/api/optimizers)
+ - [Activation](https://keras.io/api/layers/activations)
+ - [Loss](https://keras.io/api/losses)
+ - [Metrics](https://keras.io/api/metrics)
+
+%% Cell type:code id: tags:
+
+``` python
+model = keras.models.Sequential()
+
+model.add( keras.layers.Input((28,28,1)) )
+
+model.add( keras.layers.Conv2D(8, (3,3),  activation='relu') )
+model.add( keras.layers.MaxPooling2D((2,2)))
+model.add( keras.layers.Dropout(0.2))
+
+model.add( keras.layers.Conv2D(16, (3,3), activation='relu') )
+model.add( keras.layers.MaxPooling2D((2,2)))
+model.add( keras.layers.Dropout(0.2))
+
+model.add( keras.layers.Flatten())
+model.add( keras.layers.Dense(100, activation='relu'))
+model.add( keras.layers.Dropout(0.5))
+
+model.add( keras.layers.Dense(10, activation='softmax'))
+```
+
+%% Cell type:code id: tags:
+
+``` python
+model.summary()
+
+model.compile(optimizer='adam',
+              loss='sparse_categorical_crossentropy',
+              metrics=['accuracy'])
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 5 - Train the model
+
+%% Cell type:code id: tags:
+
+``` python
+batch_size  = 512
+epochs      =  16
+
+history = model.fit(  x_train, y_train,
+                      batch_size      = batch_size,
+                      epochs          = epochs,
+                      verbose         = fit_verbosity,
+                      validation_data = (x_test, y_test))
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+Note : With a DNN, we had a precision of the order of : 97.7%
+
+%% Cell type:code id: tags:
+
+``` python
+score = model.evaluate(x_test, y_test, verbose=0)
+
+print(f'Test loss     : {score[0]:4.4f}')
+print(f'Test accuracy : {score[1]:4.4f}')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.2 - Plot history
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.history(history, figsize=(6,4), save_as='03-history')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.3 - Plot results
+
+%% Cell type:code id: tags:
+
+``` python
+#y_pred   = model.predict_classes(x_test)           Deprecated after 01/01/2021 !!
+
+y_sigmoid = model.predict(x_test, verbose=fit_verbosity)
+y_pred    = np.argmax(y_sigmoid, axis=-1)
+
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id: tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id: tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.Lightning/01-DNN-MNIST_Lightning.ipynb

+%% Cell type:markdown id:86fe2213-fb44-4bd4-a371-a541cba6a744 tags:
+
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [LMNIST1] - Simple classification with DNN
+<!-- DESC --> An example of classification using a dense neural network for the famous MNIST dataset, using PyTorch Lightning
+<!-- AUTHOR : MBOGOL Touye Achille (AI/ML Engineer EFELIA-MIAI/SIMAP Lab) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with pytorch lightning
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id:7f16101a-6612-4e02-93e9-c45ce1ac911d tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id:743c77d3-0983-491c-90be-ef2219861a47 tags:
+
+``` python
+import pandas as pd
+import numpy as np
+
+import torch
+import torch.nn as nn
+import lightning.pytorch as pl
+import torch.nn.functional as F
+import torchvision.transforms as T
+
+import sys,os
+import multiprocessing
+
+from torchvision import datasets
+from torchmetrics.functional import accuracy
+from torch.utils.data import Dataset, DataLoader
+from lightning.pytorch import loggers as pl_loggers
+from modules.progressbar import CustomTrainProgressBar
+from lightning.pytorch.loggers.tensorboard import TensorBoardLogger
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('LMNIST1')
+```
+
+%% Cell type:markdown id:df10dcda-aa63-476b-8665-9b1610fe51c6 tags:
+
+## Step 2 Retrieve data
+
+MNIST is one of the most famous historic dataset include in torchvision Datasets. `torchvision` provides many built-in datasets in the `torchvision.datasets`.
+
+%% Cell type:code id:6668e50c-f0c6-43cf-b733-9ac29d6a3900 tags:
+
+``` python
+# Load data sets
+train_dataset = datasets.MNIST(root="data", train=True,  download=True, transform=None)
+
+test_dataset  = datasets.MNIST(root="data", train=False, download=True, transform=None)
+```
+
+%% Cell type:code id:b543b885-6336-461d-abbe-6d3171405771 tags:
+
+``` python
+# print info for train data
+print(train_dataset)
+
+print()
+
+# print info for test data
+print(test_dataset)
+```
+
+%% Cell type:code id:44a489f5-3e53-4a2b-8069-f265b2814dc0 tags:
+
+``` python
+# See the shape of train data and test data
+print("x_train : ",train_dataset.data.shape)
+print("y_train : ",train_dataset.targets.shape)
+
+print()
+
+print("x_test  : ",test_dataset.data.shape)
+print("y_test  : ",test_dataset.targets.shape)
+
+print()
+
+# print number of labels or class
+print("Number of Targets :",len(np.unique(train_dataset.targets)))
+print("Targets Values :",    np.unique(train_dataset.targets))
+
+
+print("\nRemark that we work with torch tensors and not numpy array, not tensorflow tensor")
+print(" -> x_train.dtype = ",train_dataset.data.dtype)
+print(" -> y_train.dtype = ",train_dataset.targets.dtype)
+```
+
+%% Cell type:markdown id:b418adb7-33ea-450c-9793-3cdce5d5fa8c tags:
+
+## Step 3 -  Preparing your data for training with DataLoaders
+The Dataset retrieves our dataset’s features and labels one sample at a time. While training a model, we typically want to pass samples in `minibatches`, reshuffle the data at every epoch to reduce model overfitting, and use Python’s multiprocessing to speed up data retrieval. DataLoader is an iterable that abstracts this complexity for us in an easy API.
+
+%% Cell type:code id:8af0bc4c-acb3-46d9-aae2-143b0327d970 tags:
+
+``` python
+# Before normalization:
+x_train=train_dataset.data
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+#  After normalization:
+## T.Compose creates a pipeline where the provided transformations are run in sequence
+transforms = T.Compose(
+    [
+        # This transforms takes a np.array or a PIL image of integers
+        # in the range 0-255 and transforms it to a float tensor in the
+        # range 0.0 - 1.0
+        T.ToTensor()
+
+    ]
+)
+
+
+train_dataset = datasets.MNIST(root="data", train=True, download=True,  transform=transforms)
+test_dataset  = datasets.MNIST(root="data", train=False, download=True, transform=transforms)
+
+
+# print image and label After normalization.
+## iter() followed by next() is used to get some images and label.
+image,label=next(iter(train_dataset))
+print('After normalization  : Min={}, max={}'.format(image.min(),image.max()))
+```
+
+%% Cell type:markdown id:35d50a57-8274-4660-8765-d0f2bf7214bd tags:
+
+### Have a look
+
+%% Cell type:code id:a172ebc5-8858-4f30-8e2c-1e9c123ae0ee tags:
+
+``` python
+x_train=train_dataset.data
+y_train=train_dataset.targets
+```
+
+%% Cell type:code id:5a487760-b43a-4f7c-bfd8-1ce2c9652769 tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5, y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:code id:ca0a63ae-e6d6-4940-b8ff-9b11cb2737bb tags:
+
+``` python
+# train bacth data
+train_loader= DataLoader(
+  dataset=train_dataset,
+  shuffle=True,
+  batch_size=512,
+  num_workers=2
+)
+
+
+# test batch data
+test_loader= DataLoader(
+  dataset=test_dataset,
+  shuffle=False,
+  batch_size=512,
+  num_workers=2
+)
+
+# print image and label after normalization.
+image, label=next(iter(train_loader))
+print('Shape of first training data batch after use pytorch dataloader :\nbatch images = {} \nbatch labels = {}'.format(image.shape,label.shape))
+```
+
+%% Cell type:markdown id:51bf21ee-76ca-42fa-b67f-066dbd239a72 tags:
+
+## Step 4 - Create Model
+About informations about :
+ - [Optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)
+ - [Activation](https://www.tensorflow.org/api_docs/python/tf/keras/activations)
+ - [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses)
+ - [Metrics](https://www.tensorflow.org/api_docs/python/tf/keras/metrics)
+
+ `Note :` PyTorch provides losses such as the cross-entropy loss (`nn.CrossEntropyLoss`) usually use for classification problem. we're using the softmax function to predict class probabilities. With a softmax output, you want to use cross-entropy as the loss. To actually calculate the loss, we need to pass in the raw output of our network into the loss, not the output of the softmax function. This raw output is usually called the *logits* or *scores*. because in pytorch the cross entropy contain softmax function already.
+
+%% Cell type:code id:16701119-71eb-4f59-a50a-f153b07a74ae tags:
+
+``` python
+class MyNet(nn.Module):
+
+    def __init__(self,num_class=10):
+        super().__init__()
+        self.num_class = num_class
+        self.model     = nn.Sequential(
+
+            # Input vector:
+            nn.Flatten(),                    # convert each 2D 28x28 image into a contiguous array of 784 pixel values
+
+            # first hidden layer
+            nn.Linear(in_features=1*28*28, out_features=100),
+            nn.ReLU(),
+            nn.Dropout1d(0.1),               # Combat overfitting
+
+            # second hidden layer
+            nn.Linear(in_features=100, out_features=100),
+            nn.ReLU(),
+            nn.Dropout1d(0.1),               # Combat overfitting
+
+            # logits outpout
+            nn.Linear(100, num_class)
+        )
+
+
+
+    # forward pass
+    def forward(self, x):
+        return self.model(x)
+
+```
+
+%% Cell type:code id:37abf99b-f8ec-4048-a65d-f173ee18b234 tags:
+
+``` python
+class LitModel(pl.LightningModule):
+
+    def __init__(self, MyNet):
+        super().__init__()
+        self.MyNet = MyNet
+
+    # forward pass
+    def forward(self, x):
+        return self.MyNet(x)
+
+
+    def configure_optimizers(self):
+        # optimizer
+        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
+        return optimizer
+
+
+    def training_step(self, batch, batch_idx):
+        # defines the train loop
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        # computes the cross entropy loss between input logits and target
+        loss  = F.cross_entropy(y_hat, y)
+
+        # accuracy metrics
+        acc   = accuracy(y_hat, y,task="multiclass",num_classes=10)
+
+        metrics = {"train_loss": loss,
+                   "train_acc" : acc
+                  }
+
+        # logs metrics for each training_step
+        self.log_dict(metrics,
+                      on_step  = False,
+                      on_epoch = True,
+                      prog_bar = True,
+                      logger   = True
+                     )
+
+        return loss
+
+
+    def validation_step(self, batch, batch_idx):
+        #  defines the valid loop.
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        # computes the cross entropy loss between input logits and target
+        loss  = F.cross_entropy(y_hat, y)
+
+        # accuracy metrics
+        acc   = accuracy(y_hat, y,task="multiclass",num_classes=10)
+
+        metrics = {"test_loss": loss,
+                   "test_acc": acc
+                  }
+
+        # logs metrics for each validation_step
+        self.log_dict(metrics,
+                      on_step  = False,
+                      on_epoch = True,
+                      prog_bar = True,
+                      logger   = True
+                     )
+
+        return metrics
+
+
+    def predict_step(self, batch, batch_idx):
+        # defnie the predict loop
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        return y_hat
+
+```
+
+%% Cell type:code id:7546b27e-d492-420a-8d5d-109201b47830 tags:
+
+``` python
+# print summary model
+model=LitModel(MyNet())
+print(model)
+```
+
+%% Cell type:markdown id:fb32e85d-bd92-4ca5-a3dc-ddb5ed50ba6b tags:
+
+## Step 5 - Train Model
+
+%% Cell type:code id:96f0e087-f21a-4afc-85c5-3a3c0c353fe1 tags:
+
+``` python
+# loggers data
+os.makedirs(f'{run_dir}/logs',   mode=0o750, exist_ok=True)
+logger= TensorBoardLogger(save_dir=f'{run_dir}/logs',name="DNN_logs")
+```
+
+%% Cell type:code id:ce975c03-d05d-40c4-92ff-0cc90699c13e tags:
+
+``` python
+# train model
+# trainer= pl.Trainer(accelerator='auto',
+#                     max_epochs=20,
+#                     logger=logger,
+#                     num_sanity_val_steps=0,
+#                     callbacks=[CustomTrainProgressBar()]
+#                    )
+
+trainer= pl.Trainer(accelerator='auto',
+                    max_epochs=20,
+                    logger=logger,
+                    num_sanity_val_steps=0
+                   )
+
+trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=test_loader,)
+```
+
+%% Cell type:markdown id:a1191f05-4454-415c-a5ed-e63d9ae56651 tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+
+%% Cell type:code id:9f45316e-0d2d-4fc1-b9a8-5fb8aaf5586a tags:
+
+``` python
+score=trainer.validate(model=model,dataloaders=test_loader, verbose=False)
+
+print('x_test / acc      : {:5.4f}'.format(score[0]['test_acc']))
+print('x_test / loss     : {:5.4f}'.format(score[0]['test_loss']))
+```
+
+%% Cell type:markdown id:e352e48d-b473-4162-a1aa-72d6d4f7aa38 tags:
+
+## 6.2 - Plot history
+
+To access logs with tensorboad :
+- Under **Docker**, from a terminal launched via the jupyterlab launcher, use the following command:<br>
+```tensorboard --logdir <path-to-logs> --host 0.0.0.0```
+- If you're **not using Docker**, from a terminal :<br>
+```tensorboard --logdir <path-to-logs>```
+
+**Note:** One tensorboard instance can be used simultaneously.
+
+%% Cell type:markdown id:f00ded6b-a7db-4c5d-b1b2-72264db20bdb tags:
+
+###  6.3 - Plot results
+
+%% Cell type:code id:e387a70d-9c23-4d16-8ef7-879aec7791e2 tags:
+
+``` python
+# logits outpout by batch size
+y_logits=trainer.predict(model=model,dataloaders=test_loader)
+
+# Concat into single tensor
+y_logits=torch.cat(y_logits)
+
+# output probabilities values
+y_pred_values=F.softmax(y_logits,dim=1)
+
+# Returns the indices of the maximum output probabilities values
+y_pred=torch.argmax(y_pred_values,dim=-1).numpy()
+```
+
+%% Cell type:code id:fb2b2eeb-fcd8-453c-93ef-59a960a8bbd5 tags:
+
+``` python
+x_test=test_dataset.data
+y_test=test_dataset.targets
+```
+
+%% Cell type:code id:71187fa9-2ad3-4b23-94b9-1846045bd070 tags:
+
+``` python
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id:2fc7b2b9-9115-4848-9aae-2798bf7aa79a tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id:e55f17c4-fce7-423a-9adf-f2511c534ef5 tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id:fea1b396-70ca-4b00-851d-0538a4b347fb tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id:e982c032-cce8-4c71-8cdc-2af4b31b2914 tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id:233838c2-c97f-4489-8c79-9247d7b7456b tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id:51b87aa0-d4e9-48bb-8205-4b583f4b0b61 tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.Lightning/02-CNN-MNIST_Lightning.ipynb

+%% Cell type:markdown id:86fe2213-fb44-4bd4-a371-a541cba6a744 tags:
+
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+## <!-- TITLE --> [LMNIST2] - Simple classification with CNN
+<!-- DESC --> An example of classification using a convolutional neural network for the famous MNIST dataset, using PyTorch Lightning
+<!-- AUTHOR : MBOGOL Touye Achille (AI/ML Engineer MIAI/SIMaP) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with pytorch lightning
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id:7f16101a-6612-4e02-93e9-c45ce1ac911d tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id:743c77d3-0983-491c-90be-ef2219861a47 tags:
+
+``` python
+import pandas as pd
+import numpy as np
+
+import torch
+import torch.nn as nn
+import lightning.pytorch as pl
+import torch.nn.functional as F
+import torchvision.transforms as T
+
+import sys,os
+import multiprocessing
+import matplotlib.pyplot as plt
+
+from torchvision import datasets
+from torchmetrics.functional import accuracy
+from torch.utils.data import Dataset, DataLoader
+from modules.progressbar import CustomTrainProgressBar
+from lightning.pytorch.loggers import TensorBoardLogger
+
+
+# Init Fidle environment
+import fidle
+
+run_id, run_dir, datasets_dir = fidle.init('LMNIST2')
+```
+
+%% Cell type:markdown id:df10dcda-aa63-476b-8665-9b1610fe51c6 tags:
+
+## Step 2 Retrieve data
+
+MNIST is one of the most famous historic dataset include in torchvision Datasets. `torchvision` provides many built-in datasets in the `torchvision.datasets`.
+
+%% Cell type:code id:6668e50c-f0c6-43cf-b733-9ac29d6a3900 tags:
+
+``` python
+# Load data sets
+train_dataset = datasets.MNIST(root="data", train=True, download=True, transform=None)
+
+test_dataset= datasets.MNIST(root="data", train=False, download=False, transform=None)
+```
+
+%% Cell type:code id:a14d6fc2-b913-4eaa-9cde-5ca6785bfa12 tags:
+
+``` python
+# print info for train data
+print(train_dataset)
+
+print()
+
+# print info for test data
+print(test_dataset)
+```
+
+%% Cell type:code id:44a489f5-3e53-4a2b-8069-f265b2814dc0 tags:
+
+``` python
+# See the shape of train data and test data
+print("x_train : ",train_dataset.data.shape)
+print("y_train : ",train_dataset.targets.shape)
+
+print()
+
+print("x_test  : ",test_dataset.data.shape)
+print("y_test  : ",test_dataset.targets.shape)
+
+print()
+
+# print number of targets and  values targets
+print("Number of Targets :",len(np.unique(train_dataset.targets)))
+print("Targets Values    :",    np.unique(train_dataset.targets))
+
+print()
+
+print("Remark that we work with torch tensors and not numpy array, not tensorflow tensor")
+print(" -> x_train.dtype = ",train_dataset.data.dtype)
+print(" -> y_train.dtype = ",train_dataset.targets.dtype)
+```
+
+%% Cell type:markdown id:b418adb7-33ea-450c-9793-3cdce5d5fa8c tags:
+
+## Step 3 -  Preparing your data for training with DataLoaders
+The Dataset retrieves our dataset’s features and labels one sample at a time. While training a model, we typically want to pass samples in `minibatches`, reshuffle the data at every epoch to reduce model overfitting, and use Python’s multiprocessing to speed up data retrieval. DataLoader is an iterable that abstracts this complexity for us in an easy API.
+
+%% Cell type:code id:8af0bc4c-acb3-46d9-aae2-143b0327d970 tags:
+
+``` python
+# Before normalization:
+x_train=train_dataset.data
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+# After normalization:
+## T.Compose creates a pipeline where the provided transformations are run in sequence
+transforms = T.Compose(
+    [
+        # This transforms takes a np.array or a PIL image of integers
+        # in the range 0-255 and transforms it to a float tensor in the
+        # range 0.0 - 1.0
+        T.ToTensor(),
+
+    ]
+)
+
+
+train_dataset = datasets.MNIST(root="data", train=True, download=True, transform=transforms)
+test_dataset= datasets.MNIST(root="data", train=False, download=True, transform=transforms)
+
+
+# print image and label After normalization.
+# iter() followed by next() is used to get some images and label
+image,label=next(iter(train_dataset))
+print('After normalization  : Min={}, max={}'.format(image.min(),image.max()))
+```
+
+%% Cell type:markdown id:35d50a57-8274-4660-8765-d0f2bf7214bd tags:
+
+### Have a look
+
+%% Cell type:code id:a172ebc5-8858-4f30-8e2c-1e9c123ae0ee tags:
+
+``` python
+x_train=train_dataset.data
+y_train=train_dataset.targets
+```
+
+%% Cell type:code id:5a487760-b43a-4f7c-bfd8-1ce2c9652769 tags:
+
+``` python
+fidle.scrawler.images(x_train, y_train, [27],  x_size=5,y_size=5, colorbar=True, save_as='01-one-digit')
+fidle.scrawler.images(x_train, y_train, range(5,41), columns=12, save_as='02-many-digits')
+```
+
+%% Cell type:code id:ca0a63ae-e6d6-4940-b8ff-9b11cb2737bb tags:
+
+``` python
+# train bacth data
+train_loader= DataLoader(
+  dataset=train_dataset,
+  shuffle=True,
+  batch_size=512,
+  num_workers=2
+)
+
+# test batch data
+test_loader= DataLoader(
+  dataset=test_dataset,
+  shuffle=False,
+  batch_size=512,
+  num_workers=2
+)
+
+# print image and label  After normalization and batch_size.
+image, label=next(iter(train_loader))
+print('Shape of first training data batch after use pytorch dataloader :\nbatch images = {} \nbatch labels = {}'.format(image.shape,label.shape))
+```
+
+%% Cell type:markdown id:51bf21ee-76ca-42fa-b67f-066dbd239a72 tags:
+
+## Step 4 - Create Model
+About informations about :
+ - [Optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)
+ - [Activation](https://www.tensorflow.org/api_docs/python/tf/keras/activations)
+ - [Loss](https://www.tensorflow.org/api_docs/python/tf/keras/losses)
+ - [Metrics](https://www.tensorflow.org/api_docs/python/tf/keras/metrics)
+
+ `Note :` PyTorch provides losses such as the cross-entropy loss (`nn.CrossEntropyLoss`) usually use for classification problem. we're using the softmax function to predict class probabilities. With a softmax output, you want to use cross-entropy as the loss. To actually calculate the loss, we need to pass in the raw output of our network into the loss, not the output of the softmax function. This raw output is usually called the *logits* or *scores*. because in pytorch the cross entropy contain softmax function already.
+
+%% Cell type:code id:16701119-71eb-4f59-a50a-f153b07a74ae tags:
+
+``` python
+class MyNet(nn.Module):
+
+    def __init__(self,num_class=10):
+        super().__init__()
+        self.num_class=num_class
+        self.model = nn.Sequential(
+
+            # first convolution
+            nn.Conv2d(in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=0),
+            nn.ReLU(),
+            nn.MaxPool2d((2,2)),
+            nn.Dropout2d(0.1),            # Combat overfitting
+
+            # second convolution
+            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=3, stride=1, padding=0),
+            nn.ReLU(),
+            nn.MaxPool2d((2,2)),
+            nn.Dropout2d(0.1),            # Combat overfitting
+
+            nn.Flatten(),                 # convert feature map into feature vectors
+
+            # MLP network
+            nn.Linear(16*5*5,100),
+            nn.ReLU(),
+            nn.Dropout1d(0.1),            # Combat overfitting
+
+            nn.Linear(100, num_class),    # logits outpout
+        )
+
+    def forward(self, x):
+        x=self.model(x)                   # forward pass
+        return x
+```
+
+%% Cell type:code id:37abf99b-f8ec-4048-a65d-f173ee18b234 tags:
+
+``` python
+class LitModel(pl.LightningModule):
+
+    def __init__(self, MyNet):
+        super().__init__()
+        self.MyNet = MyNet
+
+    # forward pass
+    def forward(self, x):
+        return self.MyNet(x)
+
+
+     # optimizer
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
+        return optimizer
+
+
+    def training_step(self, batch, batch_idx):
+        # defines the train loop
+        x, y  = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        # loss function
+        loss= F.cross_entropy(y_hat, y)
+
+        # metrics accuracy
+        acc=accuracy(y_hat, y,task="multiclass",num_classes=10)
+
+        metrics = {"train_loss": loss, "train_acc": acc}
+
+        # logs metrics for each training_step
+        self.log_dict(metrics,
+                      on_step=False ,
+                      on_epoch=True,
+                      prog_bar=True,
+                      logger=True)
+
+        return loss
+
+
+    def validation_step(self, batch, batch_idx):
+        # defines the valid loop.
+        x, y = batch
+
+        # forward pass
+        y_hat= self.MyNet(x)
+
+        # loss function MSE
+        loss = F.cross_entropy(y_hat, y)
+
+        # metrics accuracy
+        acc=accuracy(y_hat, y, task="multiclass", num_classes=10)
+
+
+        metrics = {"test_loss": loss, "test_acc": acc}
+
+        # logs metrics for each validation_step
+        self.log_dict(metrics,
+                      on_step  = False,
+                      on_epoch = True,
+                      prog_bar = True,
+                      logger   = True
+                     )
+
+
+        return metrics
+
+
+
+    def predict_step(self, batch, batch_idx):
+        # defnie the predict loop
+        x, y = batch
+
+        # forward pass
+        y_hat = self.MyNet(x)
+
+        return y_hat
+```
+
+%% Cell type:code id:489af62f-8f7c-4d1b-a6d0-5a0417e79869 tags:
+
+``` python
+# print summary model
+model=LitModel(MyNet())
+print(model)
+```
+
+%% Cell type:markdown id:fb32e85d-bd92-4ca5-a3dc-ddb5ed50ba6b tags:
+
+## Step 5 - Train Model
+
+%% Cell type:code id:756d5e19-6a10-42b8-8971-411389f7d19c tags:
+
+``` python
+# loggers data
+os.makedirs(f'{run_dir}/logs',   mode=0o750, exist_ok=True)
+logger= TensorBoardLogger(save_dir=f'{run_dir}/logs',name="CNN_logs")
+```
+
+%% Cell type:code id:ce975c03-d05d-40c4-92ff-0cc90699c13e tags:
+
+``` python
+# train model
+# trainer = pl.Trainer(accelerator='auto',
+#                      max_epochs=16,
+#                      logger=logger,
+#                      num_sanity_val_steps=0,
+#                      callbacks=[CustomTrainProgressBar()]
+#                     )
+
+trainer = pl.Trainer(accelerator='auto',
+                     max_epochs=16,
+                     logger=logger,
+                     num_sanity_val_steps=0
+                    )
+
+trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=test_loader)
+```
+
+%% Cell type:markdown id:a1191f05-4454-415c-a5ed-e63d9ae56651 tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+Note : With a DNN, we had a precision of the order of : 97.7%
+
+%% Cell type:code id:9f45316e-0d2d-4fc1-b9a8-5fb8aaf5586a tags:
+
+``` python
+# evaluate your model
+score=trainer.validate(model=model,dataloaders=test_loader, verbose=False)
+
+print('x_test / acc      : {:5.4f}'.format(score[0]['test_acc']))
+print('x_test / loss     : {:5.4f}'.format(score[0]['test_loss']))
+```
+
+%% Cell type:code id:5cfe9bd6-654b-42e0-b430-5f3b816526b0 tags:
+
+``` python
+score=trainer.validate(model=model,dataloaders=test_loader, verbose=False)
+```
+
+%% Cell type:markdown id:e352e48d-b473-4162-a1aa-72d6d4f7aa38 tags:
+
+## 6.2 - Plot history
+To access logs with tensorboad :
+- Under **Docker**, from a terminal launched via the jupyterlab launcher, use the following command:<br>
+```tensorboard --logdir <path-to-logs> --host 0.0.0.0```
+- If you're **not using Docker**, from a terminal :<br>
+```tensorboard --logdir <path-to-logs>```
+
+**Note:** One tensorboard instance can be used simultaneously.
+
+%% Cell type:markdown id:f00ded6b-a7db-4c5d-b1b2-72264db20bdb tags:
+
+###  6.3 - Plot results
+
+%% Cell type:code id:e387a70d-9c23-4d16-8ef7-879aec7791e2 tags:
+
+``` python
+# logits outpout by batch size
+y_logits= trainer.predict(model=model,dataloaders=test_loader)
+
+# Concat into single tensor
+y_logits= torch.cat(y_logits)
+
+# output probabilities values
+y_pred_values=F.softmax(y_logits,dim=1)
+
+# Returns the indices of the maximum output probabilities values
+y_pred=torch.argmax(y_pred_values,dim=-1)
+```
+
+%% Cell type:code id:fb2b2eeb-fcd8-453c-93ef-59a960a8bbd5 tags:
+
+``` python
+x_test=test_dataset.data
+y_test=test_dataset.targets
+```
+
+%% Cell type:code id:71187fa9-2ad3-4b23-94b9-1846045bd070 tags:
+
+``` python
+fidle.scrawler.images(x_test, y_test, range(0,200), columns=12, x_size=1, y_size=1, y_pred=y_pred, save_as='04-predictions')
+```
+
+%% Cell type:markdown id:2fc7b2b9-9115-4848-9aae-2798bf7aa79a tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id:e55f17c4-fce7-423a-9adf-f2511c534ef5 tags:
+
+``` python
+errors=[ i for i in range(len(x_test)) if y_pred[i]!=y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(x_test, y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=y_pred, save_as='05-some-errors')
+```
+
+%% Cell type:code id:fea1b396-70ca-4b00-851d-0538a4b347fb tags:
+
+``` python
+fidle.scrawler.confusion_matrix(y_test,y_pred,range(10),normalize=True, save_as='06-confusion-matrix')
+```
+
+%% Cell type:code id:e982c032-cce8-4c71-8cdc-2af4b31b2914 tags:
+
+``` python
+fidle.end()
+```
+
+%% Cell type:markdown id:233838c2-c97f-4489-8c79-9247d7b7456b tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id:51b87aa0-d4e9-48bb-8205-4b583f4b0b61 tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.Lightning/modules/progressbar.py

+# ------------------------------------------------------------------
+#     _____ _     _ _
+#    |  ___(_) __| | | ___
+#    | |_  | |/ _` | |/ _ \
+#    |  _| | | (_| | |  __/
+#    |_|   |_|\__,_|_|\___|
+# ------------------------------------------------------------------
+# Formation Introduction au Deep Learning  (FIDLE)
+# CNRS/SARI/DEVLOG 2023 
+# ------------------------------------------------------------------
+# 2.0 version by Achille Mbogol Touye (EFELIA-MIAI/SIMAP¨), sep 2023
+
+from tqdm import tqdm as _tqdm
+from lightning.pytorch.callbacks import TQDMProgressBar
+
+# Créez un callback de barre de progression pour afficher les métriques d'entraînement
+class CustomTrainProgressBar(TQDMProgressBar):
+    def __init__(self):
+        super().__init__()
+        self._val_progress_bar     = _tqdm()
+        self._predict_progress_bar = _tqdm()
+        
+    def init_predict_tqdm(self):
+        bar=super().init_test_tqdm()
+        bar.set_description("Predicting")
+        return bar
+
+    def init_train_tqdm(self):
+        bar=super().init_train_tqdm()
+        bar.set_description("Training")
+        return bar    
+
+    @property
+    def val_progress_bar(self):
+        if self._val_progress_bar is None:
+            raise ValueError("The `_val_progress_bar` reference has not been set yet.")
+        return self._val_progress_bar
+
+    @property
+    def predict_progress_bar(self) -> _tqdm:
+        if self._predict_progress_bar is None:
+            raise TypeError(f"The `{self.__class__.__name__}._predict_progress_bar` reference has not been set yet.")
+        return self._predict_progress_bar    
+    
+
+    def on_validation_start(self, trainer, pl_module):
+        # Désactivez l'affichage de la barre de progression de validation
+        self.val_progress_bar.disable = True  
+
+    def on_predict_start(self, trainer, pl_module):
+        # Désactivez l'affichage de la barre de progression de validation
+        self.predict_progress_bar.disable = True 
\ No newline at end of file

MNIST.PyTorch/01-DNN-MNIST_PyTorch.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [PMNIST1] - Simple classification with DNN
+<!-- DESC -->Example of classification with a fully connected neural network, using Pytorch
+<!-- AUTHOR : Laurent Risser (CNRS/IMT) -->
+
+## Objectives :
+ - Recognizing handwritten numbers
+ - Understanding the principle of a classifier DNN network
+ - Implementation with PyTorch
+
+
+The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) (Modified National Institute of Standards and Technology) is a must for Deep Learning.
+It consists of 60,000 small images of handwritten numbers for learning and 10,000 for testing.
+
+
+## What we're going to do :
+
+ - Retrieve data
+ - Preparing the data
+ - Create a model
+ - Train the model
+ - Evaluate the result
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Init python stuff
+
+%% Cell type:code id: tags:
+
+``` python
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+import torchvision  #to get the MNIST dataset
+
+
+import numpy as np
+import matplotlib.pyplot as plt
+import sys,os
+
+import fidle
+from modules.fidle_pwk_additional import convergence_history_CrossEntropyLoss
+
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('PMNIST1')
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Retrieve data
+MNIST is one of the most famous historic dataset.
+Include in [torchvision datasets](https://pytorch.org/vision/stable/datasets.html)
+
+%% Cell type:code id: tags:
+
+``` python
+#get and format the training set
+mnist_trainset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=None)
+x_train=mnist_trainset.data.type(torch.DoubleTensor)
+y_train=mnist_trainset.targets
+
+
+#get and format the test set
+mnist_testset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=None)
+x_test=mnist_testset.data.type(torch.DoubleTensor)
+y_test=mnist_testset.targets
+
+#check data shape and format
+print("Size of the train and test observations")
+print(" -> x_train : ",x_train.shape)
+print(" -> y_train : ",y_train.shape)
+print(" -> x_test  : ",x_test.shape)
+print(" -> y_test  : ",y_test.shape)
+
+print("\nRemark that we work with torch tensors and not numpy arrays:")
+print(" -> x_train.dtype = ",x_train.dtype)
+print(" -> y_train.dtype = ",y_train.dtype)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 - Preparing the data
+
+%% Cell type:code id: tags:
+
+``` python
+print('Before normalization : Min={}, max={}'.format(x_train.min(),x_train.max()))
+
+xmax=x_train.max()
+x_train = x_train / xmax
+x_test  = x_test  / xmax
+
+print('After normalization  : Min={}, max={}'.format(x_train.min(),x_train.max()))
+```
+
+%% Cell type:markdown id: tags:
+
+### Have a look
+
+%% Cell type:code id: tags:
+
+``` python
+np_x_train=x_train.numpy().astype(np.float64)
+np_y_train=y_train.numpy().astype(np.uint8)
+
+fidle.scrawler.images(np_x_train,np_y_train , [27],  x_size=5,y_size=5, colorbar=True)
+fidle.scrawler.images(np_x_train,np_y_train, range(5,41), columns=12)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Create model
+About informations about :
+ - [Optimizer](https://pytorch.org/docs/stable/optim.html)
+ - [Basic neural-network blocks](https://pytorch.org/docs/stable/nn.html)
+ - [Loss](https://pytorch.org/docs/stable/nn.html#loss-functions)
+
+%% Cell type:code id: tags:
+
+``` python
+class MyModel(nn.Module):
+    """
+    Basic fully connected neural-network
+    """
+    def __init__(self):
+        hidden1     = 100
+        hidden2     = 100
+        super(MyModel, self).__init__()
+        self.hidden1 = nn.Linear(784, hidden1)
+        self.hidden2 = nn.Linear(hidden1, hidden2)
+        self.hidden3 = nn.Linear(hidden2, 10)
+
+    def forward(self, x):
+        x = x.view(-1,784)   #flatten the images before using fully-connected layers
+        x = self.hidden1(x)
+        x = F.relu(x)
+        x = self.hidden2(x)
+        x = F.relu(x)
+        x = self.hidden3(x)
+        x = F.softmax(x, dim=0)
+        return x
+
+
+
+model = MyModel()
+
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 5 - Train the model
+
+### 5.1 - Stochastic gradient descent strategy to fit the model
+
+%% Cell type:code id: tags:
+
+``` python
+def fit(model,X_train,Y_train,X_test,Y_test, EPOCHS = 5, BATCH_SIZE = 32):
+
+    loss = nn.CrossEntropyLoss()
+    optimizer = torch.optim.Adam(model.parameters(),lr=1e-3) #lr is the learning rate
+    model.train()
+
+    history=convergence_history_CrossEntropyLoss()
+
+    history.update(model,X_train,Y_train,X_test,Y_test)
+
+    n=X_train.shape[0] #number of observations in the training data
+
+    #stochastic gradient descent
+    for epoch in range(EPOCHS):
+
+        batch_start=0
+        epoch_shuffler=np.arange(n)
+        np.random.shuffle(epoch_shuffler) #remark that 'utilsData.DataLoader' could be used instead
+
+        while batch_start+BATCH_SIZE < n:
+            #get mini-batch observation
+            mini_batch_observations = epoch_shuffler[batch_start:batch_start+BATCH_SIZE]
+            var_X_batch = Variable(X_train[mini_batch_observations,:,:]).float() #the input image is flattened
+            var_Y_batch = Variable(Y_train[mini_batch_observations])
+
+            #gradient descent step
+            optimizer.zero_grad()               #set the parameters gradients to 0
+            Y_pred_batch = model(var_X_batch)   #predict y with the current NN parameters
+
+            curr_loss = loss(Y_pred_batch, var_Y_batch)  #compute the current loss
+            curr_loss.backward()                         #compute the loss gradient w.r.t. all NN parameters
+            optimizer.step()                             #update the NN parameters
+
+            #prepare the next mini-batch of the epoch
+            batch_start+=BATCH_SIZE
+
+        history.update(model,X_train,Y_train,X_test,Y_test)
+
+    return history
+
+```
+
+%% Cell type:markdown id: tags:
+
+### 5.2 - Fit the model
+
+%% Cell type:code id: tags:
+
+``` python
+
+batch_size  = 512
+epochs      = 128
+
+
+history=fit(model,x_train,y_train,x_test,y_test,EPOCHS=epochs,BATCH_SIZE = batch_size)
+
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 6 - Evaluate
+### 6.1 - Final loss and accuracy
+
+%% Cell type:code id: tags:
+
+``` python
+var_x_test = Variable(x_test[:,:,:]).float()
+var_y_test = Variable(y_test[:])
+y_pred = model(var_x_test)
+
+loss = nn.CrossEntropyLoss()
+curr_loss = loss(y_pred, var_y_test)
+
+val_loss = curr_loss.item()
+val_accuracy  = float( (torch.argmax(y_pred, dim= 1) == var_y_test).float().mean() )
+
+
+print('Test loss     :', val_loss)
+print('Test accuracy :', val_accuracy)
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.2 - Plot history
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.history(history, figsize=(6,4))
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.3 - Plot results
+
+%% Cell type:code id: tags:
+
+``` python
+y_pred = model(var_x_test)
+np_y_pred_label = torch.argmax(y_pred, dim= 1).numpy().astype(np.uint8)
+
+np_x_test=x_test.numpy().astype(np.float64)
+np_y_test=y_test.numpy().astype(np.uint8)
+
+fidle.scrawler.images(np_x_test, np_y_test, range(0,60), columns=12, x_size=1, y_size=1, y_pred=np_y_pred_label)
+```
+
+%% Cell type:markdown id: tags:
+
+### 6.4 - Plot some errors
+
+%% Cell type:code id: tags:
+
+``` python
+errors=[ i for i in range(len(np_y_test)) if np_y_pred_label[i]!=np_y_test[i] ]
+errors=errors[:min(24,len(errors))]
+fidle.scrawler.images(np_x_test, np_y_test, errors[:15], columns=6, x_size=2, y_size=2, y_pred=np_y_pred_label)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+fidle.scrawler.confusion_matrix(np_y_test,np_y_pred_label, range(10))
+```
+
+%% Cell type:markdown id: tags:
+
+<div class="todo">
+    A few things you can do for fun:
+    <ul>
+        <li>Changing the network architecture (layers, number of neurons, etc.)</li>
+        <li>Display a summary of the network</li>
+        <li>Retrieve and display the softmax output of the network, to evaluate its "doubts".</li>
+    </ul>
+</div>
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

MNIST.PyTorch/modules/fidle_pwk_additional.py

+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+import numpy as np
+
+
+class convergence_history_CrossEntropyLoss:
+  def __init__(self):
+    """
+    Class to save the training converge properties
+    """
+    self.loss=nn.CrossEntropyLoss()
+    self.history={}                #Save convergence measures in the end of each epoch
+    self.history['loss']=[]        #value of the cost function on training data
+    self.history['accuracy']=[]         #percentage of  correctly classified instances on training data (if classification)
+    self.history['val_loss']=[]    #value of the cost function on validation data
+    self.history['val_accuracy']=[]     #percentage of  correctly classified instances on validation data (if classification)
+  
+  def update(self,current_model,xtrain,ytrain,xtest,ytest):
+    
+    #convergence information on the training data 
+    nb_training_obs=xtrain.shape[0]
+    if nb_training_obs>xtest.shape[0]:
+        nb_training_obs=xtest.shape[0]
+        
+    epoch_shuffler=np.arange(xtrain.shape[0]) 
+    np.random.shuffle(epoch_shuffler)
+    mini_batch_observations = epoch_shuffler[:nb_training_obs]
+    var_X_batch = Variable(xtrain[mini_batch_observations,:]).float()
+    var_y_batch = Variable(ytrain[mini_batch_observations])
+    y_pred_batch = current_model(var_X_batch)
+    curr_loss = self.loss(y_pred_batch, var_y_batch)
+
+    self.history['loss'].append(curr_loss.item())
+    self.history['accuracy'].append( float( (torch.argmax(y_pred_batch, dim= 1) == var_y_batch).float().mean()) )
+    
+    #convergence information on the test data 
+    var_X_batch = Variable(xtest[:,:]).float()
+    var_y_batch = Variable(ytest[:])
+    y_pred_batch = current_model(var_X_batch)
+    curr_loss = self.loss(y_pred_batch, var_y_batch)
+
+    self.history['val_loss'].append(curr_loss.item())
+    self.history['val_accuracy'].append( float( (torch.argmax(y_pred_batch, dim= 1) == var_y_batch).float().mean()) )
+
+
+
+class convergence_history_MSELoss:
+  def __init__(self):
+    """
+    Class to save the training converge properties
+    """
+    self.loss = nn.MSELoss()
+    self.MAE_loss = nn.L1Loss()
+    self.history={}                #Save convergence measures in the end of each epoch
+    self.history['loss']=[]        #value of the cost function on training data
+    self.history['mae']=[]         #mean absolute error on training data
+    self.history['val_loss']=[]    #value of the cost function on validation data
+    self.history['val_mae']=[]     #mean absolute error on validation data
+  
+  def update(self,current_model,xtrain,ytrain,xtest,ytest):
+    
+    #convergence information on the training data 
+    nb_training_obs=xtrain.shape[0]
+    if nb_training_obs>xtest.shape[0]:
+        nb_training_obs=xtest.shape[0]
+        
+    epoch_shuffler=np.arange(xtrain.shape[0]) 
+    np.random.shuffle(epoch_shuffler)
+    mini_batch_observations = epoch_shuffler[:nb_training_obs]
+    var_X_batch = Variable(xtrain[mini_batch_observations,:]).float()
+    var_y_batch = Variable(ytrain[mini_batch_observations]).float()
+    y_pred_batch = current_model(var_X_batch)
+    curr_loss = self.loss(y_pred_batch.view(-1), var_y_batch.view(-1))
+
+    self.history['loss'].append(curr_loss.item())
+    self.history['mae'].append(self.MAE_loss(y_pred_batch.view(-1), var_y_batch.view(-1)).item())
+    
+    #convergence information on the test data 
+    var_X_batch = Variable(xtest[:,:]).float()
+    var_y_batch = Variable(ytest[:]).float()
+    y_pred_batch = current_model(var_X_batch)
+    curr_loss = self.loss(y_pred_batch.view(-1), var_y_batch.view(-1))
+
+    self.history['val_loss'].append(curr_loss.item())
+    self.history['val_mae'].append(self.MAE_loss(y_pred_batch.view(-1), var_y_batch.view(-1)).item())

Misc/00-Numpy.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [NP1] - A short introduction to Numpy
+<!-- DESC --> Numpy is an essential tool for the Scientific Python.
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Understand the main principles of Numpy and its potential
+
+Note : This notebook is strongly inspired by the UGA Python Introduction Course
+See : **https://gricad-gitlab.univ-grenoble-alpes.fr/python-uga/py-training-2017**
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - Numpy the beginning
+
+Code using `numpy` usually starts with the import statement
+
+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+```
+
+%% Cell type:markdown id: tags:
+
+NumPy provides the type `np.ndarray`. Such array are multidimensionnal sequences of homogeneous elements. They can be created for example with the commands:
+
+%% Cell type:code id: tags:
+
+``` python
+# from a list
+l = [10.0, 12.5, 15.0, 17.5, 20.0]
+np.array(l)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# fast but the values can be anything
+np.empty(4)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# slower than np.empty but the values are all 0.
+np.zeros([2, 6])
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# multidimensional array
+a = np.ones([2, 3, 4])
+print(a.shape, a.size, a.dtype)
+a
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# like range but produce 1D numpy array
+np.arange(4)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# np.arange can produce arrays of floats
+np.arange(4.)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# another convenient function to generate 1D arrays
+np.linspace(10, 20, 5)
+```
+
+%% Cell type:markdown id: tags:
+
+A NumPy array can be easily converted to a Python list.
+
+%% Cell type:code id: tags:
+
+``` python
+a = np.linspace(10, 20 ,5)
+list(a)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Or even better
+a.tolist()
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 2 - Access elements
+
+Elements in a `numpy` array can be accessed using indexing and slicing in any dimension. It also offers the same functionalities available in Fortan or Matlab.
+
+### 2.1 - Indexes and slices
+For example, we can create an array `A` and perform any kind of selection operations on it.
+
+%% Cell type:code id: tags:
+
+``` python
+A = np.random.random([4, 5])
+A
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Get the element from second line, first column
+A[1, 0]
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Get the first two lines
+A[:2]
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Get the last column
+A[:, -1]
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Get the first two lines and the columns with an even index
+A[:2, ::2]
+```
+
+%% Cell type:markdown id: tags:
+
+### 2.2 -  Using a mask to select elements validating a condition:
+
+%% Cell type:code id: tags:
+
+``` python
+cond = A > 0.5
+print(cond)
+print(A[cond])
+```
+
+%% Cell type:markdown id: tags:
+
+The mask is in fact a particular case of the advanced indexing capabilities provided by NumPy. For example, it is even possible to use lists for indexing:
+
+%% Cell type:code id: tags:
+
+``` python
+# Selecting only particular columns
+print(A)
+A[:, [0, 1, 4]]
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 3 -  Perform array manipulations
+### 3.1 - Apply arithmetic operations to whole arrays (element-wise):
+
+%% Cell type:code id: tags:
+
+``` python
+(A+5)**2
+```
+
+%% Cell type:markdown id: tags:
+
+### 3.2 - Apply functions element-wise:
+
+%% Cell type:code id: tags:
+
+``` python
+np.exp(A) # With numpy arrays, use the functions from numpy !
+```
+
+%% Cell type:markdown id: tags:
+
+### 3.3 - Setting parts of arrays
+
+%% Cell type:code id: tags:
+
+``` python
+A[:, 0] = 0.
+print(A)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# BONUS: Safe element-wise inverse with masks
+cond = (A != 0)
+A[cond] = 1./A[cond]
+print(A)
+```
+
+%% Cell type:markdown id: tags:
+
+## Step 4 - Attributes and methods of `np.ndarray` (see the [doc](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html#numpy.ndarray))
+
+%% Cell type:code id: tags:
+
+``` python
+for i,v in enumerate([s for s in dir(A) if not s.startswith('__')]):
+    print(f'{v:16}', end='')
+    if (i+1) % 6 == 0 :print('')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+
+# Ex1: Get the mean through different dimensions
+
+print(A)
+print('Mean value',  A.mean())
+print('Mean line',   A.mean(axis=0))
+print('Mean column', A.mean(axis=1))
+```
+
+%% Cell type:code id: tags:
+
+``` python
+
+# Ex2: Convert a 2D array in 1D keeping all elements
+
+print(A)
+print(A.shape)
+A_flat = A.flatten()
+print(A_flat, A_flat.shape)
+```
+
+%% Cell type:markdown id: tags:
+
+### 4.1 - Remark: dot product
+
+%% Cell type:code id: tags:
+
+``` python
+b = np.linspace(0, 10, 11)
+c = b @ b
+# before 3.5:
+# c = b.dot(b)
+print(b)
+print(c)
+```
+
+%% Cell type:markdown id: tags:
+
+### 4.2 -  For Matlab users
+
+|     ` `       | Matlab | Numpy |
+| ------------- | ------ | ----- |
+| element wise  |  `.*`  |  `*`  |
+|  dot product  |  `*`   |  `@`  |
+
+%% Cell type:markdown id: tags:
+
+`numpy` arrays can also be sorted, even when they are composed of complex data if the type of the columns are explicitly stated with `dtypes`.
+
+%% Cell type:markdown id: tags:
+
+### 4.3 -  NumPy and SciPy sub-packages:
+
+We already saw `numpy.random` to generate `numpy` arrays filled with random values. This submodule also provides functions related to distributions (Poisson, gaussian, etc.) and permutations.
+
+%% Cell type:markdown id: tags:
+
+To perform linear algebra with dense matrices, we can use the submodule `numpy.linalg`. For instance, in order to compute the determinant of a random matrix, we use the method `det`
+
+%% Cell type:code id: tags:
+
+``` python
+A = np.random.random([5,5])
+print(A)
+np.linalg.det(A)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+squared_subA = A[1:3, 1:3]
+print(squared_subA)
+np.linalg.inv(squared_subA)
+```
+
+%% Cell type:markdown id: tags:
+
+### 4.4 -  Introduction to Pandas: Python Data Analysis Library
+
+Pandas is an open source library providing high-performance, easy-to-use data structures and data analysis tools for Python.
+
+[Pandas tutorial](https://pandas.pydata.org/pandas-docs/stable/10min.html)
+[Grenoble Python Working Session](https://github.com/iutzeler/Pres_Pandas/)
+[Pandas for SQL Users](http://sergilehkyi.com/translating-sql-to-pandas/)
+[Pandas Introduction Training HPC Python@UGA](https://gricad-gitlab.univ-grenoble-alpes.fr/python-uga/training-hpc/-/blob/master/ipynb/11_pandas.ipynb)
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

Misc/01-Activation-Functions.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [ACTF1] - Activation functions
+<!-- DESC --> Some activation functions, with their derivatives.
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - View the main activation functions
+
+Les fonctions d'activation dans Keras :
+https://www.tensorflow.org/api_docs/python/tf/keras/activations
+
+## What we're going to do :
+
+ - Juste visualiser les principales fonctions d'activation
+
+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+import matplotlib
+import matplotlib.pyplot as plt
+
+import math
+from math import erfc, sqrt, exp
+from math import pi as PI
+from math import e as E
+import sys
+
+import fidle
+
+# Init Fidle environment
+run_id, run_dir, datasets_dir = fidle.init('ACTF1')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)
+SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5
+
+
+def heaviside(z):
+    return np.where(z<0,0,1)
+
+def sign(z):
+    return np.where(z<0,-1,1)
+#    return np.sign(z)
+
+def sigmoid(z):
+    return 1 / (1 + np.exp(-z))
+
+def tanh(z):
+    return np.tanh(z)
+
+def relu(z):
+    return np.maximum(0, z)
+
+def leaky_relu(z,a=0.05):
+    return np.maximum(a*z, z)
+
+def elu(z,a=1):
+    #y=z.copy()
+    y=a*(np.exp(z)-1)
+    y[z>0]=z[z>0]
+    return y
+
+def selu(z):
+    return SELU_L*elu(z,a=SELU_A)
+
+def derivative(f, z, eps=0.000001):
+    return (f(z + eps) - f(z - eps))/(2 * eps)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+pw=5
+ph=5
+
+z = np.linspace(-5, 5, 200)
+
+
+# ------ Heaviside
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(0, 0, "rx", markersize=10)
+ax.plot(z, heaviside(z),              linestyle='-',  label="Heaviside")
+ax.plot(z, derivative(heaviside, z),  linewidth=3, alpha=0.6, label="dHeaviside/dx")
+# ax.plot(z, sign(z),                  label="Heaviside")
+ax.set_title("Heaviside")
+fidle.scrawler.save_fig('Heaviside')
+plt.show()
+
+
+# ----- Logit/Sigmoid
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(z, sigmoid(z),             label="Sigmoid")
+ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label="dSigmoid/dx")
+ax.set_title("Logit")
+fidle.scrawler.save_fig('Logit')
+plt.show()
+
+# ----- Tanh
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(z, tanh(z),             label="Tanh")
+ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label="dTanh/dx")
+ax.set_title("Tanh")
+fidle.scrawler.save_fig('Tanh')
+plt.show()
+
+# ----- Relu
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(z, relu(z),             label="ReLU")
+ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label="dReLU/dx")
+ax.set_title("ReLU")
+fidle.scrawler.save_fig('ReLU')
+plt.show()
+
+# ----- Leaky Relu
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(z, leaky_relu(z),              label="Leaky ReLU")
+ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label="dLeakyReLU/dx")
+ax.set_title("Leaky ReLU (α=0.05)")
+fidle.scrawler.save_fig('LeakyReLU')
+plt.show()
+
+# ----- Elu
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(z, elu(z),              label="ReLU")
+ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label="dExpReLU/dx")
+ax.set_title("ELU (α=1)")
+fidle.scrawler.save_fig('ELU')
+plt.show()
+
+# ----- Selu
+#
+fig, ax = plt.subplots(1, 1)
+fig.set_size_inches(pw,ph)
+ax.set_xlim(-5, 5)
+ax.set_ylim(-2, 2)
+ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
+ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
+ax.plot(z, selu(z),              label="SeLU")
+ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label="dSeLU/dx")
+ax.set_title("ELU (SELU)")
+fidle.scrawler.save_fig('SeLU')
+plt.show()
+
+```
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>

Misc/02-Using-pandas.ipynb

+%% Cell type:markdown id: tags:
+
+<img width="800px" src="../fidle/img/header.svg"></img>
+
+# <!-- TITLE --> [PANDAS1] - Quelques exemples avec Pandas
+<!-- DESC --> pandas is another essential tool for the Scientific Python.
+<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
+
+## Objectives :
+ - Understand how to slice a dataset
+
+%% Cell type:markdown id: tags:
+
+## Step 1 - A little cooking with datasets
+
+%% Cell type:code id: tags:
+
+``` python
+import pandas as pd
+import numpy  as np
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Get some data
+a = np.arange(50).reshape(10,5)
+print('Starting data: \n',a)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Create a DataFrame
+df_all = pd.DataFrame(a, columns=['A','B','C','D','E'])
+print('\nDataFrame :')
+display(df_all)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Shuffle data
+df_all = df_all.sample(frac=1, axis=0)
+print('\nDataFrame randomly shuffled :')
+display(df_all)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+# Get a train part
+df_train = df_all.sample(frac=0.8, axis=0)
+print('\nTrain set (80%) :')
+display(df_train)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+
+# Get test set as all - train
+df_test = df_all.drop(df_train.index)
+print('\nTest set (all - train) :')
+display(df_test)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+x_train = df_train.drop('E',  axis=1)
+y_train = df_train['E']
+x_test  = df_test.drop('E',   axis=1)
+y_test  = df_test['E']
+display(x_train)
+display(y_train)
+```
+
+%% Cell type:markdown id: tags:
+
+---
+<img width="80px" src="../fidle/img/logo-paysage.svg"></img>