Skip to content
Snippets Groups Projects
Commit c7db2aee authored by Jean-Luc Parouty's avatar Jean-Luc Parouty
Browse files

Add full convolutions notebook and pytables for pandas

parent 0663d0b1
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
GTSRB/01-Preparation-of-data.ipynb
+ 17
− 312
View file @ c7db2aee
This diff is collapsed.
GTSRB/02-First-convolutions.ipynb
+ 46
− 9
View file @ c7db2aee
source diff could not be displayed: it is too large. Options to address this: view the blob.
GTSRB/07-Full-convolutions.ipynb 0 → 100644
+ 541
− 0
View file @ c7db2aee
%% Cell type:markdown id: tags:
German Traffic Sign Recognition Benchmark (GTSRB)
=================================================
---
Introduction au Deep Learning (IDLE) - S. Arias, E. Maldonado, JL. Parouty - CNRS/SARI/DEVLOG - 2020
Vesion : 1.2.1
## Episode 7 : Full Convolutions
Our main steps:
- Try n models with n datasets
## 1/ Import and init
%% Cell type:code id: tags:
``` python
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.callbacks import TensorBoard
import numpy as np
import matplotlib.pyplot as plt
import h5py
import os,time
import pandas as pd
import idle.pwk as ooo
from importlib import reload
from IPython.display import display
ooo.init()
```
%% Output
IDLE 2020 - Practical Work Module
Version : 0.1.4
Run time : Friday 17 January 2020, 21:38:34
Matplotlib style : idle/talk.mplstyle
TensorFlow version : 2.0.0
Keras version : 2.2.4-tf
%% Cell type:markdown id: tags:
## 2/ Load dataset functions
%% Cell type:code id: tags:
``` python
def read_dataset(name):
'''Reads h5 dataset from ./data
Arguments: dataset name, without .h5
Returns: x_train,y_train,x_test,y_test data'''
# ---- Read dataset
filename='./data/'+name+'.h5'
with h5py.File(filename) as f:
x_train = f['x_train'][:]
y_train = f['y_train'][:]
x_test = f['x_test'][:]
y_test = f['y_test'][:]
return x_train,y_train,x_test,y_test
```
%% Cell type:markdown id: tags:
## 3/ Models collection
%% Cell type:code id: tags:
``` python
# A basic model
#
def get_model_v1(lx,ly,lz):
model = keras.models.Sequential()
model.add( keras.layers.Conv2D(96, (3,3), activation='relu', input_shape=(lx,ly,lz)))
model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Conv2D(192, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Flatten())
model.add( keras.layers.Dense(1500, activation='relu'))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Dense(43, activation='softmax'))
return model
# A more sophisticated model
#
def get_model_v2(lx,ly,lz):
model = keras.models.Sequential()
model.add( keras.layers.Conv2D(64, (3, 3), padding='same', input_shape=(lx,ly,lz), activation='relu'))
model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Conv2D(256, (3, 3), padding='same',activation='relu'))
model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add( keras.layers.Dropout(0.2))
model.add( keras.layers.Flatten())
model.add( keras.layers.Dense(512, activation='relu'))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Dense(43, activation='softmax'))
return model
# My sphisticated model, but small and fast
#
def get_model_v3(lx,ly,lz):
model = keras.models.Sequential()
model.add( keras.layers.Conv2D(32, (3,3), activation='relu', input_shape=(lx,ly,lz)))
model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Conv2D(64, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Conv2D(128, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Conv2D(256, (3, 3), activation='relu'))
model.add( keras.layers.MaxPooling2D((2, 2)))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Flatten())
model.add( keras.layers.Dense(1152, activation='relu'))
model.add( keras.layers.Dropout(0.5))
model.add( keras.layers.Dense(43, activation='softmax'))
return model
```
%% Cell type:markdown id: tags:
## 4/ Callbacks
%% Cell type:code id: tags:
``` python
%%bash
# To clean old logs and saved model, run this cell
#
/bin/rm -r ./run/logs 2>/dev/null
/bin/rm -r ./run/models 2>/dev/null
/bin/ls -l ./run 2>/dev/null
```
%% Output
total 0
%% Cell type:code id: tags:
``` python
ooo.mkdir('./run/models')
ooo.mkdir('./run/logs')
# ---- Callback tensorboard
log_dir = "./run/logs/tb_" + ooo.tag_now()
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
# ---- Callback ModelCheckpoint - Save best model
save_dir = "./run/models/best-model.h5"
bestmodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, monitor='accuracy', save_best_only=True)
# ---- Callback ModelCheckpoint - Save model each epochs
save_dir = "./run/models/model-{epoch:04d}.h5"
savemodel_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_dir, verbose=0, save_freq=2000*5)
```
%% Cell type:markdown id: tags:
## 6/ Multiple datasets, multiple models ;-)
%% Cell type:code id: tags:
``` python
def multi_run(datasets, models, batch_size=64, epochs=16):
# ---- Columns of report
#
report={}
report['Dataset']=[]
report['Size'] =[]
for m in models:
report[m+' Accuracy'] = []
report[m+' Duration'] = []
# ---- Let's go
#
for dname in datasets:
print("\nDataset : ",dname)
# ---- Read dataset
x_train,y_train,x_test,y_test = read_dataset(dname)
dsize=os.path.getsize('./data/'+dname+'.h5')/(1024*1024)
report['Dataset'].append(dname)
report['Size'].append(dname)
# ---- Get the shape
(n,lx,ly,lz) = x_train.shape
# ---- For each model
for kmodel,fmodel in models.items():
print(" Run model {} : ".format(kmodel), end='')
# ---- get model
try:
model=fmodel(lx,ly,lz)
# ---- Compile it
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
# ---- Train
start_time = time.time()
history = model.fit( x_train[:1000], y_train[:1000],
batch_size = batch_size,
epochs = epochs,
verbose = 0,
validation_data = (x_test, y_test),
callbacks = [tensorboard_callback, bestmodel_callback, savemodel_callback])
# ---- Result
end_time = time.time()
duration = end_time-start_time
accuracy = max(history.history["val_accuracy"])*100
#
report[kmodel+' Accuracy'].append(accuracy)
report[kmodel+' Duration'].append(duration)
print("Accuracy={:.2f} and Duration={:.2f})".format(accuracy,duration))
except:
report[kmodel+' Accuracy'].append('-')
report[kmodel+' Duration'].append('-')
print('-')
print("\n")
return report
```
%% Cell type:code id: tags:
``` python
%%time
# datasets = ['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']
# models = {'v1':get_model_v1, 'v2':get_model_v2, 'v3':get_model_v3}
datasets = ['set-24x24-L', 'set-24x24-RGB']
models = {'v1':get_model_v1, 'v3':get_model_v3}
out = multi_run(datasets, models, batch_size=64, epochs=2)
report = pd.DataFrame (out)
```
%% Output
Dataset : set-24x24-L
Run model v1 : Accuracy=9.46 and Duration=7.51)
Run model v3 : -
Dataset : set-24x24-RGB
Run model v1 : Accuracy=15.95 and Duration=7.95)
Run model v3 : -
CPU times: user 1min 35s, sys: 3.31 s, total: 1min 38s
Wall time: 17 s
%% Cell type:code id: tags:
``` python
display(report)
df.to_hdf('foo.h5', 'df')
```
%% Output
%% Cell type:code id: tags:
``` python
df=pd.read_hdf('foo.h5', 'df')
display(df)
```
%% Output
%% Cell type:markdown id: tags:
---
### Some results :
%% Cell type:markdown id: tags:
| Datasets | Size | Model : v1 | Model : v2 | Model : v3 |
|:------------------------:|:---------------:|:------------------:|:------------------:|:------------------:|
| set-24x24-L | 229 Mo | 95.91% 75.04s | 96.86% 102.28s | - - |
| set-24x24-RGB | 684 Mo | 96.60% 77.24s | 97.32% 103.93s | - - |
| set-48x48-L | 914 Mo | **96.71%** 123.94s | 97.68% 149.57s | 97.60% 91.53s |
| set-48x48-RGB | 2736 Mo | 96.36% 117.74s | **98.20%** 142.63s | 97.28% 91.29s |
| set-24x24-L-LHE | 229 Mo | 95.95% 66.12s | 96.75% 89.45s | - - |
| set-24x24-RGB-HE | 684 Mo | 95.30% 68.89s | 96.28% 92.15s | - - |
| set-48x48-L-LHE | 914 Mo | 96.69% 109.28s | 97.94% 135.17s | **97.97%** 83.80s |
| set-48x48-RGB-HE | 2736 Mo | 95.29% 117.70s | **98.13%** 141.56s | 97.00% 89.38s |
%% Cell type:code id: tags:
``` python
```
environment.yml
+ 9
− 3
View file @ c7db2aee
......@@ -10,6 +10,8 @@ dependencies:
- backcall=0.1.0=py37_0
- blas=1.0=mkl
- bleach=3.1.0=py_0
- blosc=1.16.3=hd408876_0
- bzip2=1.0.8=h7b6447c_0
- c-ares=1.15.0=h7b6447c_1001
- ca-certificates=2019.11.27=0
- certifi=2019.11.28=py37_0
......@@ -68,6 +70,8 @@ dependencies:
- libuuid=1.0.3=h1bed415_2
- libxcb=1.13=h1bed415_1
- libxml2=2.9.9=hea5a465_1
- lz4-c=1.8.1.2=h14c3975_0
- lzo=2.10=h49e0be7_2
- markdown=3.1.1=py37_0
- markupsafe=1.1.1=py37h7b6447c_0
- matplotlib=3.1.1=py37h5429711_0
......@@ -76,12 +80,14 @@ dependencies:
- mkl-service=2.3.0=py37he904b0f_0
- mkl_fft=1.0.15=py37ha843d7b_0
- mkl_random=1.1.0=py37hd6b4f25_0
- mock=3.0.5=py37_0
- more-itertools=8.0.2=py_0
- nbconvert=5.6.1=py37_0
- nbformat=4.4.0=py37_0
- ncurses=6.1=he6710b0_1
- networkx=2.4=py_0
- notebook=6.0.2=py37_0
- numexpr=2.7.0=py37h9e4a6bb_0
- numpy=1.17.4=py37hc1035e2_0
- numpy-base=1.17.4=py37hde5b4d6_0
- olefile=0.46=py_0
......@@ -105,6 +111,7 @@ dependencies:
- pyparsing=2.4.5=py_0
- pyqt=5.9.2=py37h05f1152_2
- pyrsistent=0.15.6=py37h7b6447c_0
- pytables=3.6.1=py37h71ec239_0
- python=3.7.5=h0371630_0
- python-dateutil=2.8.1=py_0
- pytz=2019.3=py_0
......@@ -120,6 +127,7 @@ dependencies:
- setuptools=42.0.2=py37_0
- sip=4.19.8=py37hf484d3e_0
- six=1.13.0=py37_0
- snappy=1.1.7=hbae5bb6_3
- sqlite=3.30.1=h7b6447c_0
- statsmodels=0.10.1=py37hdd07704_0
- tensorboard=2.0.0=pyhb38c66f_1
......@@ -144,7 +152,5 @@ dependencies:
- zipp=0.6.0=py_0
- zlib=1.2.11=h7b6447c_3
- zstd=1.3.7=h0b5b093_0
- pip:
- dask==2.9.0
prefix: /home/paroutyj/.conda/envs/deeplearning2
prefix: /home/pjluc/anaconda3/envs/deeplearning2
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment