From eda5f96807933ffb25cfde1faf989b240491f64d Mon Sep 17 00:00:00 2001
From: Jean-Luc Parouty <Jean-Luc.Parouty@simap.grenoble-inp.fr>
Date: Mon, 10 Oct 2022 17:38:20 +0200
Subject: [PATCH] Update notebooks (v2.02b4)
---
AE/01-Prepare-MNIST-dataset.ipynb | 21 ++++++++--------
AE/02-AE-with-MNIST.ipynb | 5 ++--
AE/03-AE-with-MNIST-post.ipynb | 2 +-
AE/04-ExtAE-with-MNIST.ipynb | 5 ++--
AE/05-ExtAE-with-MNIST.ipynb | 5 ++--
GTSRB/01-Preparation-of-data.ipynb | 4 ++--
GTSRB/03-Tracking-and-visualizing.ipynb | 16 ++++++++-----
GTSRB/04-Data-augmentation.ipynb | 7 +++---
GTSRB/06-Notebook-as-a-batch.ipynb | 9 +++++--
GTSRB/07-Show-report.ipynb | 14 +++++++----
IMDB/01-One-hot-encoding.ipynb | 14 +++++++----
IMDB/02-Keras-embedding.ipynb | 14 +++++++----
IMDB/03-Prediction.ipynb | 20 ++++++++++------
IMDB/04-Show-vectors.ipynb | 18 +++++++++-----
IMDB/05-LSTM-Keras.ipynb | 24 ++++++++++---------
SYNOP/LADYB1-Ladybug.ipynb | 32 +++++++++++++------------
SYNOP/SYNOP1-Preparation-of-data.ipynb | 18 +++++++-------
SYNOP/SYNOP2-First-predictions.ipynb | 28 ++++++++++++----------
SYNOP/SYNOP3-12h-predictions.ipynb | 21 +++++++++-------
fidle/about.yml | 2 +-
20 files changed, 164 insertions(+), 115 deletions(-)
diff --git a/AE/01-Prepare-MNIST-dataset.ipynb b/AE/01-Prepare-MNIST-dataset.ipynb
index d8e2366..5d4a658 100644
--- a/AE/01-Prepare-MNIST-dataset.ipynb
+++ b/AE/01-Prepare-MNIST-dataset.ipynb
@@ -44,11 +44,10 @@
"import modules.MNIST\n",
"from modules.MNIST import MNIST\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir='./run/AE1'\n",
- "run_id, run_dir, datasets_dir = fidle.init('AE1', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('AE1')"
]
},
{
@@ -69,7 +68,7 @@
"source": [
"prepared_dataset = './data/mnist-noisy.h5'\n",
"scale = 1\n",
- "progress_verbosity = 1"
+ "progress_verbosity = 2"
]
},
{
@@ -127,7 +126,7 @@
"def noise_it(data):\n",
" new_data = np.copy(data)\n",
" for i,image in enumerate(new_data):\n",
- " fidle.utils.update_progress('Add noise : ',i+1,len(new_data),verbosity=progress_verbosity)\n",
+ " fidle.utils.update_progress('Add noise : ',i+1,len(data),verbosity=progress_verbosity)\n",
" image=random_noise(image, mode='gaussian', mean=0, var=0.3)\n",
" image=random_noise(image, mode='s&p', amount=0.2, salt_vs_pepper=0.5)\n",
" image=random_noise(image, mode='poisson') \n",
@@ -216,11 +215,8 @@
}
],
"metadata": {
- "interpreter": {
- "hash": "8e38643e33497db9a306e3f311fa98cb1e65371278ca73ee4ea0c76aa5a4f387"
- },
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -234,7 +230,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.10"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/AE/02-AE-with-MNIST.ipynb b/AE/02-AE-with-MNIST.ipynb
index 906fd79..8d9eed3 100644
--- a/AE/02-AE-with-MNIST.ipynb
+++ b/AE/02-AE-with-MNIST.ipynb
@@ -260,7 +260,8 @@
"metadata": {},
"outputs": [],
"source": [
- "pwk.chrono_start()\n",
+ "chrono = fidle.Chrono()\n",
+ "chrono.start()\n",
"\n",
"history = ae.fit(noisy_train, clean_train,\n",
" batch_size = batch_size,\n",
@@ -269,7 +270,7 @@
" validation_data = (noisy_test, clean_test),\n",
" callbacks = callbacks_list )\n",
"\n",
- "pwk.chrono_show()"
+ "chrono.show()"
]
},
{
diff --git a/AE/03-AE-with-MNIST-post.ipynb b/AE/03-AE-with-MNIST-post.ipynb
index a01b1d2..2be5ab0 100644
--- a/AE/03-AE-with-MNIST-post.ipynb
+++ b/AE/03-AE-with-MNIST-post.ipynb
@@ -241,7 +241,7 @@
"\n",
"# ---- Select images\n",
"\n",
- "x_show, y_show = pwk.pick_dataset(clean_data, class_data, n=n_show)\n",
+ "x_show, y_show = fidle.utils.pick_dataset(clean_data, class_data, n=n_show)\n",
"\n",
"# ---- Get latent points\n",
"\n",
diff --git a/AE/04-ExtAE-with-MNIST.ipynb b/AE/04-ExtAE-with-MNIST.ipynb
index 2e31ed4..fe23ccc 100644
--- a/AE/04-ExtAE-with-MNIST.ipynb
+++ b/AE/04-ExtAE-with-MNIST.ipynb
@@ -323,7 +323,8 @@
"metadata": {},
"outputs": [],
"source": [
- "pwk.chrono_start()\n",
+ "chrono = fidle.Chrono()\n",
+ "chrono.start()\n",
"\n",
"history = model.fit(noisy_train, [clean_train, class_train],\n",
" batch_size = batch_size,\n",
@@ -332,7 +333,7 @@
" validation_data = (noisy_test, [clean_test, class_test]),\n",
" callbacks = callbacks_list )\n",
"\n",
- "pwk.chrono_show()"
+ "chrono.show()"
]
},
{
diff --git a/AE/05-ExtAE-with-MNIST.ipynb b/AE/05-ExtAE-with-MNIST.ipynb
index a9125af..4dc1e5e 100644
--- a/AE/05-ExtAE-with-MNIST.ipynb
+++ b/AE/05-ExtAE-with-MNIST.ipynb
@@ -354,7 +354,8 @@
"metadata": {},
"outputs": [],
"source": [
- "pwk.chrono_start()\n",
+ "chrono = fidle.Chrono()\n",
+ "chrono.start()\n",
"\n",
"history = model.fit(noisy_train, [clean_train, class_train],\n",
" batch_size = batch_size,\n",
@@ -363,7 +364,7 @@
" validation_data = (noisy_test, [clean_test, class_test]),\n",
" callbacks = callbacks_list )\n",
"\n",
- "pwk.chrono_show()"
+ "chrono.show()"
]
},
{
diff --git a/GTSRB/01-Preparation-of-data.ipynb b/GTSRB/01-Preparation-of-data.ipynb
index 3f7a7d3..e01d6a1 100644
--- a/GTSRB/01-Preparation-of-data.ipynb
+++ b/GTSRB/01-Preparation-of-data.ipynb
@@ -94,9 +94,9 @@
"# scale = 1\n",
"# output_dir = f'{datasets_dir}/GTSRB/enhanced'\n",
"\n",
- "# ---- Verbosity - 0 = silent, 1 = progress bar, 2 = one line\n",
+ "# ---- Verbosity - 0 = silent, 1 = one line, 2 = full progress bar\n",
"#\n",
- "progress_verbosity = 1"
+ "progress_verbosity = 2"
]
},
{
diff --git a/GTSRB/03-Tracking-and-visualizing.ipynb b/GTSRB/03-Tracking-and-visualizing.ipynb
index cc2f686..2054dce 100644
--- a/GTSRB/03-Tracking-and-visualizing.ipynb
+++ b/GTSRB/03-Tracking-and-visualizing.ipynb
@@ -50,11 +50,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/GTSRB3.001'\n",
- "run_id, run_dir, datasets_dir = fidle.init('GTSRB3', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('GTSRB3')"
]
},
{
@@ -238,7 +237,7 @@
"fidle.utils.mkdir(run_dir + '/logs')\n",
"\n",
"# ---- Callback tensorboard\n",
- "log_dir = run_dir + \"/logs/tb_\" + Chrono.tag_now()\n",
+ "log_dir = run_dir + \"/logs/tb_\" + fidle.Chrono.tag_now()\n",
"tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
"\n",
"# ---- Callback ModelCheckpoint - Save best model\n",
@@ -536,7 +535,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -550,7 +549,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/GTSRB/04-Data-augmentation.ipynb b/GTSRB/04-Data-augmentation.ipynb
index bc9d39a..51fbe3f 100644
--- a/GTSRB/04-Data-augmentation.ipynb
+++ b/GTSRB/04-Data-augmentation.ipynb
@@ -48,11 +48,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/GTSRB4.001'\n",
- "run_id, run_dir, datasets_dir = fidle.init('GTSRB4', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('GTSRB4')"
]
},
{
@@ -204,7 +203,7 @@
"fidle.utils.mkdir(run_dir + '/logs')\n",
"\n",
"# ---- Callback tensorboard\n",
- "log_dir = run_dir + \"/logs/tb_\" + Chrono.tag_now()\n",
+ "log_dir = run_dir + \"/logs/tb_\" + fidle.Chrono.tag_now()\n",
"tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
"\n",
"# ---- Callback ModelCheckpoint - Save best model\n",
diff --git a/GTSRB/06-Notebook-as-a-batch.ipynb b/GTSRB/06-Notebook-as-a-batch.ipynb
index 08825ec..e97608a 100644
--- a/GTSRB/06-Notebook-as-a-batch.ipynb
+++ b/GTSRB/06-Notebook-as-a-batch.ipynb
@@ -181,7 +181,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -195,7 +195,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.5"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/GTSRB/07-Show-report.ipynb b/GTSRB/07-Show-report.ipynb
index 1cc3da1..7f2aeb5 100644
--- a/GTSRB/07-Show-report.ipynb
+++ b/GTSRB/07-Show-report.ipynb
@@ -35,11 +35,10 @@
"from pathlib import Path\n",
"from IPython.display import display, Markdown\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/GTSRB7'\n",
- "run_id, run_dir, datasets_dir = fidle.init('GTSRB7', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('GTSRB7')"
]
},
{
@@ -154,7 +153,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -168,7 +167,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.5"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/IMDB/01-One-hot-encoding.ipynb b/IMDB/01-One-hot-encoding.ipynb
index 5fd95fc..e39ec99 100644
--- a/IMDB/01-One-hot-encoding.ipynb
+++ b/IMDB/01-One-hot-encoding.ipynb
@@ -55,11 +55,10 @@
"import os,sys,h5py,json\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/IMDB1'\n",
- "run_id, run_dir, datasets_dir = fidle.init('IMDB1', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('IMDB1')"
]
},
{
@@ -589,7 +588,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -603,7 +602,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/IMDB/02-Keras-embedding.ipynb b/IMDB/02-Keras-embedding.ipynb
index c08156c..d658281 100644
--- a/IMDB/02-Keras-embedding.ipynb
+++ b/IMDB/02-Keras-embedding.ipynb
@@ -53,11 +53,10 @@
"import os,sys,h5py,json\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/IMDB2'\n",
- "run_id, run_dir, datasets_dir = fidle.init('IMDB2', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('IMDB2')"
]
},
{
@@ -408,7 +407,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -422,7 +421,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/IMDB/03-Prediction.ipynb b/IMDB/03-Prediction.ipynb
index 56cdd99..ffcb5c8 100644
--- a/IMDB/03-Prediction.ipynb
+++ b/IMDB/03-Prediction.ipynb
@@ -48,11 +48,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/IMDB2'\n",
- "run_id, run_dir, datasets_dir = fidle.init('IMDB3', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('IMDB3')"
]
},
{
@@ -63,6 +62,7 @@
"The words in the vocabulary are classified from the most frequent to the rarest. \n",
"`vocab_size` is the number of words we will remember in our vocabulary (the other words will be considered as unknown). \n",
"`review_len` is the review length \n",
+ "`saved_models` where our models were previously saved \n",
"`dictionaries_dir` is where we will go to save our dictionaries. (./data is a good choice)"
]
},
@@ -75,6 +75,7 @@
"vocab_size = 10000\n",
"review_len = 256\n",
"\n",
+ "saved_models = './run/IMDB2'\n",
"dictionaries_dir = './data'"
]
},
@@ -91,7 +92,7 @@
"metadata": {},
"outputs": [],
"source": [
- "fidle.override('vocab_size', 'review_len', 'dictionaries_dir')"
+ "fidle.override('vocab_size', 'review_len', 'saved_models', 'dictionaries_dir')"
]
},
{
@@ -216,7 +217,7 @@
"metadata": {},
"outputs": [],
"source": [
- "model = keras.models.load_model(f'{run_dir}/models/best_model.h5')"
+ "model = keras.models.load_model(f'{saved_models}/models/best_model.h5')"
]
},
{
@@ -274,7 +275,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -288,7 +289,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/IMDB/04-Show-vectors.ipynb b/IMDB/04-Show-vectors.ipynb
index 168d87d..350a550 100644
--- a/IMDB/04-Show-vectors.ipynb
+++ b/IMDB/04-Show-vectors.ipynb
@@ -47,10 +47,9 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/IMDB2'\n",
+ "# Init Fidle environment\n",
"run_id, run_dir, datasets_dir = fidle.init('IMDB4')"
]
},
@@ -62,6 +61,7 @@
"The words in the vocabulary are classified from the most frequent to the rarest. \n",
"`vocab_size` is the number of words we will remember in our vocabulary (the other words will be considered as unknown). \n",
"`review_len` is the review length \n",
+ "`saved_models` where our models were previously saved \n",
"`dictionaries_dir` is where we will go to save our dictionaries. (./data is a good choice)"
]
},
@@ -74,6 +74,7 @@
"vocab_size = 5000\n",
"review_len = 256\n",
"\n",
+ "saved_models = './run/IMDB2'\n",
"dictionaries_dir = './data'"
]
},
@@ -90,7 +91,7 @@
"metadata": {},
"outputs": [],
"source": [
- "fidle.override('vocab_size', 'review_len', 'dictionaries_dir')"
+ "fidle.override('vocab_size', 'review_len', 'saved_models', 'dictionaries_dir')"
]
},
{
@@ -113,7 +114,7 @@
"metadata": {},
"outputs": [],
"source": [
- "model = keras.models.load_model(f'{run_dir}/models/best_model.h5')\n",
+ "model = keras.models.load_model(f'{saved_models}/models/best_model.h5')\n",
"print('Model loaded.')\n",
"\n",
"with open(f'{dictionaries_dir}/index_word.json', 'r') as fp:\n",
@@ -251,7 +252,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -265,7 +266,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/IMDB/05-LSTM-Keras.ipynb b/IMDB/05-LSTM-Keras.ipynb
index 7775121..7c61576 100644
--- a/IMDB/05-LSTM-Keras.ipynb
+++ b/IMDB/05-LSTM-Keras.ipynb
@@ -52,11 +52,10 @@
"import os,sys,h5py,json\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/IMDB5'\n",
- "run_id, run_dir, datasets_dir = fidle.init('IMDB5', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('IMDB5')"
]
},
{
@@ -89,7 +88,7 @@
"batch_size = 128\n",
"\n",
"fit_verbosity = 1\n",
- "scale = 1"
+ "scale = 0.2"
]
},
{
@@ -349,7 +348,8 @@
"metadata": {},
"outputs": [],
"source": [
- "pwk.chrono_start()\n",
+ "chrono = fidle.Chrono()\n",
+ "chrono.start()\n",
"\n",
"history = model.fit(x_train,\n",
" y_train,\n",
@@ -359,7 +359,7 @@
" verbose = fit_verbosity,\n",
" callbacks = [savemodel_callback])\n",
"\n",
- "pwk.chrono_show()"
+ "chrono.show()"
]
},
{
@@ -434,11 +434,8 @@
}
],
"metadata": {
- "interpreter": {
- "hash": "8e38643e33497db9a306e3f311fa98cb1e65371278ca73ee4ea0c76aa5a4f387"
- },
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -452,7 +449,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/SYNOP/LADYB1-Ladybug.ipynb b/SYNOP/LADYB1-Ladybug.ipynb
index 77cc5c9..f26b1b3 100644
--- a/SYNOP/LADYB1-Ladybug.ipynb
+++ b/SYNOP/LADYB1-Ladybug.ipynb
@@ -46,11 +46,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/LADYB1'\n",
- "run_id, run_dir, datasets_dir = fidle.init('LADYB1', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('LADYB1')"
]
},
{
@@ -78,7 +77,7 @@
"\n",
"# ---- About training\n",
"#\n",
- "scale = 1 # Percentage of dataset to be used (1=all)\n",
+ "scale = .2 # Percentage of dataset to be used (1=all)\n",
"train_prop = .8 # Percentage for train (the rest being for the test)\n",
"batch_size = 32\n",
"epochs = 5\n",
@@ -319,7 +318,8 @@
"metadata": {},
"outputs": [],
"source": [
- "pwk.chrono_start()\n",
+ "chrono=fidle.Chrono()\n",
+ "chrono.start()\n",
"\n",
"history=model.fit(train_generator, \n",
" epochs = epochs, \n",
@@ -327,7 +327,7 @@
" validation_data = test_generator,\n",
" callbacks = [bestmodel_callback])\n",
"\n",
- "pwk.chrono_show()"
+ "chrono.show()"
]
},
{
@@ -388,8 +388,8 @@
"print('sequence true shape :',sequence_true.shape)\n",
"print('sequence pred shape :',sequence_pred.shape)\n",
"\n",
- "fidle.scrawler.segment_2d(sequence_true, sequence_pred)\n",
- "fidle.scrawler.multivariate_serie(sequence_true, predictions=sequence_pred, labels=['Axis=0', 'Axis=1'],save_as='04-one-step-prediction')"
+ "fidle.scrawler.segment_2d(sequence_true, sequence_pred, save_as='04-one-step-prediction')\n",
+ "fidle.scrawler.multivariate_serie(sequence_true, predictions=sequence_pred, labels=['Axis=0', 'Axis=1'],save_as='05-one-step-prediction-2axis')"
]
},
{
@@ -447,8 +447,8 @@
"source": [
"sequence_true, sequence_pred = get_prediction(x_test, loaded_model, iterations=5)\n",
"\n",
- "fidle.scrawler.segment_2d(sequence_true, sequence_pred, ms=8, save_as='02-prediction-norm')\n",
- "fidle.scrawler.multivariate_serie(sequence_true, predictions=sequence_pred, hide_ticks=True, labels=['Axis=0', 'Axis=1'],save_as='02-prediction-norm')"
+ "fidle.scrawler.segment_2d(sequence_true, sequence_pred, ms=8, save_as='06-n-steps-prediction-norm')\n",
+ "fidle.scrawler.multivariate_serie(sequence_true, predictions=sequence_pred, hide_ticks=True, labels=['Axis=0', 'Axis=1'],save_as='07-n-steps-prediction-norm')"
]
},
{
@@ -470,11 +470,8 @@
}
],
"metadata": {
- "interpreter": {
- "hash": "8e38643e33497db9a306e3f311fa98cb1e65371278ca73ee4ea0c76aa5a4f387"
- },
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -488,7 +485,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/SYNOP/SYNOP1-Preparation-of-data.ipynb b/SYNOP/SYNOP1-Preparation-of-data.ipynb
index b8ac7f6..d62fbbd 100644
--- a/SYNOP/SYNOP1-Preparation-of-data.ipynb
+++ b/SYNOP/SYNOP1-Preparation-of-data.ipynb
@@ -52,11 +52,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/SYNOP'\n",
- "run_id, run_dir, datasets_dir = fidle.init('SYNOP1', run_dir)\n",
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('SYNOP1')\n",
"\n",
"pd.set_option('display.max_rows',200)"
]
@@ -341,6 +340,7 @@
"# ---- Save it\n",
"#\n",
"fidle.utils.mkdir(output_dir)\n",
+ "print('Output directory :',output_dir)\n",
"\n",
"filedata = f'{output_dir}/{dataset_filename}'\n",
"filedesc = f'{output_dir}/{description_filename}'\n",
@@ -374,11 +374,8 @@
}
],
"metadata": {
- "interpreter": {
- "hash": "8e38643e33497db9a306e3f311fa98cb1e65371278ca73ee4ea0c76aa5a4f387"
- },
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -392,7 +389,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/SYNOP/SYNOP2-First-predictions.ipynb b/SYNOP/SYNOP2-First-predictions.ipynb
index 83b8b58..38ca46a 100644
--- a/SYNOP/SYNOP2-First-predictions.ipynb
+++ b/SYNOP/SYNOP2-First-predictions.ipynb
@@ -49,11 +49,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/SYNOP'\n",
- "run_id, run_dir, datasets_dir = fidle.init('SYNOP2', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('SYNOP2')"
]
},
{
@@ -71,7 +70,7 @@
"source": [
"# ---- About dataset (no need to change)\n",
"#\n",
- "dataset_dir = './data' # Enhanced dataset is very small, so ./data in a good choice :-)\n",
+ "enhanced_dir = './data' # Enhanced dataset is very small, so ./data in a good choice :-)\n",
"dataset_filename = 'synop-LYS.csv'\n",
"schema_filename = 'synop.json'\n",
"features = ['tend', 'cod_tend', 'dd', 'ff', 'td', 'u', 'ww', 'pres', 'rafper', 'rr1', 'rr3', 'tc']\n",
@@ -79,7 +78,7 @@
"\n",
"# ---- About training (Can be changed !)\n",
"#\n",
- "scale = 1 # Percentage of dataset to be used (1=all)\n",
+ "scale = .2 # Percentage of dataset to be used (1=all)\n",
"train_prop = .8 # Percentage for train (the rest being for the test)\n",
"sequence_len = 16\n",
"batch_size = 32\n",
@@ -119,7 +118,7 @@
"source": [
"# ---- Read dataset from ./data\n",
"\n",
- "df = pd.read_csv(f'{dataset_dir}/{dataset_filename}', header=0, sep=';')\n",
+ "df = pd.read_csv(f'{enhanced_dir}/{dataset_filename}', header=0, sep=';')\n",
"\n",
"# ---- Scaling\n",
"\n",
@@ -265,7 +264,8 @@
"metadata": {},
"outputs": [],
"source": [
- "pwk.chrono_start()\n",
+ "chrono = fidle.Chrono()\n",
+ "chrono.start()\n",
"\n",
"history=model.fit(train_generator, \n",
" epochs = epochs, \n",
@@ -273,7 +273,7 @@
" validation_data = test_generator,\n",
" callbacks = [bestmodel_callback])\n",
"\n",
- "pwk.chrono_show()"
+ "chrono.show()"
]
},
{
@@ -398,11 +398,8 @@
}
],
"metadata": {
- "interpreter": {
- "hash": "8e38643e33497db9a306e3f311fa98cb1e65371278ca73ee4ea0c76aa5a4f387"
- },
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -416,7 +413,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/SYNOP/SYNOP3-12h-predictions.ipynb b/SYNOP/SYNOP3-12h-predictions.ipynb
index 682d6e8..e543389 100644
--- a/SYNOP/SYNOP3-12h-predictions.ipynb
+++ b/SYNOP/SYNOP3-12h-predictions.ipynb
@@ -47,11 +47,10 @@
"\n",
"from importlib import reload\n",
"\n",
- "# Init Fidle environment\n",
"import fidle\n",
"\n",
- "run_dir = './run/SYNOP'\n",
- "run_id, run_dir, datasets_dir = fidle.init('SYNOP3', run_dir)"
+ "# Init Fidle environment\n",
+ "run_id, run_dir, datasets_dir = fidle.init('SYNOP3')"
]
},
{
@@ -69,7 +68,8 @@
"source": [
"# ---- About dataset (no need to change)\n",
"#\n",
- "dataset_dir = './data' # Enhanced dataset is very small, so ./data in a good choice :-)\n",
+ "enhanced_dir = './data' # Enhanced dataset is very small, so ./data in a good choice :-)\n",
+ "models_dir = './run/SYNOP2/' # Where to get our previously trained models\n",
"dataset_filename = 'synop-LYS.csv'\n",
"schema_filename = 'synop.json'\n",
"features = ['tend', 'cod_tend', 'dd', 'ff', 'td', 'u', 'ww', 'pres', 'rafper', 'rr1', 'rr3', 'tc']\n",
@@ -116,7 +116,7 @@
"source": [
"# ---- Read dataset\n",
"\n",
- "df = pd.read_csv(f'{dataset_dir}/{dataset_filename}', header=0, sep=';')\n",
+ "df = pd.read_csv(f'{enhanced_dir}/{dataset_filename}', header=0, sep=';')\n",
"\n",
"# ---- Scaling\n",
"\n",
@@ -158,7 +158,7 @@
"metadata": {},
"outputs": [],
"source": [
- "loaded_model = tf.keras.models.load_model(f'{run_dir}/best_model.h5')"
+ "loaded_model = tf.keras.models.load_model(f'{models_dir}/best_model.h5')"
]
},
{
@@ -308,7 +308,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "Python 3.9.2 ('fidle-env')",
"language": "python",
"name": "python3"
},
@@ -322,7 +322,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.9.2"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
+ }
}
},
"nbformat": 4,
diff --git a/fidle/about.yml b/fidle/about.yml
index 46d8091..21ca862 100644
--- a/fidle/about.yml
+++ b/fidle/about.yml
@@ -13,7 +13,7 @@
#
# This file describes the notebooks used by the Fidle training.
-version: 2.02b3
+version: 2.02b4
content: notebooks
name: Notebooks Fidle
description: All notebooks used by the Fidle training
--
GitLab