From bddc2ac0d9861dd09b7b2f99101bd63ea165ecb0 Mon Sep 17 00:00:00 2001
From: Jean-Luc Parouty <Jean-Luc.Parouty@grenoble-inp.fr>
Date: Wed, 6 Jan 2021 19:01:44 +0100
Subject: [PATCH] Add new beautifull overriding system :-)
---
VAE/08-VAE-with-CelebA.ipynb | 88 +++++++++++--------------
VAE/batch_slurm.sh | 54 ++++++++-------
VAE/modules/callbacks.py | 2 +-
fidle/log/finished.json | 10 ++-
fidle/pwk.py | 124 ++++++++++++++++++++++++++---------
5 files changed, 168 insertions(+), 110 deletions(-)
diff --git a/VAE/08-VAE-with-CelebA.ipynb b/VAE/08-VAE-with-CelebA.ipynb
index 01abdcd..ad89e9b 100644
--- a/VAE/08-VAE-with-CelebA.ipynb
+++ b/VAE/08-VAE-with-CelebA.ipynb
@@ -37,7 +37,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [
{
@@ -117,15 +117,14 @@
"text": [
"Version : 0.6.1 DEV\n",
"Notebook id : VAE8\n",
- "Run time : Tuesday 5 January 2021, 14:49:56\n",
- "TensorFlow version : 2.4.0\n",
- "Keras version : 2.4.0\n",
- "Datasets dir : /gpfswork/rech/mlh/uja62cb/datasets\n",
- "Run dir : ./run/CelebA-s.001\n",
- "CI running mode : none\n",
+ "Run time : Wednesday 6 January 2021, 18:46:14\n",
+ "TensorFlow version : 2.2.0\n",
+ "Keras version : 2.3.0-tf\n",
+ "Datasets dir : /home/pjluc/datasets/fidle\n",
+ "Run dir : ./run/CelebA.001\n",
"Update keras cache : False\n",
"Save figs : True\n",
- "Path figs : ./run/CelebA-s.001/figs\n"
+ "Path figs : ./run/CelebA.001/figs\n"
]
},
{
@@ -145,8 +144,8 @@
"output_type": "stream",
"text": [
"Version : 1.2\n",
- "TensorFlow version : 2.4.0\n",
- "Keras version : 2.4.0\n"
+ "TensorFlow version : 2.2.0\n",
+ "Keras version : 2.3.0-tf\n"
]
},
{
@@ -166,8 +165,8 @@
"output_type": "stream",
"text": [
"Version : 0.4.1\n",
- "TensorFlow version : 2.4.0\n",
- "Keras version : 2.4.0\n"
+ "TensorFlow version : 2.2.0\n",
+ "Keras version : 2.3.0-tf\n"
]
}
],
@@ -190,7 +189,7 @@
"sys.path.append('..')\n",
"import fidle.pwk as pwk\n",
"\n",
- "run_dir = './run/CelebA-s.001' # Output directory\n",
+ "run_dir = './run/CelebA.001' # Output directory\n",
"datasets_dir = pwk.init('VAE8', run_dir)\n",
"\n",
"VAE.about()\n",
@@ -199,17 +198,9 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "rm: cannot remove './run/CelebA-s.001/logs': No such file or directory\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# To clean run_dir, uncomment and run this next line\n",
"# ! rm -r \"$run_dir\"/images-* \"$run_dir\"/logs \"$run_dir\"/figs \"$run_dir\"/models ; rmdir \"$run_dir\""
@@ -233,24 +224,30 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# ---- For tests\n",
- "# scale = 0.3\n",
- "# image_size = (128,128)\n",
- "# enhanced_dir = './data'\n",
+ "scale = 0.3\n",
+ "image_size = (128,128)\n",
+ "enhanced_dir = './data'\n",
+ "latent_dim = 300\n",
+ "r_loss_factor = 0.6\n",
"\n",
"# ---- Training with a full dataset\n",
- "scale = 1.\n",
- "image_size = (128,128)\n",
- "enhanced_dir = f'{datasets_dir}/celeba/enhanced'\n",
+ "# scale = 1.\n",
+ "# image_size = (128,128)\n",
+ "# enhanced_dir = f'{datasets_dir}/celeba/enhanced'\n",
+ "# latent_dim = 300\n",
+ "# r_loss_factor = 0.6\n",
"\n",
"# ---- Training with a full dataset of large images\n",
- "# scale = 1.\n",
- "# image_size = (192,160)\n",
- "# enhanced_dir = f'{datasets_dir}/celeba/enhanced'"
+ "# scale = 1.\n",
+ "# image_size = (192,160)\n",
+ "# enhanced_dir = f'{datasets_dir}/celeba/enhanced'\n",
+ "# latent_dim = 300\n",
+ "# r_loss_factor = 0.6\n"
]
},
{
@@ -262,23 +259,21 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Train directory is : /gpfswork/rech/mlh/uja62cb/datasets/celeba/enhanced/clusters-128x128\n"
+ "Train directory is : ./data/clusters-128x128\n"
]
}
],
"source": [
- "# ---- Used for continous integration or batch mode - Just forget this 2 lines\n",
+ "# ---- Override parameters (batch mode) - Just forget this line\n",
"#\n",
- "scale = pwk.override('scale', scale)\n",
- "image_size = pwk.override('image_size', image_size)\n",
- "enhanced_dir = pwk.override('enhanced_dir', enhanced_dir)\n",
+ "pwk.override()\n",
"\n",
"# ---- the place of the clusters files\n",
"#\n",
@@ -296,14 +291,14 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Data generator is ready with : 6331 batchs of 32 images, or 202599 images\n"
+ "Data generator is ready with : 379 batchs of 32 images, or 12155 images\n"
]
}
],
@@ -320,15 +315,6 @@
"## Step 3 - Build model"
]
},
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [],
- "source": [
- "latent_dim = 300"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -422,7 +408,7 @@
}
],
"source": [
- "vae = VAE(encoder, decoder, r_loss_factor=0.6)\n",
+ "vae = VAE(encoder, decoder, r_loss_factor)\n",
"\n",
"vae.compile(optimizer=keras.optimizers.Adam())"
]
diff --git a/VAE/batch_slurm.sh b/VAE/batch_slurm.sh
index 8a66cdd..35535d3 100755
--- a/VAE/batch_slurm.sh
+++ b/VAE/batch_slurm.sh
@@ -1,16 +1,4 @@
#!/bin/bash
-
-#SBATCH --job-name="VAE" # nom du job
-#SBATCH --ntasks=1 # nombre de tâche (un unique processus ici)
-#SBATCH --gres=gpu:1 # nombre de GPU à réserver (un unique GPU ici)
-#SBATCH --cpus-per-task=10 # nombre de coeurs à réserver (un quart du noeud)
-#SBATCH --hint=nomultithread # on réserve des coeurs physiques et non logiques
-#SBATCH --time=01:00:00 # temps exécution maximum demande (HH:MM:SS)
-#SBATCH --output="VAE_%j.out" # nom du fichier de sortie
-#SBATCH --error="VAE_%j.err" # nom du fichier d'erreur (ici commun avec la sortie)
-#SBATCH --mail-user=Jean-Luc.Parouty@grenoble-inp.fr
-#SBATCH --mail-type=ALL
-
# -----------------------------------------------
# _ _ _
# | |__ __ _| |_ ___| |__
@@ -27,25 +15,44 @@
# Soumission : sbatch /(...)/fidle/VAE/batch_slurm.sh
# Suivi : squeue -u $USER
-# ---- Parameters -------------------------------
+# ==== Job parameters ==============================================
+
+#SBATCH --job-name="VAE" # nom du job
+#SBATCH --ntasks=1 # nombre de tâche (un unique processus ici)
+#SBATCH --gres=gpu:1 # nombre de GPU à réserver (un unique GPU ici)
+#SBATCH --cpus-per-task=10 # nombre de coeurs à réserver (un quart du noeud)
+#SBATCH --hint=nomultithread # on réserve des coeurs physiques et non logiques
+#SBATCH --time=01:00:00 # temps exécution maximum demande (HH:MM:SS)
+#SBATCH --output="VAE_%j.out" # nom du fichier de sortie
+#SBATCH --error="VAE_%j.err" # nom du fichier d'erreur (ici commun avec la sortie)
+#SBATCH --mail-user=Jean-Luc.Parouty@grenoble-inp.fr
+#SBATCH --mail-type=ALL
+
+# ==== Notebook parameters =========================================
MODULE_ENV="tensorflow-gpu/py3/2.4.0"
NOTEBOOK_DIR="$WORK/fidle/VAE"
+# ---- VAE MNIST
+#
# NOTEBOOK_SRC="01-VAE-with-MNIST.ipynb"
-# FIDLE_RUN_DIR="./run/MNIST.$SLURM_JOB_ID"
-
+# FIDLE_OVERRIDE_VAE8_run_dir="./run/MNIST.$SLURM_JOB_ID"
+# ---- VAE CelebA
+#
NOTEBOOK_SRC="08-VAE-with-CelebA.ipynb"
+#
+export FIDLE_OVERRIDE_VAE8_run_dir="./run/CelebA.$SLURM_JOB_ID"
+export FIDLE_OVERRIDE_VAE8_scale="0.05"
+export FIDLE_OVERRIDE_VAE8_image_size="(128,128)"
+export FIDLE_OVERRIDE_VAE8_enhanced_dir='{datasets_dir}/celeba/enhanced'
+export FIDLE_OVERRIDE_VAZ8_r_loss_factor="0.5"
-FIDLE_OVERRIDE_VAE8_run_dir="./run/CelebA.$SLURM_JOB_ID"
-FIDLE_OVERRIDE_VAE8_scale="0.05"
-FIDLE_OVERRIDE_VAE8_image_size="(128,128)"
-FIDLE_OVERRIDE_VAE8_enhanced_dir='{datasets_dir}/celeba/enhanced'
-
+# ---- By default (no need to modify)
+#
NOTEBOOK_OUT="${NOTEBOOK_SRC%.*}==${SLURM_JOB_ID}==.ipynb"
-# ------------------------------------------------
+# ==================================================================
echo '------------------------------------------------------------'
echo "Start : $0"
@@ -57,10 +64,10 @@ echo '------------------------------------------------------------'
echo "Notebook dir : $NOTEBOOK_DIR"
echo "Notebook src : $NOTEBOOK_SRC"
echo "Notebook out : $NOTEBOOK_OUT"
-echo "Run dir : $FIDLE_OVERRIDE_VAE8_run_dir"
echo "Environment : $MODULE_ENV"
echo '------------------------------------------------------------'
-
+env | grep FIDLE_OVERRIDE | awk 'BEGIN { FS = "=" } ; { printf("%-35s : %s\n",$1,$2) }'
+echo '------------------------------------------------------------'
# ---- Module
@@ -70,7 +77,6 @@ module load "$MODULE_ENV"
# ---- Run it...
cd $NOTEBOOK_DIR
-export FIDLE_RUN_DIR
jupyter nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --output "$NOTEBOOK_OUT" --execute "$NOTEBOOK_SRC"
diff --git a/VAE/modules/callbacks.py b/VAE/modules/callbacks.py
index 25d12d3..2248251 100644
--- a/VAE/modules/callbacks.py
+++ b/VAE/modules/callbacks.py
@@ -76,4 +76,4 @@ class BestModelCallback(Callback):
if current < self.loss:
self.loss = current
self.model.save(self.filename)
- print('Saved - loss=',current)
+ print(f'Saved - loss={current:.6f}')
diff --git a/fidle/log/finished.json b/fidle/log/finished.json
index 06288e3..0acd1f6 100644
--- a/fidle/log/finished.json
+++ b/fidle/log/finished.json
@@ -150,8 +150,8 @@
"duration": "00:00:08 736ms"
},
"VAE8": {
- "path": "/gpfsdswork/projects/rech/mlh/uja62cb/fidle/VAE",
- "start": "Wednesday 6 January 2021, 12:17:53",
+ "path": "/home/pjluc/dev/fidle/VAE",
+ "start": "Wednesday 6 January 2021, 18:46:14",
"end": "",
"duration": "Unfinished..."
},
@@ -160,5 +160,11 @@
"start": "Tuesday 5 January 2021, 15:22:23",
"end": "Tuesday 5 January 2021, 15:23:32",
"duration": "00:01:10 597ms"
+ },
+ "VAE12": {
+ "path": "/home/pjluc/dev/fidle/VAE",
+ "start": "Wednesday 6 January 2021, 18:29:53",
+ "end": "",
+ "duration": "Unfinished..."
}
}
\ No newline at end of file
diff --git a/fidle/pwk.py b/fidle/pwk.py
index fea2934..27d2bbf 100644
--- a/fidle/pwk.py
+++ b/fidle/pwk.py
@@ -10,7 +10,7 @@
# A simple module to host some common functions for practical work
# Jean-Luc Parouty 2020
-import os
+import os,sys
import glob
import shutil
from datetime import datetime
@@ -80,7 +80,7 @@ def init(name=None, run_dir='./run'):
# ---- run_dir
#
- run_dir = override('run_dir',run_dir)
+ override('run_dir')
mkdir(run_dir)
# ---- Update Keras cache
@@ -152,46 +152,106 @@ def error_datasets_not_found():
display_md('----')
assert False, 'datasets folder not found, please set FIDLE_DATASETS_DIR env var.'
-
-# -------------------------------------------------------------
-# param_override
-# -------------------------------------------------------------
-# Try to override a given parameter 'param'.
-#
-def override(name, value):
+
+
+def override(*names):
'''
- Try to override a given parameter (name,value) with an environment variable.
- Env variable name is : FIDLE_OVERRIDE_<NOTEBOOK-ID>_<NAME>
- If no env variable is available, return the given value.
- If type is str, substitution is done with notebook_id and datasets_dir
+ Try to override attributes given par name with environment variables.
+ Environment variables name must be : FIDLE_OVERRIDE_<NOTEBOOK-ID>_<NAME>
+ If no env variable is available for a given name, nothing is change.
+ If type is str, substitution is done with 'notebook_id' and 'datasets_dir'
+ Example : override('image_size','nb_epochs')
params:
- name : parameter name
- value: parameter value
+ names : list of attributes names as a str list
+ if empty, all attributes can be override
return :
- eval(env variable), if it env variable exist, or given value
+ nothing
'''
- # ---- Environment variable name
+ # ---- Where to override
#
- env_name = f'FIDLE_OVERRIDE_{notebook_id}_{name}'
- env_value = os.environ.get(env_name)
+ main=sys.modules['__main__']
- # ---- Doesn't exist ?
+ # ---- No names : mean all
#
- if env_value is None:
- return value
-
- # ---- Exist
+ if len(names)==0:
+ names=[]
+ for name in dir(main):
+ if name.startswith('_'): continue
+ v=getattr(main,name)
+ if type(v) not in [str, int, float, bool, tuple, list, dict]: continue
+ names.append(name)
+
+ # ---- Search for names
#
- if isinstance(value, str) :
- new_value = env_value.format(datasets_dir=datasets_dir, notebook_id=notebook_id)
+ for name in names:
+
+ # ---- Environment variable name
+ #
+ env_name = f'FIDLE_OVERRIDE_{notebook_id}_{name}'
+ env_value = os.environ.get(env_name)
+
+ # ---- Environment variable : Doesn't exist
+ #
+ if env_value is None: continue
+
+ # ---- Environment variable : Exist
+ #
+ value_old = getattr(main,name)
+ value_type = type(value_old)
+
+ if value_type in [ str ] :
+ new_value = env_value.format(datasets_dir=datasets_dir, notebook_id=notebook_id)
+
+ if value_type in [ int, float, bool, tuple, list, dict]:
+ new_value = eval(env_value)
+
+ # ---- Override value
+ #
+ setattr(main,name,new_value)
+ print(f'Override : Attribute [{name}={value_old}] with [{new_value}]')
+
+
+
+
+# -------------------------------------------------------------
+# param_override
+# -------------------------------------------------------------
+# Try to override a given parameter 'param'.
+#
+# def override(name, value):
+# '''
+# Try to override a given parameter (name,value) with an environment variable.
+# Env variable name is : FIDLE_OVERRIDE_<NOTEBOOK-ID>_<NAME>
+# If no env variable is available, return the given value.
+# If type is str, substitution is done with notebook_id and datasets_dir
+# params:
+# name : parameter name
+# value: parameter value
+# return :
+# eval(env variable), if it env variable exist, or given value
+# '''
+# # ---- Environment variable name
+# #
+# env_name = f'FIDLE_OVERRIDE_{notebook_id}_{name}'
+# env_value = os.environ.get(env_name)
+
+# # ---- Doesn't exist ?
+# #
+# if env_value is None:
+# return value
+
+# # ---- Exist
+# #
+# if isinstance(value, str) :
+# new_value = env_value.format(datasets_dir=datasets_dir, notebook_id=notebook_id)
- if type(value) in [ tuple, int, float]:
- new_value = eval(env_value)
+# if type(value) in [ tuple, int, float]:
+# new_value = eval(env_value)
- # ---- Return
- #
- print(f'Override : Parameter [{name}={value}] set to [{new_value}]')
- return new_value
+# # ---- Return
+# #
+# print(f'Override : Parameter [{name}={value}] set to [{new_value}]')
+# return new_value
# -------------------------------------------------------------
--
GitLab