From b43dbe2abc14c41c7f2c02a011426eb4a41a0c04 Mon Sep 17 00:00:00 2001
From: Jean-Luc Parouty <Jean-Luc.Parouty@simap.grenoble-inp.fr>
Date: Tue, 2 Nov 2021 09:26:11 +0100
Subject: [PATCH] Update CI stuffs
---
fidle/02-running-ci-tests.ipynb | 26 ++++++++++++++++----------
fidle/batch-tests-slurm.sh | 4 +++-
fidle/ci/smart_gpu.yml | 20 ++++++++++----------
3 files changed, 29 insertions(+), 21 deletions(-)
diff --git a/fidle/02-running-ci-tests.ipynb b/fidle/02-running-ci-tests.ipynb
index c97ff3d..b8b5b07 100644
--- a/fidle/02-running-ci-tests.ipynb
+++ b/fidle/02-running-ci-tests.ipynb
@@ -22,14 +22,22 @@
"outputs": [],
"source": [
"import cookci\n",
- "import os"
+ "import os\n",
+ "import pwk\n",
+ "datasets_dir = pwk.init('RUNCI')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Step 2 - Profile"
+ "## Step 2 - Profile and parameters\n",
+ "`profile_name`: ci profile name - see en ./ci \n",
+ "`reset`: reset or not the catalog of results \n",
+ "`filters`: regex to define witch notebook will be run, examples : \n",
+ "- `.*`\n",
+ "- `Nb_GTSRB.*|Nb_AE.*`\n",
+ "- `Nb_VAE3`"
]
},
{
@@ -39,11 +47,10 @@
"outputs": [],
"source": [
"profile_name = './ci/small_cpu.yml'\n",
+ "reset = False\n",
+ "filter = 'Nb_LINR1'\n",
"\n",
- "reset=True\n",
- "filter='.*'\n",
- "\n",
- "profile_name = os.getenv('FIDLE_OVERRIDE_PROFILE', profile_name )"
+ "pwk.override('profile_name', 'reset', 'filter')"
]
},
{
@@ -71,7 +78,7 @@
"source": [
"%%bash\n",
"tree ./run/ci\n",
- "find .. -name \"*==ci==.*\" -ls | sort -k11\n",
+ "# find .. -name \"*==ci==.*\" -ls | sort -k11\n",
"# rm $(find .. -name \"*==ci==.ipynb\")\n",
"# rm $(find .. -name \"*==ci==.html\")"
]
@@ -87,11 +94,10 @@
],
"metadata": {
"interpreter": {
- "hash": "8e38643e33497db9a306e3f311fa98cb1e65371278ca73ee4ea0c76aa5a4f387"
+ "hash": "7822d55dc7294a4f6f06b86d8ad2ca65bd6e1ee5d72628c47c30a06bbf89aef6"
},
"kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
+ "display_name": "Python 3.9.7 64-bit ('fidle': conda)",
"name": "python3"
},
"language_info": {
diff --git a/fidle/batch-tests-slurm.sh b/fidle/batch-tests-slurm.sh
index d3856d3..52a406f 100644
--- a/fidle/batch-tests-slurm.sh
+++ b/fidle/batch-tests-slurm.sh
@@ -30,7 +30,9 @@
MODULE_ENV="tensorflow-gpu/py3/2.5.0"
NOTEBOOK_DIR="$WORK/fidle/fidle"
-FIDLE_OVERRIDE_PROFILE="./ci/small_cpu.yml"
+FIDLE_OVERRIDE_RUNCI_profile_name='./ci/small_cpu.yml'
+FIDLE_OVERRIDE_RUNCI_reset='False'
+FIDLE_OVERRIDE_RUNCI_filter='Nb_GTSRB.*|Nb_AE.*'
NOTEBOOK_SRC1="02-running-ci-tests.ipynb"
diff --git a/fidle/ci/smart_gpu.yml b/fidle/ci/smart_gpu.yml
index 51f7b47..45fd3c6 100644
--- a/fidle/ci/smart_gpu.yml
+++ b/fidle/ci/smart_gpu.yml
@@ -90,7 +90,7 @@ Nb_GTSRB2:
notebook_src: 02-First-convolutions.ipynb
notebook_tag: default
overrides:
- run_dir: ./run/GTSRB2_done
+ run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
dataset_name: set-24x24-L
batch_size: 64
@@ -104,7 +104,7 @@ Nb_GTSRB3:
notebook_src: 03-Tracking-and-visualizing.ipynb
notebook_tag: default
overrides:
- run_dir: ./run/GTSRB3_done
+ run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
dataset_name: set-24x24-L
batch_size: 64
@@ -118,7 +118,7 @@ Nb_GTSRB4:
notebook_src: 04-Data-augmentation.ipynb
notebook_tag: default
overrides:
- run_dir: ./run/GTSRB4_done
+ run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
dataset_name: set-24x24-L
batch_size: 64
@@ -130,7 +130,7 @@ Nb_GTSRB5_r1:
notebook_id: GTSRB5
notebook_dir: GTSRB
notebook_src: 05-Full-convolutions.ipynb
- notebook_tag: =1==ci==
+ notebook_tag: =1==done==
overrides:
run_dir: ./run/GTSRB5_done
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
@@ -146,9 +146,9 @@ Nb_GTSRB5_r2:
notebook_id: GTSRB5
notebook_dir: GTSRB
notebook_src: 05-Full-convolutions.ipynb
- notebook_tag: =1==ci==
+ notebook_tag: =2==done==
overrides:
- run_dir: ./run/GTSRB5_done
+ run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
datasets: "['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']"
models: "{'v1':'get_model_v1', 'v2':'get_model_v2', 'v3':'get_model_v3'}"
@@ -162,9 +162,9 @@ Nb_GTSRB5_r3:
notebook_id: GTSRB5
notebook_dir: GTSRB
notebook_src: 05-Full-convolutions.ipynb
- notebook_tag: =2==ci==
+ notebook_tag: =3==done==
overrides:
- run_dir: ./run/GTSRB5_done
+ run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
datasets: "['set-48x48-L', 'set-48x48-RGB']"
models: "{'v2':'get_model_v2', 'v3':'get_model_v3'}"
@@ -186,8 +186,8 @@ Nb_GTSRB7:
notebook_src: 07-Show-report.ipynb
notebook_tag: default
overrides:
- run_dir: ./run/GTSRB7_done
- report_dir: ./run/GTSRB5_done
+ run_dir: default
+ report_dir: ./run/GTSRB5
#
# ------ IMDB ------------------------------------------------------
#
--
GitLab