From c3cf7448d96d15d863737c6f7df6c01d9d44718e Mon Sep 17 00:00:00 2001
From: Jean-Luc Parouty <Jean-Luc.Parouty@simap.grenoble-inp.fr>
Date: Mon, 14 Mar 2022 15:06:05 +0100
Subject: [PATCH] Add run_if_exist options for ci
---
fidle/02-running-ci-tests.ipynb | 15 ++++++++++-----
fidle/{batch-tests-slurm.sh => batch-ci.sh} | 7 +++++--
fidle/ci/smart_gpu.yml | 2 +-
fidle/cookci.py | 14 ++++++++------
4 files changed, 24 insertions(+), 14 deletions(-)
rename fidle/{batch-tests-slurm.sh => batch-ci.sh} (94%)
diff --git a/fidle/02-running-ci-tests.ipynb b/fidle/02-running-ci-tests.ipynb
index 128f2b9..4889877 100644
--- a/fidle/02-running-ci-tests.ipynb
+++ b/fidle/02-running-ci-tests.ipynb
@@ -33,7 +33,8 @@
"source": [
"## Step 2 - Profile and parameters\n",
"`profile_name`: ci profile name - see en ./ci \n",
- "`reset`: reset or not the catalog of results \n",
+ "`reset`: reset the catalog.json file of results (False) \n",
+ "`run_if_exist`: run if html output exist (False) \n",
"`filters`: regex to define witch notebook will be run, examples : \n",
"- `.*`\n",
"- `Nb_GTSRB.*|Nb_AE.*`\n",
@@ -47,10 +48,11 @@
"outputs": [],
"source": [
"profile_name = './ci/small_cpu.yml'\n",
- "reset = True\n",
- "filter = '.*'\n",
+ "reset = False\n",
+ "run_if_exist = False\n",
+ "filter = 'No_one.*'\n",
"\n",
- "pwk.override('profile_name', 'reset', 'filter')"
+ "pwk.override('profile_name', 'reset', 'run_if_exist', 'filter')"
]
},
{
@@ -66,7 +68,10 @@
"metadata": {},
"outputs": [],
"source": [
- "cookci.run_profile(profile_name, reset, filter)\n",
+ "cookci.run_profile( profile_name, \n",
+ " reset = reset, \n",
+ " run_if_exist = run_if_exist, \n",
+ " filter = filter )\n",
"cookci.build_ci_report(profile_name)"
]
},
diff --git a/fidle/batch-tests-slurm.sh b/fidle/batch-ci.sh
similarity index 94%
rename from fidle/batch-tests-slurm.sh
rename to fidle/batch-ci.sh
index 8ed56d8..b44e016 100644
--- a/fidle/batch-tests-slurm.sh
+++ b/fidle/batch-ci.sh
@@ -32,8 +32,11 @@ NOTEBOOK_DIR="$WORK/fidle/fidle"
export FIDLE_OVERRIDE_RUNCI_profile_name='./ci/smart_gpu.yml'
-# Reset or not report file (./run/done/report.json)
-export FIDLE_OVERRIDE_RUNCI_reset='True'
+# Reset report file (./run/done/report.json)
+export FIDLE_OVERRIDE_RUNCI_reset='False'
+
+# Run if html output still exist
+export FIDLE_OVERRIDE_RUNCI_run_if_exist='False'
# Notebooks to run (regex)
export FIDLE_OVERRIDE_RUNCI_filter='.*'
diff --git a/fidle/ci/smart_gpu.yml b/fidle/ci/smart_gpu.yml
index 395a1bd..fc8cd71 100644
--- a/fidle/ci/smart_gpu.yml
+++ b/fidle/ci/smart_gpu.yml
@@ -519,7 +519,7 @@ Nb_VAE9_r3:
Nb_VAE10:
notebook_id: VAE10
notebook_dir: VAE
- notebook_src: 010-VAE-with-CelebA-post.ipynb
+ notebook_src: 10-VAE-with-CelebA-post.ipynb
notebook_tag: default
overrides:
run_dir: default
diff --git a/fidle/cookci.py b/fidle/cookci.py
index 1bf5026..3c7c7ba 100644
--- a/fidle/cookci.py
+++ b/fidle/cookci.py
@@ -49,7 +49,7 @@ def load_profile(filename):
return profile
-def run_profile(profile_name, reset=False, filter=r'.*', top_dir='..'):
+def run_profile(profile_name, reset=False, run_if_exist=False, filter=r'.*', top_dir='..'):
'''
Récupère la liste des notebooks et des paramètres associés,
décrit dans le profile, et pour chaque notebook :
@@ -128,6 +128,11 @@ def run_profile(profile_name, reset=False, filter=r'.*', top_dir='..'):
else:
output_name = notebook_name + notebook_tag
+ # ---- Run if exist ---------------------------------------------------
+ #
+ done_html = os.path.abspath( f'{top_dir}/{output_html}/{notebook_dir}/{output_name}.html' )
+ if (os.path.isfile(done_html) is True) and (run_if_exist is False) : continue
+
# ---- Go to the right place ------------------------------------------
#
os.chdir(f'{top_dir}/{notebook_dir}')
@@ -155,10 +160,6 @@ def run_profile(profile_name, reset=False, filter=r'.*', top_dir='..'):
#
os.chdir(f'{top_dir}/{notebook_dir}')
- # ---- Read notebook
- #
- notebook = nbformat.read( f'{notebook_src}', nbformat.NO_CONVERT)
-
# ---- Top chrono - Start
#
chrono_start('nb')
@@ -168,6 +169,7 @@ def run_profile(profile_name, reset=False, filter=r'.*', top_dir='..'):
#
print(' - Run notebook...',end='')
try:
+ notebook = nbformat.read( f'{notebook_src}', nbformat.NO_CONVERT)
ep = ExecutePreprocessor(timeout=6000, kernel_name="python3")
ep.preprocess(notebook)
except CellExecutionError as e:
@@ -461,7 +463,7 @@ def build_ci_report(profile_name, top_dir='..'):
state = entry['state']
cols = []
- cols.append( f'<a href="{dir}" target="_blank">{dir}</a>' )
+ cols.append( f'<a href="{dir}/{out}" target="_blank">{dir}</a>' )
cols.append( f'<a href="{dir}/{out}" target="_blank">{id}</a>' )
cols.append( f'<a href="{dir}/{out}" target="_blank">{src}</a>' )
cols.append( start )
--
GitLab