Skip to content
Snippets Groups Projects
Commit be9e742b authored by Jean-Luc Parouty's avatar Jean-Luc Parouty
Browse files

Add ci_ERROR.txt for ci tests

parent b0e3ba38
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
fidle/02-running-ci-tests.ipynb
+ 1
− 1
View file @ be9e742b
......@@ -125,7 +125,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.7"
"version": "3.8.5"
}
},
"nbformat": 4,
......
fidle/03-ci-report==done==.ipynb deleted 100644 → 0
+ 0
− 514
View file @ b0e3ba38
%% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
## Continuous integration report
Réalisation des tests : [02-running-ci-tests.ipynb](02-running-ci-tests.ipynb)
Résultats bruts : [./logs/ci_report.json](./logs/ci_report.json)
%% Cell type:code id: tags:
``` python
import cookci
cookci.build_ci_report(report_name='./logs/ci_report.json')
```
%% Output
### About :
**Version** : 1.0
**Output_Tag** : ==done==
**Save_Figs** : True
**Description** : Full profile for GPU
**Host** : r11i2n4
**Profile** : ./ci/full_gpu.yml
**Start** : 27/01/21 18:20:51
**End** : 27/01/21 20:34:28
**Duration** : 2:13:37
### Details :
<br>HTML report saved as : [./logs/ci_report.html](./logs/ci_report.html)
%% Cell type:markdown id: tags:
---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
%% Cell type:code id: tags:
``` python
```
fidle/ci/full_ci.yml 0 → 100644
+ 292
− 0
View file @ be9e742b
_metadata_:
version: '1.0'
output_tag: ==ci==
save_figs: true
description: Full profile for CI
#
# ------ LinearReg -------------------------------------------------
#
LINR1:
notebook_id: LINR1
notebook_dir: LinearReg
notebook_src: 01-Linear-Regression.ipynb
notebook_tag: default
GRAD1:
notebook_id: GRAD1
notebook_dir: LinearReg
notebook_src: 02-Gradient-descent.ipynb
notebook_tag: default
POLR1:
notebook_id: POLR1
notebook_dir: LinearReg
notebook_src: 03-Polynomial-Regression.ipynb
notebook_tag: default
LOGR1:
notebook_id: LOGR1
notebook_dir: LinearReg
notebook_src: 04-Logistic-Regression.ipynb
notebook_tag: default
PER57:
notebook_id: PER57
notebook_dir: IRIS
notebook_src: 01-Simple-Perceptron.ipynb
notebook_tag: default
#
# ------ BHPD ------------------------------------------------------
#
BHPD1:
notebook_id: BHPD1
notebook_dir: BHPD
notebook_src: 01-DNN-Regression.ipynb
notebook_tag: default
BHPD2:
notebook_id: BHPD2
notebook_dir: BHPD
notebook_src: 02-DNN-Regression-Premium.ipynb
notebook_tag: default
#
# ------ MNIST -----------------------------------------------------
#
MNIST1:
notebook_id: MNIST1
notebook_dir: MNIST
notebook_src: 01-DNN-MNIST.ipynb
notebook_tag: default
#
# ------ GTSRB -----------------------------------------------------
#
GTSRB1:
notebook_id: GTSRB1
notebook_dir: GTSRB
notebook_src: 01-Preparation-of-data.ipynb
notebook_tag: default
overrides:
scale: 0.02
output_dir: './data'
GTSRB2:
notebook_id: GTSRB2
notebook_dir: GTSRB
notebook_src: 02-First-convolutions.ipynb
notebook_tag: default
overrides:
run_dir: ./run/GTSRB2_ci
enhanced_dir: './data'
dataset_name: set-24x24-L
batch_size: 64
epochs: 5
scale: 1
GTSRB3:
notebook_id: GTSRB3
notebook_dir: GTSRB
notebook_src: 03-Tracking-and-visualizing.ipynb
notebook_tag: default
overrides:
run_dir: ./run/GTSRB3_ci
enhanced_dir: './data'
dataset_name: set-24x24-L
batch_size: 64
epochs: 5
scale: 1
GTSRB4:
notebook_id: GTSRB4
notebook_dir: GTSRB
notebook_src: 04-Data-augmentation.ipynb
notebook_tag: default
overrides:
run_dir: ./run/GTSRB4_ci
enhanced_dir: './data'
dataset_name: set-24x24-L
batch_size: 64
epochs: 5
scale: 1
GTSRB5_r1:
notebook_id: GTSRB5
notebook_dir: GTSRB
notebook_src: 05-Full-convolutions.ipynb
notebook_tag: =1==ci==
overrides:
run_dir: ./run/GTSRB5_ci
enhanced_dir: './data'
datasets: "['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']"
models: "{'v1':'get_model_v1', 'v2':'get_model_v2', 'v3':'get_model_v3'}"
batch_size: 64
epochs: 16
scale: 1
with_datagen: False
verbose: 0
GTSRB5_r2:
notebook_id: GTSRB5
notebook_dir: GTSRB
notebook_src: 05-Full-convolutions.ipynb
notebook_tag: =2==ci==
overrides:
run_dir: ./run/GTSRB5_ci
enhanced_dir: './data'
datasets: "['set-48x48-L', 'set-48x48-RGB']"
models: "{'v2':'get_model_v2', 'v3':'get_model_v3'}"
batch_size: 64
epochs: 16
scale: 1
with_datagen: False
verbose: 0
GTSRB6:
notebook_id: GTSRB6
notebook_dir: GTSRB
notebook_src: 06-Notebook-as-a-batch.ipynb
notebook_tag: default
GTSRB7:
notebook_id: GTSRB7
notebook_dir: GTSRB
notebook_src: 07-Show-report.ipynb
notebook_tag: default
overrides:
run_dir: ./run/GTSRB7_ci
report_dir: ./run/GTSRB5_ci
#
# ------ IMDB ------------------------------------------------------
#
IMDB1:
notebook_id: IMDB1
notebook_dir: IMDB
notebook_src: 01-Embedding-Keras.ipynb
notebook_tag: default
IMDB2:
notebook_id: IMDB1
notebook_dir: IMDB
notebook_src: 02-Prediction.ipynb
notebook_tag: default
IMDB3:
notebook_id: IMDB1
notebook_dir: IMDB
notebook_src: 03-LSTM-Keras.ipynb
notebook_tag: default
#
# ------ SYNOP -----------------------------------------------------
#
SYNOP1:
notebook_id: SYNOP1
notebook_dir: SYNOP
notebook_src: 01-Preparation-of-data.ipynb
notebook_tag: default
SYNOP2:
notebook_id: SYNOP2
notebook_dir: SYNOP
notebook_src: 02-First-predictions.ipynb
notebook_tag: default
overrides:
scale: 0.5
train_prop: 0.8
sequence_len: 16
batch_size: 32
epochs: 10
SYNOP3:
notebook_id: SYNOP3
notebook_dir: SYNOP
notebook_src: 03-12h-predictions.ipynb
notebook_tag: default
overrides:
iterations: 4
scale: 1
train_prop: 0.8
sequence_len: 16
batch_size: 32
epochs: 10
#
# ------ AE --------------------------------------------------------
#
AE1:
notebook_id: AE1
notebook_dir: AE
notebook_src: 01-AE-with-MNIST.ipynb
notebook_tag: default
AE2:
notebook_id: AE2
notebook_dir: AE
notebook_src: 02-AE-with-MNIST-post.ipynb
notebook_tag: default
#
# ------ VAE -------------------------------------------------------
#
VAE1:
notebook_id: VAE1
notebook_dir: VAE
notebook_src: 01-VAE-with-MNIST.ipynb
notebook_tag: default
overrides:
run_dir: ./run/VAE1_ci
scale: 1
latent_dim: 2
r_loss_factor: 0.994
batch_size: 64
epochs: 10
VAE2:
notebook_id: VAE2
notebook_dir: VAE
notebook_src: 02-VAE-with-MNIST-post.ipynb
notebook_tag: default
overrides:
run_dir: ./run/VAE1_ci
VAE5:
notebook_id: VAE5
notebook_dir: VAE
notebook_src: 05-About-CelebA.ipynb
notebook_tag: default
VAE6:
notebook_id: VAE6
notebook_dir: VAE
notebook_src: 06-Prepare-CelebA-datasets.ipynb
notebook_tag: default
overrides:
scale: 0.02
image_size: '(192,160)'
output_dir: ./data
exit_if_exist: False
VAE7:
notebook_id: VAE7
notebook_dir: VAE
notebook_src: 07-Check-CelebA.ipynb
notebook_tag: default
overrides:
image_size: '(192,160)'
enhanced_dir: './data'
VAE8:
notebook_id: VAE8
notebook_dir: VAE
notebook_src: 08-VAE-with-CelebA.ipynb
notebook_tag: default
overrides:
run_dir: ./run/VAE8_ci
scale: 1
image_size: '(192,160)'
enhanced_dir: './data'
latent_dim: 300
r_loss_factor: 0.6
batch_size: 64
epochs: 15
VAE9:
notebook_id: VAE9
notebook_dir: VAE
notebook_src: 09-VAE-with-CelebA-post.ipynb
notebook_tag: default
overrides:
run_dir: ./run/VAE8_ci
image_size: '(192,160)'
enhanced_dir: './data'
#
# ------ Misc ------------------------------------------------------
#
ACTF1:
notebook_id: ACTF1
notebook_dir: Misc
notebook_src: Activation-Functions.ipynb
notebook_tag: default
NP1:
notebook_id: NP1
notebook_dir: Misc
notebook_src: Numpy.ipynb
notebook_tag: default
TSB1:
notebook_id: TSB1
notebook_dir: Misc
notebook_src: Using-Tensorboard.ipynb
notebook_tag: default
fidle/ci/full_fad_s01.yml
+ 3
− 8
View file @ be9e742b
_metadata_:
version: '1.0'
output_tag: ==done==
output_tag: ==ci==
save_figs: true
description: Runned notebooks (done)
LINR1:
......@@ -8,16 +8,11 @@ LINR1:
notebook_dir: LinearReg
notebook_src: 01-Linear-Regression.ipynb
notebook_tag: default
GRAD1_r1:
GRAD1:
notebook_id: GRAD1
notebook_dir: LinearReg
notebook_src: 02-Gradient-descent.ipynb
notebook_tag: =1==done==
GRAD1_r2:
notebook_id: GRAD1
notebook_dir: LinearReg
notebook_src: 02-Gradient-descent.ipynb
notebook_tag: =2==done==
notebook_tag: default
POLR1:
notebook_id: POLR1
notebook_dir: LinearReg
......
fidle/cookci.py
+ 60
− 21
View file @ be9e742b
......@@ -33,6 +33,10 @@ VERSION = '1.0'
start_time = {}
end_time = {}
_report_filename = None
_error_filename = None
def get_default_profile(catalog=None, output_tag='==done==', save_figs=True):
'''
Return a default profile for continous integration.
......@@ -93,7 +97,7 @@ def load_profile(filename):
return profile
def run_profile(profile_name, report_name=None, top_dir='..'):
def run_profile(profile_name, report_name=None, error_name=None, top_dir='..'):
'''
Récupère la liste des notebooks et des paramètres associés,
décrit dans le profile, et pour chaque notebook :
......@@ -106,7 +110,7 @@ def run_profile(profile_name, report_name=None, top_dir='..'):
report_name : Nom du rapport json généré
top_dir : chemin relatif vers la racine fidle (..)
'''
print('\nRun profile session - FIDLE 2021')
print(f'Version : {VERSION}')
......@@ -119,15 +123,12 @@ def run_profile(profile_name, report_name=None, top_dir='..'):
notebooks = profile
del notebooks['_metadata_']
# ---- Create new ci_report
# ---- Report file
#
metadata = config
metadata['host'] = os.uname()[1]
metadata['profile'] = profile_name
if report_name is None:
report_name = config.CI_REPORT_JSON
report_name = os.path.abspath(report_name)
create_ci_report(report_name, metadata)
init_ci_report(report_name, error_name, metadata)
# ---- My place
#
......@@ -185,7 +186,7 @@ def run_profile(profile_name, report_name=None, top_dir='..'):
# ---- Top chrono
#
chrono_start('nb')
update_ci_report(report_name, run_id, notebook_id, notebook_dir, notebook_src, notebook_out, start=True)
update_ci_report(run_id, notebook_id, notebook_dir, notebook_src, notebook_out, start=True)
# ---- Try to run...
#
......@@ -207,7 +208,7 @@ def run_profile(profile_name, report_name=None, top_dir='..'):
# ---- Top chrono
#
chrono_stop('nb')
update_ci_report(report_name, run_id, notebook_id, notebook_dir, notebook_src, notebook_out, end=True, happy_end=happy_end)
update_ci_report(run_id, notebook_id, notebook_dir, notebook_src, notebook_out, end=True, happy_end=happy_end)
print(' Duration : ',chrono_get_delay('nb') )
# ---- Save notebook
......@@ -226,7 +227,7 @@ def run_profile(profile_name, report_name=None, top_dir='..'):
chrono_stop('main')
print('\nEnd of running process')
print(' Duration :', chrono_get_delay('main'))
complete_ci_report(report_name)
complete_ci_report()
def chrono_start(id='default'):
......@@ -259,27 +260,59 @@ def reset_chrono():
start_time, end_time = {},{}
def create_ci_report(filename, metadata, verbose=True):
def init_ci_report(report_filename, error_filename, metadata, verbose=True):
global _report_filename, _error_filename
# ---- Report filename
#
if report_filename is None:
report_filename = config.CI_REPORT_JSON
_report_filename = os.path.abspath(report_filename)
# ---- Error_filename
#
if error_filename is None:
error_filename = config.CI_ERROR_FILE
_error_filename = os.path.abspath(error_filename)
# ---- Create report
#
metadata['start']=chrono_get_start('main')
data={ '_metadata_':metadata }
with open(filename,'wt') as fp:
with open(_report_filename,'wt') as fp:
json.dump(data,fp,indent=4)
if verbose : print(f'\nCreate new ci report : {filename}')
if verbose : print(f'\nCreate new ci report : {_report_filename}')
def complete_ci_report(filename, verbose=True):
with open(filename) as fp:
# ---- Reset error
#
if os.path.exists(_error_filename):
os.remove(_error_filename)
if verbose : print(f'Remove error file : {_error_filename}')
def complete_ci_report(verbose=True):
global _report_filename, _error_filename
with open(_report_filename) as fp:
report = json.load(fp)
report['_metadata_']['end'] = chrono_get_end('main')
report['_metadata_']['duration'] = chrono_get_delay('main')
with open(filename,'wt') as fp:
with open(_report_filename,'wt') as fp:
json.dump(report,fp,indent=4)
if verbose : print(f'\nComplete ci report : {filename}')
def update_ci_report(filename, run_id, notebook_id, notebook_dir, notebook_src, notebook_out, start=False, end=False, happy_end=True):
if verbose : print(f'\nComplete ci report : {_report_filename}')
def update_ci_report(run_id, notebook_id, notebook_dir, notebook_src, notebook_out, start=False, end=False, happy_end=True):
global start_time, end_time
global _report_filename, _error_filename
# ---- Load it
with open(filename) as fp:
with open(_report_filename) as fp:
report = json.load(fp)
# ---- Update as a start
......@@ -304,10 +337,15 @@ def update_ci_report(filename, run_id, notebook_id, notebook_dir, notebook_src,
report[run_id]['state'] = 'ok' if happy_end else 'ERROR'
report[run_id]['out'] = notebook_out # changeg in case of error
# ---- Save it
with open(filename,'wt') as fp:
# ---- Save report
with open(_report_filename,'wt') as fp:
json.dump(report,fp,indent=4)
if not happy_end:
with open(_error_filename, 'a') as fp:
print(f"See : {notebook_dir}/{notebook_out} ", file=fp)
def build_ci_report(report_name=None, display_output=True, save_html=True):
......@@ -316,6 +354,7 @@ def build_ci_report(report_name=None, display_output=True, save_html=True):
#
if report_name is None:
report_name = config.CI_REPORT_JSON
with open(report_name) as infile:
ci_report = json.load( infile )
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment