From f2867b277fd1805ac4a43a5095577669760b4be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franck=20P=C3=A9rignon?= <franck.perignon@univ-grenoble-alpes.fr> Date: Fri, 8 Nov 2024 18:54:06 +0100 Subject: [PATCH] [docker-build] clean CI. Add doc image generation. --- .gitlab-ci.yml | 228 +++++++--------------------- ci/ci-templates.yml | 97 ++---------- ci/docker_images/ci_user/Dockerfile | 56 +++---- ci/hysop-doc-env.yaml | 36 +++++ ci/hysopenv.yaml | 6 + meson.build | 28 ++-- 6 files changed, 138 insertions(+), 313 deletions(-) create mode 100644 ci/hysop-doc-env.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2d00988e..07ab0682e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,11 +26,11 @@ include: ###################### ## DOCKER IMG BUILD ## ###################### -# Jobs to build docker images and save them into the Gitlab container registry + ubuntu:cpu-intel:docker-build: variables: - IMAGE_NAME: ci_cpu_intel - DOCKER_TARGET: final-stage + IMAGE_NAME: ci_user + DOCKER_TARGET: hysop-final-cpu RESULT_NAME: $CI_REGISTRY_IMAGE/ci_cpu_intel extends: .docker-build tags: @@ -43,23 +43,26 @@ ubuntu:cpu-intel:docker-build: ubuntu:gpu-nvidia:docker-build: variables: - IMAGE_NAME: ci_gpu_nvidia - DOCKER_TARGET: final-stage + IMAGE_NAME: ci_user + DOCKER_TARGET: hysop-final-gpu RESULT_NAME: $CI_REGISTRY_IMAGE/ci_gpu_nvidia extends: .docker-build tags: - lmap - - gpu + - gpu rules: - if: $CI_COMMIT_MESSAGE =~ /\[docker-build\]/ - if: $CI_COMMIT_TAG - if: $CI_COMMIT_MESSAGE =~ /\[docker-build-gpu-nvidia\]/ -ubuntu:cpu-intel-user:docker-build: +# Create a new docker image with all deps. required to build +# documentation (api and manual) +# This image is pushed to hysop-doc. +ubuntu:cpu-intel-doc: variables: IMAGE_NAME: ci_user - DOCKER_TARGET: hysop-final-cpu - RESULT_NAME: $CI_REGISTRY_IMAGE/ci_cpu_intel_user + DOCKER_TARGET: hysop-doc + RESULT_NAME: $CI_REGISTRY/particle_methods/hysop-doc/ci_cpu_intel_doc extends: .docker-build tags: - lmap @@ -67,25 +70,12 @@ ubuntu:cpu-intel-user:docker-build: rules: - if: $CI_COMMIT_MESSAGE =~ /\[docker-build\]/ - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_MESSAGE =~ /\[docker-build-cpu-intel-user\]/ + - if: $CI_COMMIT_MESSAGE =~ /\[docker-build-cpu-intel-doc\]/ -ubuntu:gpu-nvidia-user:docker-build: - variables: - IMAGE_NAME: ci_user - DOCKER_TARGET: hysop-final-gpu - RESULT_NAME: $CI_REGISTRY_IMAGE/ci_gpu_nvidia_user - extends: .docker-build - tags: - - lmap - - gpu - rules: - - if: $CI_COMMIT_MESSAGE =~ /\[docker-build\]/ - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_MESSAGE =~ /\[docker-build-gpu-nvidia-user\]/ - -############################## -## CPU Intel ## -############################## + +#################################### +## CPU Intel with a non root user ## +#################################### config:ci-cpu-intel: variables: IMAGE_NAME: $HYSOP_REGISTRY/ci_cpu_intel:latest @@ -128,167 +118,37 @@ examples:ci-cpu-intel: - .hysop-examples - .lmap-cpu -# docker:hysop:cpu-intel: -# variables: -# IMAGE_NAME: hysop_cpu_intel -# RESULT_NAME: $CI_REGISTRY/particle_methods/hysop/hysop_cpu_intel -# DOCKER_TARGET: hysop_cpu_intel -# extends: -# - .hysop-docker-hysop -# - .lmap-cpu -# needs: ["build:ci-cpu-intel", "test:ci-cpu-intel"] - -#################################### -## CPU Intel with a non root user ## -#################################### -config:ci-cpu-intel-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_cpu_intel_user:latest - extends: - - .hysop-configure-venv - - .lmap-cpu - needs: - - job: ubuntu:cpu-intel-user:docker-build # wait for docker image if docker build is on ... - optional: true - -build:ci-cpu-intel-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_cpu_intel_user:latest - needs: ["config:ci-cpu-intel-user"] - extends: - - .hysop-build-venv - - .lmap-cpu - -install:ci-cpu-intel-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_cpu_intel_user:latest - needs: ["build:ci-cpu-intel-user"] - extends: - - .hysop-install-venv - - .lmap-cpu - -test:ci-cpu-intel-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_cpu_intel_user:latest - needs: ["install:ci-cpu-intel-user"] - extends: - - .hysop-test-venv - - .lmap-cpu - -examples:ci-cpu-intel-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_cpu_intel_user:latest - needs: ["install:ci-cpu-intel-user"] - extends: - - .hysop-examples-venv - - .lmap-cpu - # Jobs to create docker images with a full hysop install, ready to use as a notebook # First for binder ... binder:mamba:cpu: variables: - HOST_TYPE: ci_cpu_intel_user + HOST_TYPE: ci_cpu_intel IMAGE_NAME: lab-mamba RESULT_NAME: $CI_REGISTRY/particle_methods/hysop_binder/hysopbinderlab DOCKER_TARGET: binderlab extends: - .hysop-docker-hysop - .lmap-cpu - needs: ["install:ci-cpu-intel-user"] + needs: ["install:ci-cpu-intel"] # then for any user jupyterlab:mamba:cpu: variables: - HOST_TYPE: ci_cpu_intel_user + HOST_TYPE: ci_cpu_intel IMAGE_NAME: lab-mamba - RESULT_NAME: $CI_REGISTRY/particle_methods/hysop/hysoplab + RESULT_NAME: $CI_REGISTRY/particle_methods/hysop/hysoplab-cpu-intel DOCKER_TARGET: jupylab extends: - .hysop-docker-hysop - .lmap-cpu - needs: ["install:ci-cpu-intel-user"] + needs: ["install:ci-cpu-intel"] ##################################### ## GPU nvidia with a non root user ## ##################################### -config:ci-gpu-nvidia-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia_user - extends: - - .hysop-configure-venv - - .hysop-gpu - needs: - - job: ubuntu:gpu-nvidia-user:docker-build # wait for docker image if docker build is on ... - optional: true - -build:ci-gpu-nvidia-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia_user - needs: ["config:ci-gpu-nvidia-user"] - extends: - - .hysop-build-venv - - .hysop-gpu - -install:ci-gpu-nvidia-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia_user - needs: ["build:ci-gpu-nvidia-user"] - extends: - - .hysop-install-venv - - .hysop-gpu - -test:ci-gpu-nvidia-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia_user - needs: ["install:ci-gpu-nvidia-user"] - extends: - - .hysop-test-venv - - .hysop-gpu - -examples:ci-gpu-nvidia-user: - variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia_user - needs: ["install:ci-gpu-nvidia-user"] - extends: - - .hysop-examples-venv - - .hysop-gpu - -# Jobs to create docker images with a full hysop install, ready to use as a notebook -# First for binder ... -binder:mamba:gpu: - variables: - HOST_TYPE: ci_gpu_nvidia_user - IMAGE_NAME: lab-mamba - RESULT_NAME: $CI_REGISTRY/particle_methods/hysop_binder/hysopbinderlab-gpu - DOCKER_TARGET: binderlab - extends: - - .hysop-docker-hysop - - .lmap-cpu - - needs: ["install:ci-gpu-nvidia-user"] - -# then for any user -jupyterlab:mamba:gpu: - variables: - HOST_TYPE: ci_gpu_nvidia_user - IMAGE_NAME: lab-mamba - RESULT_NAME: $CI_REGISTRY/particle_methods/hysop/hysoplab-gpu - DOCKER_TARGET: jupylab - extends: - - .hysop-docker-hysop - - .lmap-cpu - needs: ["install:ci-gpu-nvidia-user"] - -# - - - -################## -## Nvidia GPUs ## -################## config:ci-gpu-nvidia: variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia:latest + IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia extends: - .hysop-configure - .hysop-gpu @@ -298,7 +158,7 @@ config:ci-gpu-nvidia: build:ci-gpu-nvidia: variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia:latest + IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia needs: ["config:ci-gpu-nvidia"] extends: - .hysop-build @@ -306,7 +166,7 @@ build:ci-gpu-nvidia: install:ci-gpu-nvidia: variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia:latest + IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia needs: ["build:ci-gpu-nvidia"] extends: - .hysop-install @@ -314,7 +174,7 @@ install:ci-gpu-nvidia: test:ci-gpu-nvidia: variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia:latest + IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia needs: ["install:ci-gpu-nvidia"] extends: - .hysop-test @@ -322,21 +182,37 @@ test:ci-gpu-nvidia: examples:ci-gpu-nvidia: variables: - IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia:latest + IMAGE_NAME: $HYSOP_REGISTRY/ci_gpu_nvidia needs: ["install:ci-gpu-nvidia"] extends: - .hysop-examples - .hysop-gpu -# docker:hysop:gpu-nvidia: -# variables: -# IMAGE_NAME: hysop_gpu_nvidia -# RESULT_NAME: $CI_REGISTRY/particle_methods/hysop/hysop_gpu_nvidia -# DOCKER_TARGET: hysop_gpu_nvidia -# extends: -# - .hysop-docker-hysop -# - .lmap-cpu -# needs: ["build:ci-gpu-nvidia", "test:ci-gpu-nvidia"] +# Jobs to create docker images with a full hysop install, ready to use as a notebook +# First for binder ... +# binder:mamba:gpu: +# variables: +# HOST_TYPE: ci_gpu_nvidia +# IMAGE_NAME: lab-mamba +# RESULT_NAME: $CI_REGISTRY/particle_methods/hysop_binder/hysopbinderlab-gpu +# DOCKER_TARGET: binderlab +# extends: +# - .hysop-docker-hysop +# - .lmap-cpu + +# needs: ["install:ci-gpu-nvidia"] + +# then for any user +jupyterlab:mamba:gpu: + variables: + HOST_TYPE: ci_gpu_nvidia + IMAGE_NAME: lab-mamba + RESULT_NAME: $CI_REGISTRY/particle_methods/hysop/hysoplab-gpu-nvidia + DOCKER_TARGET: jupylab + extends: + - .hysop-docker-hysop + - .lmap-cpu + needs: ["install:ci-gpu-nvidia"] ##### # Job run when a new tag (release) is created. diff --git a/ci/ci-templates.yml b/ci/ci-templates.yml index 860a97998..f25b118ce 100644 --- a/ci/ci-templates.yml +++ b/ci/ci-templates.yml @@ -166,112 +166,37 @@ workflow: - lmap - gpu -.hysop-configure: - extends: .hysop-ctest - stage: configure - script: - - env - - "bash ci/scripts/version.sh" - - "bash ci/scripts/config.sh" - artifacts: - expire_in: 1 week - paths: - - $BUILD_DIR - -.hysop-build: - extends: .hysop-ctest - stage: build - script: - - meson compile -C $BUILD_DIR - artifacts: - expire_in: 1 week - paths: - - $BUILD_DIR - -.hysop-install: - extends: .hysop-ctest - stage: install - script: - - meson install -C $BUILD_DIR - - whoami - - ls -artl / - - echo ${PWD} - - ls -artl - - export PYTHONPATH_0=${BUILD_DIR}/install/lib/python3*/*-packages - - export PYTHONPATH=`echo $PYTHONPATH_0` - - python3 -c "import hysop; print(hysop); print(hysop.__file__)" - - python3 -c "from hysop import f2hysop; print(f2hysop); print(f2hysop.__file__)" - artifacts: - expire_in: 1 week - paths: - - $BUILD_DIR - -.hysop-test: - extends: .hysop-ctest - stage: test - script: - - export PYTHONPATH_0=${BUILD_DIR}/install/lib/python3*/*-packages - - export PYTHONPATH=`echo $PYTHONPATH_0` - - python3 -c "import hysop; print(hysop); print(hysop.__file__)" - - python3 -c "from hysop import f2hysop; print(f2hysop); print(f2hysop.__file__)" - - MESON_TESTTHREADS=1 meson test -C $BUILD_DIR - artifacts: - when: on_failure - paths: - - $BUILD_DIR - -.hysop-test-cache: - extends: .hysop-ctest - stage: test - variables: - CACHE_DIR: $CI_PROJECT_DIR/cache - script: - - "bash ci/scripts/test.sh" - cache: - paths: - - $CI_PROJECT_DIR/cache - key: "hysop_cache" - -.hysop-examples: - extends: .hysop-ctest - stage: test - script: - - export PYTHONPATH_0=${BUILD_DIR}/install/lib/python3*/*-packages - - export PYTHONPATH=`echo $PYTHONPATH_0` - - "bash ci/scripts/run_examples.sh" - -# Same as above, but with a default install in python env -.hysop-base-venv: +.hysop-base: image: $IMAGE_NAME artifacts: expire_in: 1 week paths: - $BUILD_DIR -.hysop-configure-venv: - extends: .hysop-base-venv +.hysop-configure: + extends: .hysop-base stage: configure script: - env - "bash ci/scripts/version.sh" - meson setup $BUILD_DIR $CI_PROJECT_DIR -.hysop-build-venv: - extends: .hysop-base-venv +.hysop-build: + extends: .hysop-base stage: build script: - meson compile -C $BUILD_DIR -.hysop-install-venv: - extends: .hysop-base-venv +.hysop-install: + extends: .hysop-base stage: install script: - meson install -C $BUILD_DIR - python3 -c "import hysop; print(hysop); print(hysop.__file__)" - python3 -c "from hysop import f2hysop; print(f2hysop); print(f2hysop.__file__)" -.hysop-test-venv: - extends: .hysop-base-venv +.hysop-test: + extends: .hysop-base stage: test script: - meson install -C $BUILD_DIR @@ -286,8 +211,8 @@ workflow: paths: - $BUILD_DIR -.hysop-examples-venv: - extends: .hysop-base-venv +.hysop-examples: + extends: .hysop-base stage: test script: - meson install -C $BUILD_DIR diff --git a/ci/docker_images/ci_user/Dockerfile b/ci/docker_images/ci_user/Dockerfile index a0f0b4975..81b62a48f 100644 --- a/ci/docker_images/ci_user/Dockerfile +++ b/ci/docker_images/ci_user/Dockerfile @@ -130,12 +130,6 @@ RUN cmake -S flint -B build-flint -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX= ########## FROM hysop-base-python AS hysop-full-python -# Some python packages are not available in micromamba! -RUN pip install --no-cache-dir --upgrade \ - tee colors.py primefac \ - argparse_color_formatter \ - memory-tempfile - # gpyFFT # WARNING: Old package! RUN git clone https://github.com/geggo/gpyfft.git && \ @@ -143,37 +137,6 @@ RUN git clone https://github.com/geggo/gpyfft.git && \ sed 's#finalize(self, _destroy_plan, self.plan)##' -i gpyfft/gpyfftlib.pyx && \ pip install --no-cache-dir . && cd && rm -rf gpyfft -# # -# FFTW latest version nov 2023 -# Compilation is necessary because there is no mpi version -# with the float precisions requested by hysop (single, double, long) -# --> indeed there is. See yaml file -# -# ENV FFTW_ROOT=$CONDA_PREFIX -# RUN wget -q http://www.fftw.org/fftw-3.3.10.tar.gz && \ -# tar -xzf fftw-*.tar.gz && \ -# rm -f fftw-*.tar.gz && \ -# cd fftw-* && \ -# ./configure --enable-openmp --enable-threads --enable-mpi --enable-shared --with-pic --prefix="${FFTW_ROOT}" --enable-single && \ -# make -j$(nproc) && make install && make clean && \ -# ./configure --enable-openmp --enable-threads --enable-mpi --enable-shared --with-pic --prefix="${FFTW_ROOT}" && \ -# make -j$(nproc) && make install && make clean && \ -# ./configure --enable-openmp --enable-threads --enable-mpi --enable-shared --with-pic --prefix="${FFTW_ROOT}" --enable-long-double && \ -# make -j$(nproc) && make install && make clean -# RUN rm -rf /tmp/fftw-* - -# -# PYFFTW git version (last test: july 2024) -# pyfftw is linked to fftw3 compiled above - -#ARG pip_install_opts='--no-binary=pyfftw --no-deps --no-build-isolation' -#ARG pip_install_opts='--no-cache-dir --no-binary=pyfftw --no-deps' -#RUN git clone https://github.com/pyFFTW/pyFFTW.git && \ -# cd pyFFTW && \ -## PYFFTW_INCLUDE="/include" \ -# pip install ${pip_install_opts} . && \ -# cd && rm -rf pyFFTW - ############### # Final-Stage # ############### @@ -226,3 +189,22 @@ USER hysop-user ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility ENV PYOPENCL_COMPILER_OUTPUT=1 + + +# --- Image with deps. required to build hysop-doc --- +FROM hysop-final AS hysop-doc + +# Bibtex !!! +USER root +RUN apt update && apt upgrade -y && apt install -y -qq \ + biber && \ + apt autoclean -y && \ + apt autoremove -y && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf $HOME/.cache/pip/* + +USER hysop-user +# Install conda/mamba packages listed in ./hysop_doc.yaml file +COPY --chown=hysop-user:hysop-user ci/hysop-doc-env.yaml /home/hysop-doc-env.yaml +RUN micromamba install -y -f /home/hysop-doc-env.yaml && \ + micromamba clean --all --yes diff --git a/ci/hysop-doc-env.yaml b/ci/hysop-doc-env.yaml new file mode 100644 index 000000000..aab41072a --- /dev/null +++ b/ci/hysop-doc-env.yaml @@ -0,0 +1,36 @@ +## +## Copyright (c) HySoP 2011-2024 +## +## This file is part of HySoP software. +## See "https://particle_methods.gricad-pages.univ-grenoble-alpes.fr/hysop-doc/" +## for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +name: base +channels: + - conda-forge +dependencies: + - doxygen + - graphviz + - sphinx + - sphinxcontrib-bibtex + - sphinxcontrib-doxylink + - sphinx_bootstrap_theme + - sphinx-rtd-theme + - strip-hints + - pandoc + - ipykernel + - pip + - pip: + - nbsphinx diff --git a/ci/hysopenv.yaml b/ci/hysopenv.yaml index 4a546e232..6272fe3ce 100644 --- a/ci/hysopenv.yaml +++ b/ci/hysopenv.yaml @@ -71,6 +71,12 @@ dependencies: - tox - pyfftw - jupyterlab + - pip: + - tee + - colors.py + - primefac + - argparse_color_formatter + - memory-tempfile channels: - numba dependencies: diff --git a/meson.build b/meson.build index 22c23850b..c8b83bf26 100644 --- a/meson.build +++ b/meson.build @@ -42,7 +42,7 @@ project('hysop', 'c', # hysop version is set in hysop_version.txt file version : files('hysop_version.txt'), - license: 'BSD-3', + license: 'Apache-2.0', meson_version: '>=1.3.0', # Strongly inspired by the scipy meson.build configuration default_options: [ @@ -72,7 +72,7 @@ full_test = get_option('FULL_TEST') optim = get_option('OPTIM') with_mpi_test = get_option('WITH_MPI_TESTS') fortran_layout = get_option('FORTRAN_LAYOUT') -with_documentation = get_option('WITH_DOCUMENTATION') +# with_documentation = get_option('WITH_DOCUMENTATION') enable_long_tests = get_option('ENABLE_LONG_TESTS') dev_mode = get_option('DEV_MODE') @@ -175,18 +175,18 @@ subdir('hysop_examples') subdir('src') # == Documentation == -if with_documentation == 'ON' - docs_env = environment() - docs_env.set('PYTHONPATH', py.get_install_dir()) - docs_env.set('HYSOP_VERBOSE', 'False') - doxygen = find_program('doxygen', required : true) - doxygen_dot = find_program('dot', required : true) - sphinx_api = find_program('sphinx-apidoc', required : true) - sphinx = find_program('sphinx-build', required : true) - bibtex = find_program('bibtex', required : true) - bash = find_program('bash', required:true) - subdir('docs') -endif +# if with_documentation == 'ON' +# docs_env = environment() +# docs_env.set('PYTHONPATH', py.get_install_dir()) +# docs_env.set('HYSOP_VERBOSE', 'False') +# doxygen = find_program('doxygen', required : true) +# doxygen_dot = find_program('dot', required : true) +# sphinx_api = find_program('sphinx-apidoc', required : true) +# sphinx = find_program('sphinx-build', required : true) +# bibtex = find_program('bibtex', required : true) +# bash = find_program('bash', required:true) +# subdir('docs') +# endif # ============= Summary ============= #if verbose == 'ON' -- GitLab