Compare revisions

Jean-Baptiste Keck · Jean-Baptiste Keck · 82eba84d · 82eba84d · 82eba84d · 82eba84d
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,57 +7,57 @@ stages:
  - install
  - test

-env:groovy:
-  image: keckj/hysop:groovy
+env:jammy:
+  image: keckj/hysop:jammy
  stage: env
  script:
      - "bash ci/scripts/version.sh"

-config:groovy:
-  image: keckj/hysop:groovy
+config:jammy:
+  image: keckj/hysop:jammy
  stage: configure
  script: 
-      - "bash ci/scripts/config.sh $CI_PROJECT_DIR/build/gcc   $CI_PROJECT_DIR/install/gcc gcc-10 g++-10 gfortran-10"
-      - "bash ci/scripts/config.sh $CI_PROJECT_DIR/build/clang $CI_PROJECT_DIR/install/clang clang-10 clang++-10 gfortran-10"
+      - "bash ci/scripts/config.sh $CI_PROJECT_DIR/build/gcc   $CI_PROJECT_DIR/install/gcc gcc-11 g++-11 gfortran-11"
+      - "bash ci/scripts/config.sh $CI_PROJECT_DIR/build/clang $CI_PROJECT_DIR/install/clang clang-11 clang++-11 gfortran-11"
  dependencies:
-    - env:groovy
+    - env:jammy
  artifacts:
    expire_in: 1 week
    paths:
        - $CI_PROJECT_DIR/build

-build:groovy:
-  image: keckj/hysop:groovy
+build:jammy:
+  image: keckj/hysop:jammy
  stage: build
  script: 
-      - "bash ci/scripts/build.sh $CI_PROJECT_DIR/build/gcc gcc-10 g++-10 gfortran-10"
-      - "bash ci/scripts/build.sh $CI_PROJECT_DIR/build/clang clang-10 clang++-10 gfortran-10"
+      - "bash ci/scripts/build.sh $CI_PROJECT_DIR/build/gcc gcc-11 g++-11 gfortran-11"
+      - "bash ci/scripts/build.sh $CI_PROJECT_DIR/build/clang clang-11 clang++-11 gfortran-11"
  dependencies:
-    - config:groovy
+    - config:jammy
  artifacts:
    expire_in: 1 week
    paths:
        - $CI_PROJECT_DIR/build

-install:groovy:
-  image: keckj/hysop:groovy
+install:jammy:
+  image: keckj/hysop:jammy
  stage: install
  script: 
      - "bash ci/scripts/install.sh $CI_PROJECT_DIR/build/gcc $CI_PROJECT_DIR/install/gcc"
  dependencies:
-    - build:groovy
+    - build:jammy
  artifacts:
    expire_in: 1 week
    paths:
        - $CI_PROJECT_DIR/install

-test:groovy:
-  image: keckj/hysop:groovy
+test:jammy:
+  image: keckj/hysop:jammy
  stage: test
  script:
    - "bash ci/scripts/test.sh $CI_PROJECT_DIR/install/gcc $CI_PROJECT_DIR/hysop $CI_PROJECT_DIR/cache"
  dependencies:
-    - install:groovy
+    - install:jammy
  cache:
    paths:
      - $CI_PROJECT_DIR/cache

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -126,8 +126,11 @@ set(HYSOP_LINK_LIBRARIES CACHE INTERNAL "List of external libraries.")

 # ============= Python and its packages =============
 # - Global setup (interp and lib) -
-find_package (Python COMPONENTS Interpreter Development)
-set(PYTHON_EXECUTABLE ${Python_EXECUTABLE})
+find_package (Python3 REQUIRED COMPONENTS Interpreter Development)
+if(Python3_VERSION VERSION_LESS "3.8")
+    message(FATAL_ERROR "HySoP requires Python3.8 or later.")
+endif()
+set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}")
 include(FindPythonModule)

 # - python packages -
@@ -265,7 +268,11 @@ else()
    OUTPUT_VARIABLE OPENCL_DEFAULT_OPENCL_ID)
 endif()

-string(REPLACE " " ";" MY_LIST ${OPENCL_DEFAULT_OPENCL_ID})
+string(REPLACE " " ";" MY_LIST "${OPENCL_DEFAULT_OPENCL_ID}")
+list(LENGTH MY_LIST MY_LIST_LENGTH)
+if (${MY_LIST_LENGTH} LESS 2)
+    message(FATAL_ERROR "Could not determine default OpenCL device, try to execute '${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/opencl_explore.py' locally to determine the problem.")
+endif()
 list(GET MY_LIST 0 OPENCL_DEFAULT_OPENCL_PLATFORM_ID)
 list(GET MY_LIST 1 OPENCL_DEFAULT_OPENCL_DEVICE_ID)
 display(OPENCL_DEFAULT_OPENCL_PLATFORM_ID)

--- a/README.md
+++ b/README.md
 [![Platform](https://img.shields.io/badge/platform-linux--64%20%7C%C2%A0%20osx--64-lightgrey.svg)]()
 [![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-380/)
 [![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)](https://www.python.org/downloads/release/python-390/)
-[![Licence](https://img.shields.io/badge/licence-APLv2-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0)
+[![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-3100/)
+[![Licence](https://img.shields.io/badge/licence-APLv2-green.svg)](https://www.apache.org/licenses/LICENSE-2.0)
 [![Pipeline Status](https://gricad-gitlab.univ-grenoble-alpes.fr/particle_methods/hysop/badges/master/pipeline.svg)](https://gricad-gitlab.univ-grenoble-alpes.fr/particle_methods/hysop/commits/master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/keckj/hysop.svg)](https://hub.docker.com/r/keckj/hysop/tags)

+HySoP (Hybrid Simulation with Particles) is a library dedicated to high performance direct numerical simulation of fluid related problems based on semi-lagrangian particle methods, for hybrid architectures providing multiple compute devices including CPUs, GPUs or MICs. See [https://particle_methods.gricad-pages.univ-grenoble-alpes.fr/hysop-doc](https://particle_methods.gricad-pages.univ-grenoble-alpes.fr/hysop-doc) for more information.
+
 Basics
 ======
-Download lastest hysop:
+Download and install lastest hysop:

 ```
 git clone git@gricad-gitlab.univ-grenoble-alpes.fr:particle_methods/hysop.git
 cd hysop
 mkdir build
 cd build
-cmake path-to-hysop
+cmake ..
 make -j8
 make install
 ```

+By default, cmake will try to find your most up to date Python3 installation. The minimum required version is Python3.8. You can force the python version by using the following trick during the cmake configuration step:
+```
+PYTHON_EXECUTABLE="$(which python3.8)"
+PYTHON_INCLUDE_DIR=$(${PYTHON_EXECUTABLE} -c "import sysconfig as sc; print(sc.get_paths()['include'])")  
+PYTHON_LIBRARY=$(${PYTHON_EXECUTABLE} -c "import sysconfig as sc, os; print(os.path.normpath(os.path.sep.join(sc.get_config_vars('LIBDIR', 'INSTSONAME'))))")
+cmake -DPython3_EXECUTABLE="${PYTHON_EXECUTABLE}" -DPython3_INCLUDE_DIR="${PYTHON_INCLUDE_DIR}" -DPython3_LIBRARY="${PYTHON_LIBRARY}" ..
+```
+
 Run Taylor-Green with 8 mpi processes:
 ```
-mpirun -np 8 python3.8 hysop_examples/examples/taylor_green/taylor_green.py -impl fortran -V -cp fp64 -d 64
+mpirun -np 8 python3.9 hysop_examples/examples/taylor_green/taylor_green.py -impl fortran -V -cp fp64 -d 64
 ```
-Check examples directory for complete simulation cases. Hysop is currently developped and tested with `python3.8`. It also works well with `python3.9`. You may want to `export PYTHON_EXECUTABLE=python3.9` to run some of hysop scripts that look for `python3.8` by default. Please note that `python3.10` has not been tested yet but should work as well.
+Check examples directory for complete simulation cases. Hysop is currently developped and tested with `python3.9`. It also works well with `python3.8`. You may want to `export PYTHON_EXECUTABLE=python3.8` to run some of hysop scripts that look for `python3.9` by default. Please note that `python3.10` has not been tested yet but should work as well.

 For each example you can get help with the `-h` option:
 ```
- python3.8 hysop_examples/examples/taylor_green/taylor_green.py -h
+ python3.9 hysop_examples/examples/taylor_green/taylor_green.py -h
 ```

 Dependencies:
@@ -35,15 +46,15 @@ Dependencies:
 To install hysop dependencies locally on an ubuntu machine (20.04 LTS):
 ```
 sudo apt-get install -y expat unzip xz-utils automake libtool pkg-config cmake git ssh gcc g++ gfortran lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libblas-dev liblapack-dev libgcc-10-dev libgfortran-10-dev libgmp-dev libmpfr-dev libmpc-dev python3.8-dev opencl-headers swig libgmp-dev libmpfr-dev libmpc-dev libcairo-dev libcairomm-1.0-dev python3.8-dev python3.8-tk libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev
-pip3.8 install --upgrade numpy setuptools cffi wheel pytest pybind11 cython
-pip3.8 install --upgrade -r requirements.txt
+python3.9 -m pip install --upgrade numpy setuptools cffi wheel pytest pybind11 cython
+python3.9 -m pip install --upgrade -r requirements.txt
 ```
-Additionally you may want to provide a working OpenCL platform, HPTT, llvm/llvmlite/numba, clFFT/gpyFFT, flint/arb/python-flint and tbb/mklfft. See the docker files to install those packages (`hysop/ci/docker_images`). Alternatively your can run hysop in an isolated environment by using docker containers (see next section).
+Additionally you may want to provide a working OpenCL platform, HPTT, llvm/llvmlite/numba, clFFT/gpyFFT, flint/arb/python-flint and tbb/mklfft. See the docker files to install those packages (`hysop/ci/docker_images/ubuntu`). Alternatively your can run hysop in an isolated environment by using docker containers (see next section).

 # How to use docker images ?
 Docker images can be pulled (resp. pushed) with `./ci/utils/pull_docker_image.sh [imgname]` and `./ci/utils/push_docker_image.sh [imgname]`. The docker images do not contain de hysop library and can be run with `./ci/utils/run_docker_image.sh [imgname]`. This script mounts your local hysop directory (read only) to `/hysop` inside the docker container and prompt a shell. Images have to be downloaded (pulled) prior to be run with this script.

-By default, `[imgname]` corresponds to the docker image used for gitlab continuous integration (currently `groovy`, which corresponds to ubuntu 20.10 running with python3.8) but any other ubuntu image name may be used, `hirsute` being the most up-to-date image (ubuntu 21.04 running with python3.9). Docker images ship an intel opencl platform that is compatible with intel cpus. Cuda enabled images may be a bit out-of-date (see `./ci/docker_images/ubuntu/bionic_cuda` to get started).
+By default, `[imgname]` corresponds to the docker image used for gitlab continuous integration (currently `jammy`, which corresponds to Ubuntu 22.04 running with python3.9). Docker images ship an intel OpenCL platform that is compatible with intel CPUs. Cuda enabled images may be a bit out-of-date (see `./ci/docker_images/ubuntu/bionic_cuda` to get started).

 To quickly test and/or debug hysop inside the docker you can run `./ci/utils/run_debug.sh [imgname]` which will build and install hysop inside the container and prompt a shell (read-only mount-bind `/hysop` directory is copied to read-write `/tmp/hysop` within the container). Alternatively if you just want to run tests inside docker, you can use the `./ci/utils/run_ci.sh [imgname]` script instead.


--- a/ci/docker_images/ubuntu/jammy/Dockerfile
+++ b/ci/docker_images/ubuntu/jammy/Dockerfile
+# Test docker for gitlab-ci
+FROM ubuntu:jammy
+MAINTAINER Jean-Baptiste.Keck@imag.fr
+
+# parallel builds
+ARG NTHREADS
+ENV MAKEFLAGS "-j${NTHREADS}"
+
+# upgrade initial image
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get full-upgrade -y
+
+# get build tools and required libraries
+RUN apt-get update && apt-get install -y --no-install-recommends expat unzip xz-utils automake libtool pkg-config cmake rsync git vim ssh curl wget ca-certificates gcc g++ gfortran lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libblas-dev liblapack-dev libgcc-11-dev libgfortran-11-dev libgmp-dev libmpfr-dev libmpc-dev python3.9-dev opencl-headers swig libgmp-dev libmpfr-dev libmpc-dev libcairo-dev libcairomm-1.0-dev python3.9-tk
+
+# python-pip
+RUN cd /tmp && \
+ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+ python3.9 get-pip.py && \
+ python3.9 -m pip install --upgrade pip && \
+ rm -f /tmp/get-pip.py
+
+RUN python3.9 -m pip install --upgrade numpy==1.21.5 setuptools cffi wheel pytest pybind11 cython
+
+# OpenMPI 4.1.2 + mpi4py (enable mpi1 compatibility for mpi4py)
+ENV MPI_ROOT "/usr/local"
+RUN cd /tmp && \
+ wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.2.tar.gz && \
+ tar -xvzf openmpi-*.tar.gz && \
+ rm -f openmpi-*.tar.gz && \
+ cd openmpi-* && \
+ ./configure --enable-shared --disable-static --with-threads=posix --enable-ipv6 --prefix="${MPI_ROOT}" --with-hwloc=internal --with-libevent=internal --enable-mpi1-compatibility && \
+ make && \
+ make install && \
+ rm -rf /tmp/openmpi-*
+
+ENV MPICC "${MPI_ROOT}/bin/mpicc"
+RUN ldconfig && python3.9 -m pip install --upgrade mpi4py -vvv
+
+# HPTT (CPU tensor permutation library)
+RUN cd /tmp && \
+ git clone https://gitlab.com/keckj/hptt.git && \
+ cd hptt && \
+ sed -i "s#-mavx##g" CMakeLists.txt && \
+ sed -i "s#-march=native##g" CMakeLists.txt && \
+ sed -i "s#-mtune=native##g" CMakeLists.txt && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd ../pythonAPI && \
+ python3.9 -m pip install --upgrade . && \
+ cd /tmp && \
+ rm -rf /tmp/hptt
+
+# HDF5 1.12.1 + h5py 3.6.0
+RUN cd /tmp && \
+ wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.1/src/hdf5-1.12.1.tar.gz && \
+ tar -xvzf hdf5-*.tar.gz && \
+ rm -f hdf5-*.tar.gz && \
+ cd hdf5-* && \
+ CC="${MPICC}" ./configure --prefix="${MPI_ROOT}" --enable-parallel --enable-shared=yes --enable-static=no && \
+ make && \
+ make install && \
+ rm -rf /tmp/hdf5-*
+RUN CC="${MPICC}" HDF5_MPI="ON" HDF5_VERSION="1.12.1" HDF5_DIR="${MPI_ROOT}" python3.9 -m pip install --upgrade --no-binary=h5py h5py -vvv
+
+# other python packages
+RUN python3.9 -m pip install --upgrade scipy sympy matplotlib gmpy2 psutil py-cpuinfo Mako editdistance portalocker colors.py tee pycairo argparse_color_formatter networkx pyvis zarr numcodecs jsonpickle memory-tempfile primefac
+
+# llvm + numba + llvmlite (numba 0.55.1 only supports llvm 11 for now)
+RUN apt-get update && \
+ apt-get install -y llvm-11-dev libclang-11-dev clang-11
+ENV LLVM_CONFIG=llvm-config-11
+RUN python3.9 -m pip install --upgrade numba llvmlite
+
+# Intel experimental OpenCL platform with SYCL support
+# OclCpuExp 2021.12 and oneTBB 2021.5.0
+ENV TBBROOT="/opt/intel/oclcpuexp/x64"
+ENV LD_LIBRARY_PATH "${TBBROOT}:${LD_LIBRARY_PATH}"
+RUN cd /tmp && \
+ mkdir -p /opt/intel/oclcpuexp && \
+ wget https://github.com/intel/llvm/releases/download/2020-06/oclcpuexp-2020.10.6.0.4_rel.tar.gz && \
+ tar -xvzf oclcpuexp-*.tar.gz && \
+ mv x64/ /opt/intel/oclcpuexp/ && \
+ mv clbltfnshared.rtl /opt/intel/oclcpuexp/ && \
+ rm -f *.rtl && \
+ rm -rf oclcpuexp-* && \
+ wget https://github.com/oneapi-src/oneTBB/releases/download/v2020.3/tbb-2020.3-lin.tgz && \
+ tar -xvzf tbb-*.tgz && \
+ mv tbb/lib/intel64/gcc4.8/* "${TBBROOT}/" && \
+ rm -f /usr/local/lib/libOpenCL.so && \
+ rm -f /usr/local/lib/libOpenCL.so && \
+ rm -f /usr/local/lib/libOpenCL.so.1 && \
+ rm -f /usr/local/lib/libOpenCL.so.2.0 && \
+ ln -s "${TBBROOT}/libOpenCL.so" /usr/local/lib/libOpenCL.so && \
+ ln -s "${TBBROOT}/libOpenCL.so.1" /usr/local/lib/libOpenCL.so.1 && \
+ ln -s "${TBBROOT}/libOpenCL.so.2.0" /usr/local/lib/libOpenCL.so.2.0 && \
+ mkdir -p /etc/OpenCL/vendors && \
+ echo "${TBBROOT}/libintelocl.so" > /etc/OpenCL/vendors/intel_expcpu.icd && \
+ rm -rf /tmp/oneapi*
+
+# clinfo 3.0.21.02.21 (2021)
+RUN cd /tmp && \
+ wget https://github.com/Oblomov/clinfo/archive/refs/tags/3.0.21.02.21.tar.gz && \
+ tar -xvzf *.tar.gz && \
+ rm -f *.tar.gz && \
+ cd clinfo-* && \
+ make && \
+ ./clinfo && \
+ mv ./clinfo /bin/ && \
+ rm -rf /tmp/clinfo-*
+
+# clpeak 1.1.0 RC2 (2019)
+RUN cd /tmp && \
+ wget -O /usr/include/CL/cl.hpp https://www.khronos.org/registry/OpenCL/api/2.1/cl.hpp && \
+ wget https://github.com/krrishnarraj/clpeak/archive/1.1.0.tar.gz && \
+ tar -xvzf *.tar.gz && \
+ rm -f *.tar.gz && \
+ cd clpeak-* && \
+ mkdir build && \
+ cd build/ && \
+ cmake .. && \
+ make && \
+ mv ./clpeak /bin/ && \
+ rm -rf /tmp/clpeak-*
+
+# pyopencl
+RUN cd /tmp && \
+ git clone https://github.com/inducer/pyopencl.git && \
+ cd pyopencl && \
+ git checkout v2022.1 && \
+ git submodule update --init && \
+ python3.9 configure.py && \
+ make && \
+ python3.9 -m pip install --upgrade . && \
+ cd - && \
+ rm -rf /tmp/pyopencl
+
+# oclgrind
+RUN cd /tmp && \
+ git clone https://github.com/jrprice/Oclgrind.git && \
+ cd Oclgrind && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd - && \
+ rm -rf /tmp/Oclgrind
+
+# clFFT
+RUN cd /tmp && \
+ ln -s /usr/local/lib /usr/local/lib64 && \
+ git clone https://github.com/clMathLibraries/clFFT.git && \
+ cd clFFT && \
+ cd src && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd - && \
+ rm -rf /tmp/clFFT
+
+# gpyFFT, we need to fix a segfault on weakref.finalize(plan)
+# clFFT plans are destroyed when atexit(clFFT.teardown) is called
+RUN cd /tmp && \
+ git clone https://github.com/geggo/gpyfft.git && \
+ cd gpyfft && \
+ sed 's#finalize(self, _destroy_plan, self.plan)##' -i gpyfft/gpyfftlib.pyx && \
+ python3.9 -m pip install . && \
+ cd - && \
+ rm -rf /tmp/gpyfft
+
+# python flint (FLINT2 + ARB + python-flint)
+RUN cd /tmp && \
+  wget https://github.com/wbhart/flint2/archive/v2.8.4.tar.gz && \
+  tar -xvzf v*.tar.gz && \
+  rm -f v*.tar.gz && \
+  cd flint2-* && \
+  ./configure && \
+  make && \
+  make install && \
+  cd - && \
+  rm -rf flint2-*
+RUN cd /tmp && \
+  wget https://github.com/fredrik-johansson/arb/archive/2.22.1.tar.gz && \
+  tar -xvzf *.tar.gz && \
+  rm -f *.tar.gz && \
+  cd arb-* && \
+  ./configure && \
+  make && \
+  make install && \
+  cd - && \
+  rm -rf arb-*
+RUN python3.9 -m pip install --upgrade python-flint -vvv
+
+# static fftw + pyfftw (with R2R transforms)
+# Weird pyfftw bug : not passing -O2 explicitely during build causes a segfault on import...
+# See https://bugs.gentoo.org/548776
+ENV FFTW_ROOT="/usr/local"
+RUN cd /tmp && \
+ wget http://www.fftw.org/fftw-3.3.10.tar.gz && \
+ tar -xvzf fftw-*.tar.gz && \
+ rm -f fftw-*.tar.gz && \
+ cd fftw-* && \
+ ./configure --enable-openmp --enable-threads --enable-mpi --enable-static --with-pic --prefix="${FFTW_ROOT}" --enable-single && \
+ make && make install && make clean && \
+ ./configure --enable-openmp --enable-threads --enable-mpi --enable-static --with-pic --prefix="${FFTW_ROOT}" && \
+ make && make install && make clean && \
+ ./configure --enable-openmp --enable-threads --enable-mpi --enable-static --with-pic --prefix="${FFTW_ROOT}" --enable-long-double && \
+ make && make install && make clean && \
+ rm -rf /tmp/fftw-*
+
+RUN cd /tmp && \ 
+ git clone https://github.com/pyFFTW/pyFFTW && \
+ cd pyFFTW && \
+ STATIC_FFTW_DIR="${FFTW_ROOT}/lib" CFLAGS="-Wl,-Bsymbolic -fopenmp -I${FFTW_ROOT}/include -O2" python3.9 setup.py build_ext --inplace && \
+ python3.9 -m pip install --upgrade . && \
+ rm -rf /tmp/pyFFTW
+
+RUN python3.9 -m pip install ansicolors --ignore-installed
+RUN python3.9 -m pip install numpy==1.21.5 
+
+# ensure all libraries are known by the runtime linker
+RUN ldconfig
+
+# clean cached packages
+RUN rm -rf /var/lib/apt/lists/*
+RUN rm -rf $HOME/.cache/pip/*
+RUN rm -rf /tmp/*
+
+CMD ["/bin/bash"]
--- a/ci/scripts/build_and_debug.sh
+++ b/ci/scripts/build_and_debug.sh
 #!/usr/bin/env bash
 set -feu -o pipefail

+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-"$(which python3.9)"}
+PYTHON_INCLUDE_DIR=$(${PYTHON_EXECUTABLE} -c "import sysconfig as sc; print(sc.get_paths()['include'])")
+PYTHON_LIBRARY=$(${PYTHON_EXECUTABLE} -c "import sysconfig as sc, os; print(os.path.normpath(os.path.sep.join(sc.get_config_vars('LIBDIR', 'INSTSONAME'))))")
+
 # /hysop should be mounted as read only by run_tests_in_docker.sh
 if [[ ! -d '/hysop' ]]; then
    echo "This script should not be called from host, but from within a docker image."
@@ -20,7 +24,7 @@ rm -rf "${HYSOP_DIR}/build"
 cd "${HYSOP_DIR}"
 mkdir build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake -DCMAKE_BUILD_TYPE=Debug -DVERBOSE=OFF -DWITH_SCALES=ON -DFIND_FFTW_STATIC_ONLY=ON -DFIND_FFTW_VERBOSE=ON -DPython3_EXECUTABLE="${PYTHON_EXECUTABLE}" -DPython3_INCLUDE_DIR="${PYTHON_INCLUDE_DIR}" -DPython3_LIBRARY="${PYTHON_LIBRARY}" ..
 make -j8
 make install
 cd -

--- a/ci/scripts/config.sh
+++ b/ci/scripts/config.sh
 #!/bin/bash
 set -feu -o pipefail

+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-"$(which python3.9)"}
+PYTHON_INCLUDE_DIR=$(${PYTHON_EXECUTABLE} -c "import sysconfig as sc; print(sc.get_paths()['include'])")
+PYTHON_LIBRARY=$(${PYTHON_EXECUTABLE} -c "import sysconfig as sc, os; print(os.path.normpath(os.path.sep.join(sc.get_config_vars('LIBDIR', 'INSTSONAME'))))")
+
 if [ $# -ne 5 ]; then
    echo "Usage ./config build_folder install_folder CC CXX FC"
    exit 1
@@ -22,7 +26,8 @@ INSTALL_DIR="$2"

 mkdir -p "${BUILD_DIR}"
 cd "${BUILD_DIR}"
-CC="$3" CXX="$4" FC="$5" cmake -DCMAKE_BUILD_TYPE=Release -DVERBOSE=OFF -DWITH_SCALES=ON -DPYTHON_EXECUTABLE="$(which python3.8)" -DHYSOP_INSTALL="${INSTALL_DIR}" -DFIND_FFTW_STATIC_ONLY=ON -DFIND_FFTW_VERBOSE=ON "${ROOT_DIR}"
+
+CC="$3" CXX="$4" FC="$5" cmake -DCMAKE_BUILD_TYPE=Release -DVERBOSE=OFF -DWITH_SCALES=ON -DHYSOP_INSTALL="${INSTALL_DIR}" -DFIND_FFTW_STATIC_ONLY=ON -DFIND_FFTW_VERBOSE=ON -DPython3_EXECUTABLE="${PYTHON_EXECUTABLE}" -DPython3_INCLUDE_DIR="${PYTHON_INCLUDE_DIR}" -DPython3_LIBRARY="${PYTHON_LIBRARY}" "${ROOT_DIR}"

 if [ ! -f Makefile ]; then
    echo "The makefile has not been generated."

--- a/ci/scripts/install.sh
+++ b/ci/scripts/install.sh
 #!/bin/bash
 set -feu -o pipefail

-PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-"$(which python3.8)"}
+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-"$(which python3.9)"}

 if [ $# -ne 2 ]; then
    echo "Usage ./install build_folder install_folder"
@@ -24,12 +24,12 @@ INSTALL_FOLDER="$2"
 cd "${BUILD_FOLDER}"
 make install

-if [ ! -d "${INSTALL_FOLDER}/lib/python3.8/site-packages/hysop" ]; then
-    echo "${INSTALL_FOLDER}/lib/python3.8/site-packages/hysop was not created."
+if [ ! -d "${INSTALL_FOLDER}/lib/python3.9/site-packages/hysop" ]; then
+    echo "${INSTALL_FOLDER}/lib/python3.9/site-packages/hysop was not created."
    exit 1
 fi

-export PYTHONPATH="${INSTALL_FOLDER}/lib/python3.8/site-packages"
+export PYTHONPATH="${INSTALL_FOLDER}/lib/python3.9/site-packages"
 "${PYTHON_EXECUTABLE}" -c 'import hysop; print(hysop)'

 exit 0
--- a/ci/scripts/test.sh
+++ b/ci/scripts/test.sh
 #!/bin/bash
 set -feu -o pipefail

-PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-"$(which python3.8)"}
+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-"$(which python3.9)"}

 if [ $# -lt 2 ]; then
    echo "Usage ./test install_folder hysop_folder [cache_dir] [backup_cache_dir]"
@@ -67,7 +67,7 @@ if [ "${HAS_CACHE_DIR}" = true ]; then
 fi

 # Environment variables
-export PYTHONPATH="${INSTALL_DIR}/lib/python3.8/site-packages:${INSTALL_DIR}"
+export PYTHONPATH="${INSTALL_DIR}/lib/python3.9/site-packages:${INSTALL_DIR}"
 export PYTHONHASHSEED=42  # get consistent hashes accross MPI processes
 export PYOPENCL_COMPILER_OUTPUT=0
 export MPLBACKEND='cairo'

--- a/ci/utils/build_docker_image.sh
+++ b/ci/utils/build_docker_image.sh
@@ -2,6 +2,6 @@
 set -feu -o pipefail
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 NTHREADS="$(nproc)"
-UBUNTU_RELEASE=${1:-groovy}
+UBUNTU_RELEASE=${1:-jammy}

 docker build --rm=true --build-arg "NTHREADS=$NTHREADS" -t "keckj/hysop:${UBUNTU_RELEASE}" -f "${SCRIPT_DIR}/../docker_images/ubuntu/${UBUNTU_RELEASE}/Dockerfile" "${SCRIPT_DIR}/../.."
--- a/ci/utils/pull_docker_image.sh
+++ b/ci/utils/pull_docker_image.sh
 #!/usr/bin/env bash
 set -euf -o pipefail
-UBUNTU_RELEASE=${1:-groovy}
+UBUNTU_RELEASE=${1:-jammy}
 docker logout
 docker pull "keckj/hysop:${UBUNTU_RELEASE}"
--- a/ci/utils/push_docker_image.sh
+++ b/ci/utils/push_docker_image.sh
 #!/usr/bin/env bash
 set -euf -o pipefail
-UBUNTU_RELEASE=${1:-groovy}
+UBUNTU_RELEASE=${1:-jammy}
 docker login
 docker push "keckj/hysop:${UBUNTU_RELEASE}"
 docker logout
--- a/ci/utils/run_ci.sh
+++ b/ci/utils/run_ci.sh
 #!/usr/bin/env bash
 set -feu -o pipefail
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-UBUNTU_RELEASE=${1:-groovy}
+UBUNTU_RELEASE=${1:-jammy}
 DOCKER_IMG="keckj/hysop:${UBUNTU_RELEASE}"
 CONTAINER_ID='hysop_build_and_test'


--- a/ci/utils/run_debug.sh
+++ b/ci/utils/run_debug.sh
 #!/usr/bin/env bash
 set -feu -o pipefail
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-UBUNTU_RELEASE=${1:-groovy}
+UBUNTU_RELEASE=${1:-jammy}
 DOCKER_IMG="keckj/hysop:${UBUNTU_RELEASE}"
 CONTAINER_ID='hysop_build_and_debug'


--- a/ci/utils/run_docker_image.sh
+++ b/ci/utils/run_docker_image.sh
 #!/usr/bin/env bash
 set -feu -o pipefail
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-UBUNTU_RELEASE=${1:-groovy}
+UBUNTU_RELEASE=${1:-jammy}
 docker run --cap-add=SYS_PTRACE -it -v "${SCRIPT_DIR}/../..:/hysop:ro" "keckj/hysop:${UBUNTU_RELEASE}"
--- a/hysop/backend/device/codegen/base/struct_codegen.py
+++ b/hysop/backend/device/codegen/base/struct_codegen.py
@@ -35,8 +35,11 @@ class StructCodeGenerator(OpenClCodeGenerator):

    def c_decl(self):
        assert (self.context is not None)
-        (dtype, cdecl) = cl.tools.match_dtype_to_c_struct( \
-                self.device, self.ctype.replace('struct',''), self.dtype, self.context)
+        (dtype, cdecl) = cl.tools.match_dtype_to_c_struct(
+                self.device,
+                self.ctype.replace('struct', ''),
+                self.dtype,
+                self.context)
        return cdecl

    def gencode(self, comments, ctype_overrides):

--- a/hysop/backend/device/codegen/base/union_codegen.py
+++ b/hysop/backend/device/codegen/base/union_codegen.py
@@ -30,7 +30,10 @@ class UnionCodeGenerator(OpenClCodeGenerator):
    def c_decl(self):
        assert (self.context is not None)
        dtype,cdecl = cl.tools.match_dtype_to_c_struct( \
-                self.device,self.ctype.replace('union',''),self.dtype,context=self.context)
+                self.device,
+                self.ctype.replace('union',''),
+                self.dtype,
+                self.context)
        return cdecl

    def fields(self):

--- a/hysop/backend/device/codegen/symbolic/expr.py
+++ b/hysop/backend/device/codegen/symbolic/expr.py
@@ -454,6 +454,9 @@ class OpenClPrinter(C99CodePrinter):
        self.typegen = typegen
        self.codegen = codegen

+    def _handle_UnevaluatedExpr(self, expr):
+        return expr
+
    def doprint(self, expr, terminate=True):
        res = super(OpenClPrinter, self).doprint(expr)
        if terminate and (res != InstructionTermination):

--- a/hysop/backend/host/fortran/operator/scales_advection.py
+++ b/hysop/backend/host/fortran/operator/scales_advection.py
+
+try:
+    from hysop import f2hysop
+except ImportError:
+    msgE = 'Could not load f2hysop submodule. '
+    msgE += 'Try to recompile HySoP.'
+    print(msgE)
+    raise
+
 try:
    from hysop.f2hysop import scales2py as scales
 except ImportError:
-    msgE = 'scales package not available for your hysop install.'
-    msgE += 'Try to recompile with WITH_SCALES=ON'
-    raise ImportError(msgE)
+    msgE = 'Scales package not available for your hysop install. '
+    msgE += 'Try to recompile HySoP with WITH_SCALES=ON.'
+    print(msgE)
+    raise
+
 from hysop import __VERBOSE__, __DEBUG__
 from hysop.constants import HYSOP_REAL
 from hysop.tools.decorators import debug

--- a/hysop/core/tests/test_checkpoint.sh
+++ b/hysop/core/tests/test_checkpoint.sh
 #!/usr/bin/env bash
 set -feu -o pipefail
-PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3}
+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3.9}
 MPIRUN_EXECUTABLE=${MPIRUN_EXECUTABLE:-mpirun}
 MPIRUN_TASKS_OPTION='-np'
 if [ "${MPIRUN_EXECUTABLE}" = "srun" ]; then MPIRUN_TASKS_OPTION='-n'; fi

--- a/hysop/core/tests/test_tasks.sh
+++ b/hysop/core/tests/test_tasks.sh
 #!/usr/bin/env bash
 set -x
-PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3}
+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3.9}
 MPIRUN_EXECUTABLE=${MPIRUN_EXECUTABLE:-mpirun}
 MPIRUN_TASKS_OPTION='-np'
 if [ "${MPIRUN_EXECUTABLE}" = "srun" ]; then MPIRUN_TASKS_OPTION='-n'; fi
No results found