add python3 docker image, fix build for python3

CMakeLists.txt

@@ -154,8 +154,6 @@ find_python_module(argparse_color_formatter REQUIRED)
 find_python_module(primefac     REQUIRED)
 find_python_module(networkx     REQUIRED)
 find_python_module(pyfftw       REQUIRED)
-find_python_module(backports.weakref REQUIRED) # python-backports.weakref
-#find_python_module(backports.functools-lru-cache REQUIRED) # python-backports.functools-lru-cache
 find_python_module(matplotlib OPTIONAL)
 find_python_module(pyvis OPTIONAL)
 
@@ -187,7 +185,7 @@ endif()
 # Find python build dir name --> needed for tests and doc
 if(USE_CXX OR USE_FORTRAN)
   execute_process(
-    COMMAND ${PYTHON_EXECUTABLE} -c "import distutils.util as ut ; import distutils.sysconfig as sy; print 'lib.'+ut.get_platform()+'-'+sy.get_python_version()"
+    COMMAND ${PYTHON_EXECUTABLE} -c "import distutils.util as ut ; import distutils.sysconfig as sy; print('lib.'+ut.get_platform()+'-'+sy.get_python_version())"
     OUTPUT_VARIABLE ${PROJECT_NAME}_PYTHON_BUILD_DIR)
   string(STRIP ${${PROJECT_NAME}_PYTHON_BUILD_DIR} ${PROJECT_NAME}_PYTHON_BUILD_DIR)
   set(HYSOP_BUILD_PYTHONPATH ${CMAKE_BINARY_DIR}/build/${${PROJECT_NAME}_PYTHON_BUILD_DIR} CACHE INTERNAL "Python package build path")

ci/docker_images/ubuntu/groovy/Dockerfile

+# Test docker for gitlab-ci
+FROM ubuntu:groovy
+MAINTAINER Jean-Baptiste.Keck@imag.fr
+
+# parallel builds
+ARG NTHREADS
+ENV MAKEFLAGS "-j${NTHREADS}"
+
+# upgrade initial image
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update
+RUN apt-get full-upgrade -y
+
+# get build tools and required libraries
+RUN apt-get install -y --no-install-recommends expat unzip xz-utils automake libtool pkg-config cmake rsync git vim ssh curl wget ca-certificates gcc g++ gfortran lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libblas-dev liblapack-dev libgcc-9-dev libgfortran-9-dev libgmp-dev libmpfr-dev libmpc-dev python3.8-dev opencl-headers swig libgmp-dev libmpfr-dev libmpc-dev libcairo-dev libcairomm-1.0-dev python3.8-tk
+
+# python packages using pip3.8
+RUN cd /tmp && \
+ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+ python3.8 get-pip.py && \
+ pip3.8 install --upgrade pip && \
+ rm -f /tmp/get-pip.py
+RUN pip3.8 install --upgrade numpy setuptools cffi wheel pytest pybind11 cython
+
+# OpenMPI 4 + mpi4py (enable mpi1 compatibility for mpi4py)
+ENV MPI_ROOT "/usr/local"
+RUN cd /tmp && \
+ wget https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.5.tar.gz && \
+ tar -xvzf openmpi-*.tar.gz && \
+ rm -f openmpi-*.tar.gz && \
+ cd openmpi-* && \
+ ./configure --enable-shared --disable-static --with-threads=posix --enable-ipv6 --prefix="${MPI_ROOT}" --with-hwloc=internal --with-libevent=internal --enable-mpi1-compatibility && \
+ make -j$(nproc) && \
+ make install && \
+ rm -rf /tmp/openmpi-*
+
+ENV MPICC "${MPI_ROOT}/bin/mpicc"
+RUN ldconfig && pip3.8 install --upgrade mpi4py
+
+# HDF5 + h5py (v1.10.6 is currently the last supported by h5py)
+RUN cd /tmp && \
+ wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.6/src/hdf5-1.10.6.tar.gz && \
+ tar -xvzf hdf5-*.tar.gz && \
+ rm -rf hdf5-*.tar.gz && \
+ cd hdf5-* && \
+ CC="${MPICC}" ./configure --prefix="${MPI_ROOT}" --enable-parallel --enable-shared=yes --enable-static=no && \
+ make -j$(nproc) && \
+ make install && \
+ rm -rf /tmp/hdf5-*
+RUN CC="${MPICC}" HDF5_MPI="ON" HDF5_VERSION="1.10.6" HDF5_DIR="${MPI_ROOT}" pip3.8 install --upgrade --no-binary=h5py h5py
+
+# llvm + numba + llvmlite (llvmlite 0.32 has a bug with llvm8)
+RUN apt-get install -y llvm-8-dev libclang-8-dev clang-8
+ENV LLVM_CONFIG=llvm-config-8
+RUN pip3.8 install --upgrade numba llvmlite==0.31.0
+
+# other python packages
+RUN pip3.8 install --upgrade scipy sympy matplotlib gmpy2 psutil py-cpuinfo Mako subprocess32 editdistance portalocker colors.py tee primefac pycairo argparse_color_formatter networkx pyvis zarr numcodecs jsonpickle memory-tempfile
+
+# patchelf
+RUN cd /tmp && \
+ git clone https://github.com/NixOS/patchelf && \
+ cd patchelf && \
+ ./bootstrap.sh && \
+ ./configure && \
+ make && \
+ make install && \
+ cd - && \
+ rm -Rf /tmp/patchelf
+
+# Intel experimental OpenCL platform with SYCL support (2020-09)
+ENV LD_LIBRARY_PATH "/opt/intel/oclcpuexp/x64:${LD_LIBRARY_PATH}"
+RUN cd /tmp && \
+ mkdir -p /opt/intel/oclcpuexp && \
+ wget https://github.com/intel/llvm/releases/download/2020-09/oclcpuexp-2020.11.8.0.27_rel.tar.gz && \
+ tar -xvzf oclcpuexp-*.tar.gz && \
+ mv x64/ /opt/intel/oclcpuexp/ && \
+ mv clbltfnshared.rtl /opt/intel/oclcpuexp/ && \
+ rm -rf *.rtl oclcpuexp-* && \
+ wget https://github.com/oneapi-src/oneTBB/releases/download/v2020.3/tbb-2020.3-lin.tgz && \
+ tar -xvzf tbb-*.tgz && \
+ rm -f tbb-*.tgz && \
+ mv tbb/lib/intel64/gcc4.8/* /opt/intel/oclcpuexp/x64/ && \
+ rm -f /usr/local/lib/libOpenCL.so && \
+ rm -f /usr/local/lib/libOpenCL.so && \
+ rm -f /usr/local/lib/libOpenCL.so.1 && \
+ rm -f /usr/local/lib/libOpenCL.so.2.0 && \
+ ln -s /opt/intel/oclcpuexp/x64/libOpenCL.so /usr/local/lib/libOpenCL.so && \
+ ln -s /opt/intel/oclcpuexp/x64/libOpenCL.so.1 /usr/local/lib/libOpenCL.so.1 && \
+ ln -s /opt/intel/oclcpuexp/x64/libOpenCL.so.2.0 /usr/local/lib/libOpenCL.so.2.0 && \
+ mkdir -p /etc/OpenCL/vendors && \
+ echo /opt/intel/oclcpuexp/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+ rm -rf /tmp/tbb*
+
+# clinfo 2.2.18 (2018)
+RUN cd /tmp && \
+ wget https://github.com/Oblomov/clinfo/archive/2.2.18.04.06.tar.gz && \
+ tar -xvzf *.tar.gz && \
+ rm -f *.tar.gz && \
+ cd clinfo-* && \
+ make && \
+ mv clinfo /usr/local/bin && \
+ rm -rf /tmp/clinfo-*
+
+# clpeak 1.1.0 RC2 (2019)
+RUN cd /tmp && \
+ wget https://github.com/krrishnarraj/clpeak/archive/1.1.0.tar.gz && \
+ tar -xvzf *.tar.gz && \
+ rm -f *.tar.gz && \
+ cd clpeak-* && \
+ mkdir build && \
+ cd build/ && \
+ cmake .. && \
+ make && \
+ mv clpeak /usr/local/bin && \
+ rm -rf /tmp/clpeak-*
+
+# pyopencl
+RUN cd /tmp && \
+ git clone https://github.com/inducer/pyopencl && \
+ cd pyopencl && \
+ git submodule update --init && \
+ git checkout v2020.1 && \
+ python3.8 configure.py && \
+ make && \
+ pip3.8 install --upgrade . && \
+ cd - && \
+ rm -Rf /tmp/pyopencl
+
+# oclgrind
+RUN cd /tmp && \
+ git clone https://github.com/jrprice/Oclgrind && \
+ cd Oclgrind && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd - && \
+ rm -Rf /tmp/Oclgrind
+
+# clFFT
+RUN cd /tmp && \
+ ln -s /usr/local/lib /usr/local/lib64 && \
+ git clone https://github.com/clMathLibraries/clFFT && \
+ cd clFFT && \
+ cd src && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd - && \
+ rm -Rf /tmp/clFFT
+
+# gpyFFT
+RUN cd /tmp && \
+ git clone https://github.com/geggo/gpyfft && \
+ cd gpyfft && \
+ pip3.8 install . && \
+ cd - && \
+ rm -Rf /tmp/gpyfft
+
+# HPTT (CPU tensor permutation library)
+RUN cd /tmp && \
+ git clone https://gitlab.com/keckj/hptt && \
+ cd hptt && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd ../pythonAPI && \
+ pip3.8 install --upgrade . && \
+ cd /tmp && \
+ rm -Rf /tmp/hptt
+
+# python flint (FLINT2 + ARB + python-flint)
+RUN cd /tmp \
+  && wget https://github.com/wbhart/flint2/archive/v2.6.1.tar.gz \
+  && tar -xvzf v2.6.1.tar.gz \
+  && cd flint2-2.6.1 \
+  && ./configure \
+  && make -j$(nproc) \
+  && make install \
+  && cd - \
+  && rm -rf flint2-2.6.1
+RUN cd /tmp \
+  && wget https://github.com/fredrik-johansson/arb/archive/2.18.1.tar.gz \
+  && tar -xvzf 2.18.1.tar.gz \
+  && cd arb-2.18.1 \
+  && ./configure \
+  && make -j$(nproc) \
+  && make install \
+  && cd - \
+  && rm -rf arb-2.18.1
+RUN pip3.8 install --upgrade python-flint
+
+# static fftw + pyfftw (with R2R transforms)
+# Weird pyfftw bug : not passing -O2 explicitely during build causes a segfault on import...
+# See https://bugs.gentoo.org/548776
+ENV FFTW_ROOT="/usr/local"
+ADD ci/patch/pyfftw.patch /tmp/pyfftw.patch
+RUN cd /tmp && \
+ wget http://www.fftw.org/fftw-3.3.8.tar.gz && \
+ tar -xvzf fftw-*.tar.gz && \
+ rm -f fftw-*.tar.gz && \
+ cd fftw-* && \
+ ./configure --enable-openmp --enable-threads --enable-mpi --enable-static --with-pic --prefix="${FFTW_ROOT}" --enable-single && \
+ make -j$(nproc) && \
+ make install && \
+ make clean && \
+ ./configure --enable-openmp --enable-threads --enable-mpi --enable-static --with-pic --prefix="${FFTW_ROOT}" && \
+ make -j8 && \
+ make install && \
+ make clean && \
+ ./configure --enable-openmp --enable-threads --enable-mpi --enable-static --with-pic --prefix="${FFTW_ROOT}" --enable-long-double && \
+ make -j8 && \
+ make install && \
+ rm -rf /tmp/fftw-*
+
+RUN cd /tmp && \ 
+ git config --global user.email "you@example.com" && \
+ git config --global user.name "Your Name" && \
+ git clone https://github.com/grlee77/pyFFTW && \
+ cd pyFFTW && \
+ git checkout r2r-try-three && \
+ git remote add fork https://github.com/pyFFTW/pyFFTW && \
+ git fetch fork && \
+ git merge v0.12.0 && \
+ STATIC_FFTW_DIR="${FFTW_ROOT}/lib" CFLAGS="-Wl,-Bsymbolic -fopenmp -I${FFTW_ROOT}/include -O2" python3.8 setup.py build_ext --inplace && \
+ pip3.8 install --upgrade . && \
+ rm -rf /tmp/pyFFTW
+
+RUN pip3.8 uninstall --yes primefac && \
+ pip3.8 install git+git://github.com/elliptic-shiho/primefac-fork@master
+
+# ensure all libraries are known by the runtime linker
+RUN ldconfig
+
+# clean cached packages
+RUN rm -rf /var/lib/apt/lists/*
+RUN rm -rf $HOME/.cache/pip/*
+RUN rm -rf /tmp/*
+
+CMD ["/bin/bash"]

cmake/FindPythonFull.cmake

@@ -49,6 +49,7 @@ else()
     string(REGEX REPLACE ".*exec_prefix:([^\n]+).*$" "\\1" PYTHON_PREFIX ${python_config})
     string(REGEX REPLACE ".*\nversion:([^\n]+).*$" "\\1" PYTHON_VERSION ${python_config})
     string(REGEX REPLACE ".*\npy_inc_dir:([^\n]+).*$" "\\1" PYTHON_INCLUDE_DIRS ${python_config})
+    string(REGEX REPLACE ".*\npy_lib_dir:([^\n]+).*$" "\\1" PYTHON_LIBRARY_DIRS ${python_config})
     string(REGEX REPLACE ".*\nsite_packages_dir:([^\n]+).*$" "\\1" PYTHON_SITE_PACKAGES_DIR ${python_config})
     string(REGEX REPLACE "([0-9]+).([0-9]+)" "\\1\\2" PYTHON_VERSION_NO_DOTS ${PYTHON_VERSION})
     if(WIN32)
@@ -61,9 +62,8 @@ else()
       NAMES
       python${PYTHON_VERSION_NO_DOTS} python${PYTHON_VERSION}
       NO_DEFAULT_PATH
-      HINTS ${PYTHON_PREFIX} ${PYTHON_PREFIX}/lib/python${PYTHON_VERSION}/config ${PYTHON_PREFIX}/lib/python${PYTHON_VERSION}/config-${CMAKE_LIBRARY_ARCHITECTURE}
-      PATH_SUFFIXES lib libs
-      )
+      HINTS ${PYTHON_LIBRARY_DIRS} ${PYTHON_PREFIX} ${PYTHON_PREFIX}/lib/python${PYTHON_VERSION}/config ${PYTHON_PREFIX}/lib/python${PYTHON_VERSION}/config
+      PATH_SUFFIXES lib libs lib/${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE})
 
     set(PYTHON_LIBRARIES ${PYTHON_LIBRARY} CACHE FILEPATH "Python libraries" FORCE)
 

cmake/PythonInstallSetup.cmake

@@ -37,19 +37,19 @@ function(set_python_install_path)
     # and on which python is used (virtualenv or not)
     # First, we need to check if '--user' option works in the current environment.
     execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-      "import site; print site.ENABLE_USER_SITE" OUTPUT_VARIABLE ENABLE_USER)
+    "import site; print(site.ENABLE_USER_SITE)" OUTPUT_VARIABLE ENABLE_USER)
     string(STRIP ${ENABLE_USER} ENABLE_USER)
     
     if(ENABLE_USER) # --user works ...
       # Find install path for --user (site.USER_SITE)
       execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-	"import site; print site.USER_BASE" OUTPUT_VARIABLE USER_BASE)
+          "import site; print(site.USER_BASE)" OUTPUT_VARIABLE USER_BASE)
       string(STRIP ${USER_BASE} USER_BASE)
       list(APPEND python_install_options --user)#prefix=${USER_BASE})
       # Get python user site and install path = USER_SITE + project_name
       set(PYTHON_COMMAND_GET_INSTALL_DIR
-       "import site, os, sys ; print os.path.join(site.USER_BASE, os.path.join(\"lib\", os.path.join(\"python\" + str(sys.version_info.major) + '.' + str(sys.version_info.minor),
- \"site-packages\")))")
+          "import site, os, sys ; print(os.path.join(site.USER_BASE, os.path.join(\"lib\", os.path.join(\"python\" + str(sys.version_info.major) + '.' + str(sys.version_info.minor),
+ \"site-packages\"))))")
     execute_process(
       COMMAND ${PYTHON_EXECUTABLE} -c "${PYTHON_COMMAND_GET_INSTALL_DIR}"
       OUTPUT_VARIABLE PY_INSTALL_DIR)

cmake/explore_python_config.py

@@ -31,10 +31,10 @@
 # For details see the accompanying COPYING-CMAKE-SCRIPTS file.
 
 import sys
-import distutils.sysconfig
+import distutils.sysconfig as cfg
 
 print("exec_prefix:%s" % sys.exec_prefix)
 print("version:%s" % sys.version[:3])
-print("py_inc_dir:%s" % distutils.sysconfig.get_python_inc())
-print("site_packages_dir:%s" % distutils.sysconfig.get_python_lib(plat_specific=1))
-
+print("py_inc_dir:%s" % cfg.get_python_inc())
+print("py_lib_dir:%s" % cfg.get_config_var('LIBDIR'))
+print("site_packages_dir:%s" % cfg.get_python_lib(plat_specific=1))