From c342491666628ef17f4777f387efba3916a47631 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Keck <Jean-Baptiste.Keck@imag.fr>
Date: Sat, 18 Apr 2020 11:26:55 +0200
Subject: [PATCH] lexicographical topological sort
---
CMakeLists.txt | 4 +-
ci/docker_images/ubuntu/bionic/Dockerfile | 320 ++++++++--------------
hysop/core/graph/graph.py | 14 +-
hysop/core/graph/graph_builder.py | 9 +-
hysop/core/graph/tests/test_graph.py | 8 +-
hysop/numerics/fft/_mkl_fft.py | 3 +-
requirements.txt | 1 +
7 files changed, 140 insertions(+), 219 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3274ee31f..dbdad4951 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -147,10 +147,12 @@ find_python_module(tee REQUIRED)
find_python_module(colors REQUIRED) # ansicolor package
find_python_module(argparse_color_formatter REQUIRED)
find_python_module(primefac REQUIRED)
-find_python_module(graph_tool REQUIRED)
+find_python_module(networkx REQUIRED)
find_python_module(pyfftw REQUIRED)
find_python_module(backports.weakref REQUIRED) # python-backports.weakref
#find_python_module(backports.functools-lru-cache REQUIRED) # python-backports.functools-lru-cache
+find_python_module(matplotlib OPTIONAL)
+find_python_module(pyvis OPTIONAL)
find_package( OpenCL )
if(${OpenCL_LIBRARY}) # Some opencl related python package fails to import on non OpenCL machines (cluster's frontend for instance)
diff --git a/ci/docker_images/ubuntu/bionic/Dockerfile b/ci/docker_images/ubuntu/bionic/Dockerfile
index 9f1153b4d..28363a675 100644
--- a/ci/docker_images/ubuntu/bionic/Dockerfile
+++ b/ci/docker_images/ubuntu/bionic/Dockerfile
@@ -2,247 +2,153 @@
FROM ubuntu:bionic
MAINTAINER Jean-Baptiste.Keck@imag.fr
+# parallel builds
+ARG NTHREADS
+ENV MAKEFLAGS "-j${NTHREADS}"
+
# upgrade initial image
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update
RUN apt-get full-upgrade -y
# get build tools and required libraries
-RUN apt-get install -y expat
-RUN apt-get install -y unzip
-RUN apt-get install -y xz-utils
-RUN apt-get install -y automake
-RUN apt-get install -y libtool
-RUN apt-get install -y pkg-config
-RUN apt-get install -y cmake
-RUN apt-get install -y git
-RUN apt-get install -y vim
-RUN apt-get install -y ssh
-RUN apt-get install -y clang
-RUN apt-get install -y gcc
-RUN apt-get install -y gfortran
-RUN apt-get install -y cython
-RUN apt-get install -y swig
-RUN apt-get install -y lsb-core
-RUN apt-get install -y cpio
-RUN apt-get install -y libnuma1
-RUN apt-get install -y libpciaccess0
-RUN apt-get install -y libreadline-dev
-RUN apt-get install -y libboost-all-dev
-RUN apt-get install -y libblas-dev
-RUN apt-get install -y liblapack-dev
-RUN apt-get install -y libcgal-dev
-RUN apt-get install -y libatlas-base-dev
-RUN apt-get install -y libopenblas-dev
-RUN apt-get install -y libgfortran3
-RUN apt-get install -y libgcc1
-RUN apt-get install -y libopenmpi-dev
-RUN apt-get install -y libhdf5-openmpi-dev
-RUN apt-get install -y libfftw3-dev
-RUN apt-get install -y libfftw3-mpi-dev
-RUN apt-get install -y libgmp-dev
-RUN apt-get install -y libmpfr-dev
-RUN apt-get install -y libmpc-dev
-RUN apt-get install -y libsparsehash-dev
-RUN apt-get install -y libcairo-dev
-RUN apt-get install -y libcairomm-1.0-dev
-RUN apt-get install -y python
-RUN apt-get install -y python-dev
-RUN apt-get install -y python-pip
-RUN apt-get install -y python-tk
-RUN apt-get install -y opencl-headers
-RUN apt-get install -y ocl-icd-libopencl1
-RUN apt-get install -y clinfo
-
-# python packages
+RUN apt-get install -y expat unzip xz-utils automake libtool pkg-config cmake git vim ssh clang gcc gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgfortran3 libgcc1 libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev python python-dev python-pip python-tk opencl-headers ocl-icd-libopencl1 clinfo
+
+# python packages using pip
RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade backports.weakref
-RUN pip install --upgrade backports.tempfile
-RUN pip install --upgrade cffi
-RUN pip install --upgrade wheel
-RUN pip install --upgrade pytest
-RUN pip install --upgrade numpy
-RUN pip install --upgrade scipy
-RUN pip install --upgrade sympy
-RUN pip install --upgrade matplotlib
-RUN pip install --upgrade mpi4py
+RUN pip install --upgrade numpy setuptools cffi wheel pytest
+RUN pip install --upgrade backports.weakref backports.tempfile scipy sympy matplotlib mpi4py gmpy2 psutil py-cpuinfo Mako subprocess32 editdistance portalocker colors.py tee primefac pycairo weave argparse_color_formatter networkx pyvis
RUN CC=mpicc HDF5_MPI="ON" pip install --upgrade --no-binary=h5py h5py
-RUN pip install --upgrade gmpy2
-RUN pip install --upgrade psutil
-RUN pip install --upgrade py-cpuinfo
-RUN pip install --upgrade Mako
-RUN pip install --upgrade subprocess32
-RUN pip install --upgrade editdistance
-RUN pip install --upgrade portalocker
-RUN pip install --upgrade colors.py
-RUN pip install --upgrade tee
-RUN pip install --upgrade primefac
-RUN pip install --upgrade pycairo
-RUN pip install --upgrade weave
-RUN pip install --upgrade argparse_color_formatter
-RUN pip install --upgrade numba
-RUN pip install --upgrade poetry
# For documentation
-# RUN pip install --upgrade sphinx
-# RUN pip install --upgrade sphinxcontrib-bibtex
-# RUN pip install --upgrade sphinx_bootstrap_theme
-# RUN pip install --upgrade strip-hints
-# RUN cd /tmp && git clone https://github.com/sphinx-contrib/doxylink.git && cd doxylink/sphinxcontrib/doxylink \
-# && mv doxylink.py doxylink.py3 && strip-hints doxylink.py3 > doxylink.py && rm doxylink.py3 \
-# && mv parsing.py parsing.py3 && strip-hints parsing.py3 > parsing.py && rm parsing.py3 \
-# && cd ../.. && python setup.py install
-
-
-# scitools (python-scitools does not exist on ubuntu:bionic)
-RUN cd /tmp \
- && git clone https://github.com/hplgit/scitools \
- && cd scitools \
- && pip install . \
- && cd - \
- && rm -Rf /tmp/scitools
+# RUN pip install --upgrade sphinx sphinxcontrib-bibtex sphinx_bootstrap_theme strip-hints
+# RUN cd /tmp git clone https://github.com/sphinx-contrib/doxylink.git cd doxylink/sphinxcontrib/doxylink && \
+# mv doxylink.py doxylink.py3 strip-hints doxylink.py3 > doxylink.py rm doxylink.py3 && \
+# mv parsing.py parsing.py3 strip-hints parsing.py3 > parsing.py rm parsing.py3 && \
+# cd ../.. python setup.py install
+
+# llvm + numba + llvmlite (llvmlite 0.32 has a bug with llvm8)
+RUN apt-get install -y llvm-8-dev libclang-8-dev clang-8
+ENV LLVM_CONFIG=llvm-config-8
+RUN pip install --upgrade numba llvmlite==0.31.0
# patchelf
-RUN cd /tmp \
- && git clone https://github.com/NixOS/patchelf \
- && cd patchelf \
- && ./bootstrap.sh \
- && ./configure \
- && make \
- && make install \
- && cd - \
- && rm -Rf /tmp/patchelf
-
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
- && echo 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main' >> /etc/apt/sources.list \
- && echo 'deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main' >> /etc/apt/sources.list \
- && apt-get update \
- && apt-get install --assume-yes llvm-3.9 clang-3.9 libllvm3.9 libclang-3.9-dev
+RUN cd /tmp && \
+ git clone https://github.com/NixOS/patchelf && \
+ cd patchelf && \
+ ./bootstrap.sh && \
+ ./configure && \
+ make && \
+ make install && \
+ cd - && \
+ rm -Rf /tmp/patchelf
# Intel OpenCl
-RUN cd /tmp \
-&& mkdir intel \
-&& cd intel \
-&& wget http://registrationcenter-download.intel.com/akdlm/irc_nas/12556/opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz \
-&& tar -xvzf opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz \
-&& cd opencl_runtime_16.1.2_x64_rh_6.4.0.37 \
-&& ls -la \
-&& sed -i "s/ACCEPT_EULA=decline/ACCEPT_EULA=accept/g" "silent.cfg" \
-&& ./install.sh --silent ./silent.cfg \
-&& cd /tmp \
-&& rm -Rf /tmp/intel
+RUN cd /tmp && \
+ mkdir intel && \
+ cd intel && \
+ wget http://registrationcenter-download.intel.com/akdlm/irc_nas/12556/opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz && \
+ tar -xvzf opencl_runtime_16.1.2_x64_rh_6.4.0.37.tgz && \
+ cd opencl_runtime_16.1.2_x64_rh_6.4.0.37 && \
+ ls -la && \
+ sed -i "s/ACCEPT_EULA=decline/ACCEPT_EULA=accept/g" "silent.cfg" && \
+ ./install.sh --silent ./silent.cfg && \
+ cd /tmp && \
+ rm -Rf /tmp/intel
# Fix OpenCl ICD
RUN ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /usr/lib/x86_64-linux-gnu/libOpenCL.so
RUN ldconfig
# pyopencl
-RUN cd /tmp \
-&& pip install pybind11 \
-&& git clone https://github.com/inducer/pyopencl \
-&& cd pyopencl \
-&& git submodule update --init \
-&& ./configure.py \
-&& make \
-&& pip install --upgrade . \
-&& cd - \
-&& rm -Rf /tmp/pyopencl
+RUN cd /tmp && \
+ pip install pybind11 && \
+ git clone https://github.com/inducer/pyopencl && \
+ cd pyopencl && \
+ git submodule update --init && \
+ ./configure.py && \
+ make && \
+ pip install --upgrade . && \
+ cd - && \
+ rm -Rf /tmp/pyopencl
# oclgrind
-RUN apt-get install --assume-yes --allow-unauthenticated llvm-6.0 clang-6.0 libllvm6.0 libclang-6.0-dev
-RUN cd /tmp \
- && git clone https://github.com/jrprice/Oclgrind \
- && cd Oclgrind \
- && mkdir build \
- && cd build \
- && cmake -DCMAKE_BUILD_TYPE=Release .. \
- && make \
- && make install \
- && cd - \
- && rm -Rf /tmp/Oclgrind
+RUN cd /tmp && \
+ git clone https://github.com/jrprice/Oclgrind && \
+ cd Oclgrind && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd - && \
+ rm -Rf /tmp/Oclgrind
# clpeak
-RUN cd /tmp \
- && git clone https://github.com/krrishnarraj/clpeak \
- && cd clpeak/ \
- && mkdir build \
- && cd build/ \
- && cmake .. \
- && make \
- && mv clpeak /usr/local/bin/ \
- && cd - \
- && rm -Rf /tmp/clpeak
+RUN cd /tmp && \
+ git clone https://github.com/krrishnarraj/clpeak && \
+ cd clpeak/ && \
+ mkdir build && \
+ cd build/ && \
+ cmake .. && \
+ make && \
+ mv clpeak /usr/local/bin/ && \
+ cd - && \
+ rm -Rf /tmp/clpeak
# clFFT
-RUN cd /tmp \
- && ln -s /usr/local/lib /usr/local/lib64 \
- && git clone https://github.com/clMathLibraries/clFFT \
- && cd clFFT \
- && cd src \
- && mkdir build \
- && cd build \
- && cmake -DCMAKE_BUILD_TYPE=Release .. \
- && make \
- && make install \
- && cd - \
- && rm -Rf /tmp/clFFT
+RUN cd /tmp && \
+ ln -s /usr/local/lib /usr/local/lib64 && \
+ git clone https://github.com/clMathLibraries/clFFT && \
+ cd clFFT && \
+ cd src && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd - && \
+ rm -Rf /tmp/clFFT
# gpyFFT
-RUN cd /tmp \
- && git clone https://github.com/geggo/gpyfft \
- && cd gpyfft \
- && pip install . \
- && cd - \
- && rm -Rf /tmp/gpyfft
-
-# python graphtools
-RUN cd /tmp \
- && wget https://downloads.skewed.de/graph-tool/graph-tool-2.26.tar.bz2 \
- && tar -xvjf graph-tool-2.26.tar.bz2 \
- && cd graph-tool-2.26 \
- && ./autogen.sh \
- && mkdir pycairo   \
- && find /usr/ -name 'pycairo.h' -exec cp {} ./pycairo/pycairo.h \; \
- && CPPFLAGS=-I. ./configure \
- && CPPFLAGS=-I. make -j16 \
- && make install \
- && cd - \
- && rm -Rf /tmp/graph-tool-2.26
+RUN cd /tmp && \
+ git clone https://github.com/geggo/gpyfft && \
+ cd gpyfft && \
+ pip install . && \
+ cd - && \
+ rm -Rf /tmp/gpyfft
# pyfftw (with R2R transforms - experimental branch)
-RUN cd /tmp \
- && git clone https://github.com/drwells/pyFFTW \
- && cd pyFFTW \
- && git checkout r2r-try-two \
- && sed -i 's/\(fftw3[fl]\?_\)threads/\1omp/g' setup.py \
- && pip install . \
- && cd - \
- && rm -Rf /tmp/pyFFTW
+RUN cd /tmp && \
+ git clone https://github.com/drwells/pyFFTW && \
+ cd pyFFTW && \
+ git checkout r2r-try-two && \
+ sed -i 's/\(fftw3[fl]\?_\)threads/\1omp/g' setup.py && \
+ pip install . && \
+ cd - && \
+ rm -Rf /tmp/pyFFTW
# HPTT (CPU tensor permutation library)
-RUN cd /tmp
- && git clone https://gitlab.com/keckj/hptt \
- && cd hptt \
- && mkdir build \
- && cd build \
- && cmake -DCMAKE_BUILD_TYPE=Release .. \
- && make -j8 \
- && make install \
- && cd ../pythonAPI \
- && pip install --upgrade . \
- && cd /tmp \
- && rm -Rf /tmp/hptt
+RUN cd /tmp && \
+ git clone https://gitlab.com/keckj/hptt && \
+ cd hptt && \
+ mkdir build && \
+ cd build && \
+ cmake -DCMAKE_BUILD_TYPE=Release .. && \
+ make && \
+ make install && \
+ cd ../pythonAPI && \
+ pip install --upgrade . && \
+ cd /tmp && \
+ rm -Rf /tmp/hptt
# fork of memory_tempfile for python 2.7
-RUN cd /tmp
- && git clone https://gitlab.com/keckj/memory-tempfile \
- && cd memory-tempfile \
- && poetry install \
- && poetry build \
- && pip install ./dist/memory-tempfile-*.tar.gz \
- && cd /tmp \
- && rm -Rf /tmp/memory-tempfile
+RUN cd /tmp && \
+ git clone https://gitlab.com/keckj/memory-tempfile && \
+ cd memory-tempfile && \
+ pip install . && \
+ cd /tmp && \
+ rm -Rf /tmp/memory-tempfile
# ensure all libraries are known by the runtime linker
RUN ldconfig
diff --git a/hysop/core/graph/graph.py b/hysop/core/graph/graph.py
index 21466bd2d..e7bcfbe2d 100644
--- a/hysop/core/graph/graph.py
+++ b/hysop/core/graph/graph.py
@@ -4,11 +4,13 @@ from hysop.constants import MemoryOrdering
from hysop.tools.types import check_instance, first_not_None
from hysop.tools.decorators import not_implemented, debug, wraps, profile
-is_directed_acyclic_graph = networkx.algorithms.dag.is_directed_acyclic_graph
+def is_directed_acyclic_graph(graph):
+ return networkx.algorithms.dag.is_directed_acyclic_graph(graph)
def transitive_reduction(graph):
reduced_graph = networkx.algorithms.dag.transitive_reduction(graph)
- # copy back edge attributes
+ # copy back edge attributes (node data is automatically transferred
+ # because nodes are the data (VertexAttributes))
for node in reduced_graph:
for out_node in reduced_graph[node]:
for (k,v) in graph[node][out_node].items():
@@ -18,8 +20,12 @@ def transitive_reduction(graph):
def all_simple_paths(graph, src, dst):
return tuple(networkx.algorithms.simple_paths.all_simple_paths(graph, src, dst))
-def topological_sort(graph):
- return tuple(networkx.algorithms.dag.topological_sort(graph))
+def lexicographical_topological_sort(graph):
+ # Lexicographical sort ensures a unique permutations of nodes
+ # such that they are in the same topological order on each
+ # MPI process. Else operators will not be executed in the same
+ # order and everything deadlocks on MPI synchronization.
+ return tuple(networkx.algorithms.dag.lexicographical_topological_sort(graph))
def new_directed_graph():
return networkx.DiGraph()
diff --git a/hysop/core/graph/graph_builder.py b/hysop/core/graph/graph_builder.py
index d0351a65b..e1614af36 100644
--- a/hysop/core/graph/graph_builder.py
+++ b/hysop/core/graph/graph_builder.py
@@ -12,7 +12,7 @@ from hysop.topology.cartesian_topology import CartesianTopologyState
from hysop.core.graph.graph import (new_directed_graph, new_vertex, new_edge,
is_directed_acyclic_graph, transitive_reduction,
- topological_sort, all_simple_paths)
+ lexicographical_topological_sort, all_simple_paths)
from hysop.core.graph.computational_graph import ComputationalGraph
from hysop.core.graph.computational_node import ComputationalGraphNode
from hysop.core.graph.computational_operator import ComputationalGraphOperator
@@ -433,9 +433,10 @@ class GraphBuilder(object):
# ie. remove useless redondant dependencies
reduced_graph = transitive_reduction(graph)
- # Topological sort
- # ie. find out operator order for execution purposes
- sorted_nodes = topological_sort(reduced_graph)
+ # Lexicographical topological sort
+ # => find out operator order for execution purposes
+ # => have to be exactly the same on each MPI process.
+ sorted_nodes = lexicographical_topological_sort(reduced_graph)
for (i, node) in enumerate(sorted_nodes):
node.op_ordering = i
diff --git a/hysop/core/graph/tests/test_graph.py b/hysop/core/graph/tests/test_graph.py
index a142a4a75..52b0b754e 100644
--- a/hysop/core/graph/tests/test_graph.py
+++ b/hysop/core/graph/tests/test_graph.py
@@ -1,4 +1,5 @@
-from hysop.domain.box import Box
+import tempfile
+from hysop.domain.box import Box
from hysop.topology.cartesian_topology import CartesianTopology
from hysop.tools.parameters import CartesianDiscretization
from hysop.fields.continuous_field import Field
@@ -97,10 +98,13 @@ class TestGraph(object):
g.discretize()
g.setup(None)
g.apply()
+
+ with tempfile.NamedTemporaryFile(suffix='.html') as f:
+ g.to_html(f.name)
if display:
g.display()
if __name__ == '__main__':
test = TestGraph()
- test.test_graph_build(display=True)
+ test.test_graph_build(display=False)
diff --git a/hysop/numerics/fft/_mkl_fft.py b/hysop/numerics/fft/_mkl_fft.py
index 94c329dce..a095dfa05 100644
--- a/hysop/numerics/fft/_mkl_fft.py
+++ b/hysop/numerics/fft/_mkl_fft.py
@@ -4,7 +4,8 @@ FFT interface for fast Fourier Transforms using Intel MKL (numpy interface).
:class:`~hysop.numerics.MklFFTPlan`
/!\ -- OPENMP CONFLICT WITH GRAPHTOOLS --
-/!\ Only works if MKL_THREADING_LAYER is set to OMP because graph_tools is compiled against GNU OpenMP.
+/!\ Only works if MKL_THREADING_LAYER is set to OMP if some
+ dependencies are compiled against GNU OpenMP.
/!\ May also work with MKL_THREADING_LAYER=TBB and SEQUENCIAL but not INTEL.
Required version of mkl_fft is: https://gitlab.com/keckj/mkl_fft
diff --git a/requirements.txt b/requirements.txt
index f99a62727..6728ee0a5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,3 +22,4 @@ numba
configparser
backports.tempfile
backports.weakref
+networkx
--
GitLab