From c9b88f65566d41430ec337958469ac7b37608012 Mon Sep 17 00:00:00 2001
From: Jean-Matthieu Etancelin <jean-matthieu.etancelin@univ-pau.fr>
Date: Thu, 28 Feb 2019 17:10:42 +0100
Subject: [PATCH] Re-enable HDF5 parallel interface. (reverting commit
9c195f33). It works only in a few specific cases (1d topology + cuting
direction in the last varying direction)
---
CMakeLists.txt | 14 ++---
ci/docker_images/ubuntu/bionic/Dockerfile | 2 +-
hysop/__init__.py.in | 1 -
hysop/operator/hdf_io.py | 68 +++++------------------
hysop/tools/io_utils.py | 40 ++++---------
setup.py.in | 7 ++-
6 files changed, 35 insertions(+), 97 deletions(-)
mode change 100644 => 100755 setup.py.in
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3f00922a9..8982adb15 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,7 +36,6 @@ option(WITH_TESTS "Enable testing. Default = OFF" ON)
option(BUILD_SHARED_LIBS "Enable dynamic library build, default = ON." ON)
option(USE_CXX "Expand hysop with some new functions from a generated c++ to python interface, wrapped into hysop.cpp2hysop module. Default = ON." OFF)
option(WITH_SCALES "compile/create scales lib and link it with HySoP. Default = ON." ON)
-option(WITH_PARALLEL_HDF5 "Enable parallel hdf5 interface. Default = OFf." OFF)
option(WITH_FFTW "Link with fftw library (required for some HySoP solvers), default = ON" ON)
option(WITH_EXTRAS "Link with some extra fortran libraries (like arnoldi solver), default = OFF" OFF)
option(WITH_GPU "Use of GPU (required for some HySoP solvers), default = ON" ON)
@@ -232,13 +231,11 @@ if(WITH_EXTRAS)
endif()
# ========= Check parallel hdf5 availability =========
-if(WITH_PARALLEL_HDF5)
- execute_process(
- COMMAND ${PYTHON_EXECUTABLE} -c "import h5py; print(h5py.get_config().mpi)"
- OUTPUT_VARIABLE H5PY_PARALLEL_ENABLED)
- if(H5PY_PARALLEL_ENABLED EQUAL "True")
- message(FATAL_ERROR "h5py is not build with parallel support.")
- endif()
+execute_process(
+ COMMAND ${PYTHON_EXECUTABLE} -c "import h5py; print(h5py.get_config().mpi)"
+ OUTPUT_VARIABLE H5PY_PARALLEL_ENABLED)
+if(H5PY_PARALLEL_ENABLED EQUAL "True")
+ message(FATAL_ERROR "h5py is not build with parallel support.")
endif()
# ========= Check which opencl devices are available on the system =========
@@ -641,7 +638,6 @@ if(VERBOSE_MODE)
message(STATUS " Project uses Scales : ${WITH_SCALES}")
message(STATUS " Project uses FFTW : ${WITH_FFTW}")
message(STATUS " Project uses GPU : ${WITH_GPU}")
- message(STATUS " Project uses parallel hdf5 interface : ${WITH_PARALLEL_HDF5}")
message(STATUS " ${PROJECT_NAME} profile mode : ${PROFILE}")
message(STATUS " ${PROJECT_NAME} debug mode : ${DEBUG}")
message(STATUS " Enable -OO run? : ${OPTIM}")
diff --git a/ci/docker_images/ubuntu/bionic/Dockerfile b/ci/docker_images/ubuntu/bionic/Dockerfile
index de940fd5f..872147e97 100644
--- a/ci/docker_images/ubuntu/bionic/Dockerfile
+++ b/ci/docker_images/ubuntu/bionic/Dockerfile
@@ -89,7 +89,7 @@ RUN pip install --upgrade numba
# RUN cd /tmp && git clone https://github.com/sphinx-contrib/doxylink.git && cd doxylink/sphinxcontrib/doxylink \
# && mv doxylink.py doxylink.py3 && strip-hints doxylink.py3 > doxylink.py && rm doxylink.py3 \
# && mv parsing.py parsing.py3 && strip-hints parsing.py3 > parsing.py && rm parsing.py3 \
-# && python setup.py install
+# && cd ../.. && python setup.py install
# scitools (python-scitools does not exist on ubuntu:bionic)
diff --git a/hysop/__init__.py.in b/hysop/__init__.py.in
index fffc7a197..9a74933c9 100644
--- a/hysop/__init__.py.in
+++ b/hysop/__init__.py.in
@@ -33,7 +33,6 @@ __GPU_ENABLED__ = "@WITH_GPU@" is "ON"
__FFTW_ENABLED__ = "@WITH_FFTW@" is "ON"
__SCALES_ENABLED__ = "@WITH_SCALES@" is "ON"
__OPTIMIZE__ = not __debug__
-__PARALLEL_HDF5__ = "@WITH_PARALLEL_HDF5@" is "ON"
__VERBOSE__ = get_env('VERBOSE', ("@VERBOSE@" is "ON"))
__DEBUG__ = get_env('DEBUG', ("@DEBUG@" is "ON"))
diff --git a/hysop/operator/hdf_io.py b/hysop/operator/hdf_io.py
index 7570d4123..2984ccb2d 100755
--- a/hysop/operator/hdf_io.py
+++ b/hysop/operator/hdf_io.py
@@ -1,5 +1,3 @@
-# coding: utf-8
-
"""I/O operators
.. currentmodule:: hysop.operator.hdf_io
@@ -11,7 +9,6 @@
"""
import functools
from abc import ABCMeta, abstractmethod
-from hysop import __PARALLEL_HDF5__
from hysop.deps import h5py, sys
from hysop.core.graph.graph import discretized
from hysop.constants import DirectionLabels, HYSOP_REAL, Backend, TranspositionState
@@ -116,8 +113,6 @@ class HDF_IO(ComputationalGraphOperator):
self.topology = None
self._local_compute_slices = None
self._global_grid_resolution = None
- self._local_grid_resolution = None
- self._all_local_grid_resolution = None
self._global_slices = None
# Dictionnary of discrete fields. Key = name in hdf file,
# Value = discrete field
@@ -188,10 +183,6 @@ class HDF_IO(ComputationalGraphOperator):
# Global resolution for hdf5 output
self._global_grid_resolution = refmesh.grid_resolution
- # Local resolution for hdf5 output
- self._local_grid_resolution = refmesh.local_resolution
- self._all_local_grid_resolution = self.topology.cart_comm.gather(
- self._local_grid_resolution, root=self.io_params.io_leader)
local_compute_slices = {}
global_compute_slices = {}
@@ -245,13 +236,9 @@ class HDF_IO(ComputationalGraphOperator):
self._hdf_file = h5py.File(filename, mode)
compression = 'gzip'
else:
- if __PARALLEL_HDF5__:
- self._hdf_file = h5py.File(filename, mode, driver='mpio',
- comm=self.topology.comm)
- compression = None
- else:
- self._hdf_file = h5py.File(filename.format(rk=self.topology.cart_rank), mode)
- compression = 'gzip'
+ self._hdf_file = h5py.File(filename, mode, driver='mpio',
+ comm=self.topology.comm)
+ compression = None
return compression
@classmethod
@@ -369,11 +356,7 @@ class HDF_Writer(HDF_IO):
"""Set output file name for current iteration"""
msg = 'count < 0, simu must be initialized.'
assert i >= 0, msg
- if self.topology.cart_size == 1 or __PARALLEL_HDF5__:
- return self.io_params.filename + "_{0:05d}".format(i) + '.h5'
- else:
- return self.io_params.filename + "_{0:05d}".format(i) + "_rk{rk:03d}.h5"
-
+ return self.io_params.filename + "_{0:05d}".format(i) + '.h5'
@op_apply
def apply(self, simulation=None, **kwds):
@@ -406,12 +389,9 @@ class HDF_Writer(HDF_IO):
write_step = tuple(step)
ds_names = self.dataset.keys()
- joinrkfiles = None
- if self.topology.cart_size > 1 and not __PARALLEL_HDF5__:
- joinrkfiles = range(self.topology.cart_size)
grid_attributes = XMF.prepare_grid_attributes(
ds_names,
- resolution, origin, step, joinrkfiles=joinrkfiles)
+ resolution, origin, step)
self.grid_attributes_template = grid_attributes
@@ -437,13 +417,9 @@ class HDF_Writer(HDF_IO):
assert (f is not None)
assert (lastp is not None)
for (i, t) in self._xdmf_data_files:
- if self.topology.cart_size == 1 or __PARALLEL_HDF5__:
- filenames = {'filename': self._get_filename(i).split('/')[-1]}
- else:
- filenames = dict(('filename'+str(r), self._get_filename(i).format(rk=r).split('/')[-1]) for r in range(self.topology.cart_size))
- filenames.update(('resolution'+str(r), XMF._list_format(self._all_local_grid_resolution[r])) for r in range(self.topology.cart_size))
+ filename = self._get_filename(i).split('/')[-1]
grid_attrs = self.grid_attributes_template.format(
- niteration=i, time=t, **filenames)
+ niteration=i, time=t, filename=filename)
f.seek(lastp)
f.write(grid_attrs)
self._last_xmf_pos = f.tell()
@@ -454,13 +430,7 @@ class HDF_Writer(HDF_IO):
self._xdmf_data_files = []
def _step_HDF5(self, simu):
- """Write an h5 file with data on each mpi process.
-
- If parallel interface of HDF5 is not enabled, each rank is
- writing its own h5 file. All files are concatenated in the xmf
- part with a 'JOIN' function. If parallel interface enabled,
- only one h5 file is written by all ranks.
- """
+ """Write an h5 file with data on each mpi process."""
# Remarks:
# - force np.float64, ParaView seems unable to read float32
# - writing compressed hdf5 files (gzip compression seems the best)
@@ -474,21 +444,13 @@ class HDF_Writer(HDF_IO):
# Get the names of output input_fields and create the corresponding
# datasets
- if self.topology.cart_size == 1 or __PARALLEL_HDF5__:
- for name in self.dataset:
- ds = self._hdf_file.create_dataset(name,
- self._global_grid_resolution,
- dtype=npw.float64,
- compression=compression)
- # In parallel, each proc must write at the right place of the dataset
- ds[self._global_compute_slices[name]] = self._data_getters[name]()
- else:
- for name in self.dataset:
- ds = self._hdf_file.create_dataset(name,
- self._local_grid_resolution,
- dtype=npw.float64,
- compression=compression)
- ds[...] = self._data_getters[name]()
+ for name in self.dataset:
+ ds = self._hdf_file.create_dataset(name,
+ self._global_grid_resolution,
+ dtype=npw.float64,
+ compression=compression)
+ # In parallel, each proc must write at the right place of the dataset
+ ds[self._global_compute_slices[name]] = self._data_getters[name]()
# Collect datas required to write the xdmf file
# --> add tuples (counter, time).
diff --git a/hysop/tools/io_utils.py b/hysop/tools/io_utils.py
index b66ec46c9..3de3fc9a4 100755
--- a/hysop/tools/io_utils.py
+++ b/hysop/tools/io_utils.py
@@ -234,7 +234,7 @@ class IOParams(namedtuple("IOParams", ['filename', 'filepath',
IO.check_dir(filename)
return super(IOParams, cls).__new__(cls, filename, filepath,
- frequency, fileformat,
+ frequency, fileformat,
io_leader, visu_leader,
kwds)
@@ -387,7 +387,7 @@ class XMF(object):
@staticmethod
def prepare_grid_attributes(dataset_names,
- resolution, origin, step, joinrkfiles=None):
+ resolution, origin, step):
"""
Prepare XDMF header as a string.
@@ -398,7 +398,6 @@ class XMF(object):
resolution: 3d tuple
origin: 3d tuple
step: 3d tuple
- joinrkfiles : (optional)
Returns:
--------
@@ -406,10 +405,7 @@ class XMF(object):
the xml-like header formattable with the following keywords:
niteration : iteration number
time: time in seconds
- filename: target file name, in sequential or with parallel hdf5 support
- filename0, ... filenameN : target file names for each rank 0 to N, in parallel without HDF5 parallel support
- resolution0, ... resolutionN : local resolutions for each rank 0 to N, in parallel without HDF5 parallel support
-
+ filename: target file name
"""
# The header (xml-like), saved in a string.
# always use a 3D mesh because paraview does not like 2D meshes (uses axe (Y,Z) instead of (X,Y)).
@@ -437,29 +433,13 @@ class XMF(object):
xml_grid += " <Attribute Name=\""
xml_grid += name + "\""
xml_grid += " AttributeType=\"Scalar\" Center=\"Node\">\n"
- if joinrkfiles is None:
- xml_grid += " <DataItem Dimensions=\""
- xml_grid += XMF._list_format(resolution) + " \""
- xml_grid += " NumberType=\"Float\" Precision=\"8\" Format=\"HDF\""
- xml_grid += " Compression=\"Raw\">\n" #
- xml_grid += " {filename}"
- xml_grid += ":/" + name
- xml_grid += "\n </DataItem>\n"
- else:
- xml_grid += " <DataItem Dimensions=\""
- xml_grid += XMF._list_format(resolution) + " \""
- xml_grid += " ItemType=\"Function\" Function=\"JOIN("
- xml_grid += " ; ".join("$"+str(i) for i in joinrkfiles)
- xml_grid += ")\">\n"
- for i in joinrkfiles:
- xml_grid += " <DataItem Dimensions=\""
- xml_grid += "{resolution"+str(i)+"}" + " \""
- xml_grid += " NumberType=\"Float\" Precision=\"8\" Format=\"HDF\""
- xml_grid += " Compression=\"Raw\">\n" #
- xml_grid += " {filename"+str(i)+"}"
- xml_grid += ":/" + name
- xml_grid += "\n </DataItem>\n"
- xml_grid += " </DataItem>\n"
+ xml_grid += " <DataItem Dimensions=\""
+ xml_grid += XMF._list_format(resolution) + " \""
+ xml_grid += " NumberType=\"Float\" Precision=\"8\" Format=\"HDF\""
+ xml_grid += " Compression=\"Raw\">\n" #
+ xml_grid += " {filename}"
+ xml_grid += ":/" + name
+ xml_grid += "\n </DataItem>\n"
xml_grid += " </Attribute>\n"
xml_grid += " </Grid>\n"
return xml_grid
diff --git a/setup.py.in b/setup.py.in
old mode 100644
new mode 100755
index 9b1630a43..370ff9bda
--- a/setup.py.in
+++ b/setup.py.in
@@ -57,7 +57,7 @@ def parseCMakeDefines(var):
defines = parseCMakeVar(var)
if len(defines)==0:
return None
-
+
# regex to match compiler defines like -DMACRO_NAME or
# -DMACRO_NAME = MACRO_VALUE
p = re.compile('\s*(?:-D)?\s*(\w+)(?:\s*=\s*(\w+))?\s*')
@@ -126,7 +126,7 @@ def create_fortran_extension(name, pyf_file=None, src_dirs=None, sources=None,
for sdir in src_dirs:
sources += glob.glob(os.path.join(sdir, '*.f95'))
sources += glob.glob(os.path.join(sdir, '*.f90'))
-
+
# Reorder source list with fortran modules
# dependencies. It seems that this is not taken into
# account in f2py or distutils.
@@ -147,6 +147,7 @@ def create_fortran_extension(name, pyf_file=None, src_dirs=None, sources=None,
# --- set compilation flags ---
fortran_flags = ['@Fortran_FLAGS@']
+ fortran_flags = list(set([_ for _ in fortran_flags[0].split(' ') if _ != '']))
# we trust cmake for external libraries and
# add them to linker, without using libraries option
@@ -308,7 +309,7 @@ if enable_fortran is "ON":
num_dirs = []
for sd in subdirs:
num_dirs.append(os.path.join(fortran_root, sd))
-
+
hysop_libdir = [ld.strip() for ld in hysop_libdir]
# hysop_libdir = ' '.join([ '-L{}'.format(ld) if ld[1]!='L' else hysop_libdir])
num_dirs.append('@GENERATED_FORTRAN_FILES_DIR@')
--
GitLab