From 60fe35ef0bc78039a8afe654dae0deff42a460eb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Keck <Jean-Baptiste.Keck@imag.fr> Date: Thu, 28 Feb 2019 11:57:50 +0100 Subject: [PATCH] moved MultiresolutionFilter to the SpatialFiltering interface --- ci/scripts/test.sh | 95 ++++++++++--------- examples/multiresolution/scalar_advection.py | 10 +- .../python/operator/multiresolution_filter.py | 92 ------------------ .../host/python/operator/spatial_filtering.py | 16 +++- hysop/operator/base/spatial_filtering.py | 55 +++++++++++ hysop/operator/multiresolution_filter.py | 22 ----- hysop/operator/spatial_filtering.py | 19 ++-- test_examples.sh | 28 +----- 8 files changed, 143 insertions(+), 194 deletions(-) delete mode 100755 hysop/backend/host/python/operator/multiresolution_filter.py delete mode 100755 hysop/operator/multiresolution_filter.py diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 15c13d830..840465dd9 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -41,59 +41,68 @@ if [ "$HAS_CACHE_DIR" = true ]; then fi fi +DO_TESTS=${DO_TESTS:-true} +DO_EXAMPLES=${DO_EXAMPLES:-true} +DO_LONG_TESTS=${DO_LONG_TESTS:-false} + export PYTHONPATH="$INSTALL_DIR/lib/python2.7/site-packages:$INSTALL_DIR:$PYTHONPATH" +export MPLBACKEND='cairo' export HYSOP_VERBOSE=0 export HYSOP_DEBUG=0 export HYSOP_PROFILE=0 export HYSOP_KERNEL_DEBUG=0 -export MPLBACKEND='cairo' python -c 'import hysop; print hysop' -# long backend dependent tests are disabled for ci -#python "$HYSOP_DIR/backend/device/codegen/kernels/tests/test_directional_advection.py" -#python "$HYSOP_DIR/backend/device/codegen/kernels/tests/test_directional_remesh.py" +if [ "$DO_TESTS" = true ]; then + python "$HYSOP_DIR/core/arrays/tests/test_array.py" + python "$HYSOP_DIR/core/graph/tests/test_graph.py" + python "$HYSOP_DIR/fields/tests/test_fields.py" + $HYSOP_DIR/fields/tests/test_cartesian.sh + python "$HYSOP_DIR/numerics/tests/test_fft.py" + python "$HYSOP_DIR/operator/tests/test_analytic.py" + python "$HYSOP_DIR/operator/tests/test_transpose.py" + python "$HYSOP_DIR/operator/tests/test_fd_derivative.py" + python "$HYSOP_DIR/operator/tests/test_absorption.py" + python "$HYSOP_DIR/operator/tests/test_directional_advection.py" + python "$HYSOP_DIR/operator/tests/test_directional_diffusion.py" + python "$HYSOP_DIR/operator/tests/test_directional_stretching.py" + python "$HYSOP_DIR/operator/tests/test_custom_symbolic.py" + python "$HYSOP_DIR/operator/tests/test_spectral_derivative.py" + python "$HYSOP_DIR/operator/tests/test_spectral_curl.py" + python "$HYSOP_DIR/operator/tests/test_diffusion.py" + python "$HYSOP_DIR/operator/tests/test_poisson.py" + python "$HYSOP_DIR/operator/tests/test_solenoidal_projection.py" + python "$HYSOP_DIR/operator/tests/test_poisson_curl.py" -python "$HYSOP_DIR/core/arrays/tests/test_array.py" -python "$HYSOP_DIR/core/graph/tests/test_graph.py" -python "$HYSOP_DIR/fields/tests/test_fields.py" -$HYSOP_DIR/fields/tests/test_cartesian.sh -python "$HYSOP_DIR/numerics/tests/test_fft.py" -python "$HYSOP_DIR/operator/tests/test_analytic.py" -python "$HYSOP_DIR/operator/tests/test_transpose.py" -python "$HYSOP_DIR/operator/tests/test_fd_derivative.py" -python "$HYSOP_DIR/operator/tests/test_absorption.py" -python "$HYSOP_DIR/operator/tests/test_directional_advection.py" -python "$HYSOP_DIR/operator/tests/test_directional_diffusion.py" -python "$HYSOP_DIR/operator/tests/test_directional_stretching.py" -python "$HYSOP_DIR/operator/tests/test_custom_symbolic.py" -python "$HYSOP_DIR/operator/tests/test_spectral_derivative.py" -python "$HYSOP_DIR/operator/tests/test_spectral_curl.py" -python "$HYSOP_DIR/operator/tests/test_diffusion.py" -python "$HYSOP_DIR/operator/tests/test_poisson.py" -python "$HYSOP_DIR/operator/tests/test_solenoidal_projection.py" -python "$HYSOP_DIR/operator/tests/test_poisson_curl.py" + # If scales (fortran advection library) is installed + python -c "from hysop.f2hysop import scales2py as scales" && python "$HYSOP_DIR/operator/tests/test_scales_advection.py" + python -c "from hysop.f2hysop import scales2py as scales" && python "$HYSOP_DIR/operator/tests/test_bilevel_advection.py" +fi -# If scales (fortran advection library) is installed -python -c "from hysop.f2hysop import scales2py as scales" && python "$HYSOP_DIR/operator/tests/test_scales_advection.py" -python -c "from hysop.f2hysop import scales2py as scales" && python "$HYSOP_DIR/operator/tests/test_bilevel_advection.py" +if [ "$DO_LONG_TESTS" = true ]; then + python "$HYSOP_DIR/backend/device/codegen/kernels/tests/test_directional_advection.py" + python "$HYSOP_DIR/backend/device/codegen/kernels/tests/test_directional_remesh.py" +fi -export HYSOP_VERBOSE=1 -EXAMPLE_DIR="$HYSOP_DIR/../examples" -EXAMPLE_OPTIONS='-cp default -maxit 2' -python "$EXAMPLE_DIR/analytic/analytic.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/scalar_diffusion/scalar_diffusion.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/scalar_advection/scalar_advection.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/multiresolution/scalar_advection.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/shear_layer/shear_layer.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl python $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl opencl $EXAMPLE_OPTIONS -python -c "from hysop.f2hysop import scales2py as scales" && python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl fortran $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_bubble.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_bubble_levelset.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_bubble_levelset_penalization.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_jet_levelset.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_periodic.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_symmetrized.py" $EXAMPLE_OPTIONS +if [ "$DO_EXAMPLES" = true ]; then + export HYSOP_VERBOSE=1 + EXAMPLE_DIR="$HYSOP_DIR/../examples" + EXAMPLE_OPTIONS='-cp default -maxit 2' + python "$EXAMPLE_DIR/analytic/analytic.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/scalar_diffusion/scalar_diffusion.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/scalar_advection/scalar_advection.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/multiresolution/scalar_advection.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/shear_layer/shear_layer.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl python $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl opencl $EXAMPLE_OPTIONS + python -c "from hysop.f2hysop import scales2py as scales" && python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl fortran $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/bubble/periodic_bubble.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/bubble/periodic_bubble_levelset.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/bubble/periodic_bubble_levelset_penalization.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/bubble/periodic_jet_levelset.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_periodic.py" $EXAMPLE_OPTIONS + python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_symmetrized.py" $EXAMPLE_OPTIONS +fi if [ "$HAS_CACHE_DIR" = true ]; then cp -r /root/.cache/* $CACHE_DIR/ diff --git a/examples/multiresolution/scalar_advection.py b/examples/multiresolution/scalar_advection.py index 7f76270c1..a55ad7819 100644 --- a/examples/multiresolution/scalar_advection.py +++ b/examples/multiresolution/scalar_advection.py @@ -34,7 +34,7 @@ def compute(args): if (component in (None,1)): data[1][...] = 1.0 for x in coords[0]: - data[1][...] *= np.cos(x) + data[1][...] *= np.sin(x) # Define domain dim = args.ndim @@ -125,10 +125,10 @@ def compute(args): lpfilter = LowpassFilter(input_variables={scalar: snpts}, output_variables={scalar: npts}, filtering_method=FilteringMethod.SPECTRAL, - plot_input_energy=IOParams(filepath=spectral_path, - filename='E_fine_{fname}_{ite}', frequency=args.dump_freq), - plot_output_energy=IOParams(filepath=spectral_path, - filename='E_coarse_{fname}_{ite}', frequency=args.dump_freq), + dump_input_energy=IOParams(filepath=spectral_path, + filename='E_fine_{fname}', frequency=args.dump_freq), + dump_output_energy=IOParams(filepath=spectral_path, + filename='E_coarse_{fname}', frequency=args.dump_freq), implementation=impl, **extra_op_kwds) diff --git a/hysop/backend/host/python/operator/multiresolution_filter.py b/hysop/backend/host/python/operator/multiresolution_filter.py deleted file mode 100755 index 6d7fba041..000000000 --- a/hysop/backend/host/python/operator/multiresolution_filter.py +++ /dev/null @@ -1,92 +0,0 @@ -""" Data interpolation between topology that -differs only from global resolution -""" -from hysop.operator.base.redistribute_operator import RedistributeOperatorBase -from hysop.topology.cartesian_topology import CartesianTopology -from hysop.core.mpi.topo_tools import TopoTools -from hysop.core.graph.graph import op_apply -from hysop.tools.numpywrappers import npw -from hysop.constants import Backend -from hysop.tools.decorators import debug - - -class PythonMultiresolutionFilter(RedistributeOperatorBase): - """ - Interpolation between topologies when topology - differs only by global resolution. - - Source and target must: - *be CartesianTopology topologies with a target - global resolution that divide source global resolution - *be defined on the same communicator - *work on the same number of mpi processes - - Notes: - ====== - This operator currently supports only topology subdivisions - from fine resolution (as source) to coarse resolution (as target) - TODO: implement general fine to coarse filter - TODO: implement coarse to fine filter - """ - - @classmethod - def can_redistribute(cls, source_topo, target_topo): - tin = source_topo - tout = target_topo - - # source and target must be CartesianTopology topology - # defined on HostArrayBackend - if not isinstance(tin, CartesianTopology): - return False - if not isinstance(tout, CartesianTopology): - return False - - # target global resolution must divide source global resolution - source_res = tin.mesh.grid_resolution - target_res = tout.mesh.grid_resolution - if not npw.all(npw.mod(source_res, target_res) == 0): - return False - - # defined on the same communicator - # and work on the same number of mpi process - if not TopoTools.compare_comm(tin.parent, tout.parent): - return False - # must have the same mpi topology - if not npw.array_equal(tin.proc_shape, tout.proc_shape): - return False - if not npw.array_equal(tin.proc_coords, tout.proc_coords): - return False - - # Backend ckeck - if tin.backend.kind is not Backend.HOST: - return False - if tout.backend.kind is not Backend.HOST: - return False - - return True - - @debug - def __init__(self, variables=None, **kwds): - super(PythonMultiresolutionFilter, self).__init__(**kwds) - - @debug - def discretize(self): - super(PythonMultiresolutionFilter, self).discretize() - tin, tout = self.source_topo, self.target_topo - source_res = tin.mesh.grid_resolution - target_res = tout.mesh.grid_resolution - self._scale_factor = source_res / target_res - dFin = self.get_input_discrete_field(self.variable) - dFout = self.get_output_discrete_field(self.variable) - self.dFin = dFin - self.dFout = dFout - self.input_buffers = dFin.compute_buffers - self.output_buffers = dFout.compute_buffers - - @op_apply - def apply(self, **kwds): - super(PythonMultiresolutionFilter, self).apply(**kwds) - for vi, vo in zip(self.input_buffers, self.output_buffers): - vo[...] = vi[::self._scale_factor[0], - ::self._scale_factor[1], - ::self._scale_factor[2]] diff --git a/hysop/backend/host/python/operator/spatial_filtering.py b/hysop/backend/host/python/operator/spatial_filtering.py index 0c4a61645..82bcaf19d 100644 --- a/hysop/backend/host/python/operator/spatial_filtering.py +++ b/hysop/backend/host/python/operator/spatial_filtering.py @@ -6,7 +6,7 @@ from hysop.core.graph.graph import op_apply from hysop.fields.continuous_field import Field from hysop.parameters.parameter import Parameter from hysop.topology.cartesian_descriptor import CartesianTopologyDescriptors -from hysop.operator.base.spatial_filtering import RemeshLowpassFilterBase, SpectralLowpassFilterBase +from hysop.operator.base.spatial_filtering import RemeshLowpassFilterBase, SpectralLowpassFilterBase, SubgridLowpassFilterBase class PythonRemeshLowpassFilter(RemeshLowpassFilterBase, HostOperator): """ @@ -64,3 +64,17 @@ class PythonSpectralLowpassFilter(SpectralLowpassFilterBase, HostOperator): scaling = 1.0 / self.Bt.output_buffer[(0,)*self.FOUT.ndim] return scaling + +class PythonSubgridLowpassFilter(SubgridLowpassFilterBase, HostOperator): + """ + Python implementation for lowpass spatial filtering: small grid -> coarse grid + byt just taking subpoints. + """ + + @op_apply + def apply(self, simulation, **kwds): + """Apply subgrid filter.""" + super(PythonSubgridLowpassFilter, self).apply(**kwds) + self.fout[...] = self.fin[...] + self.dFout.exchange_ghosts() + diff --git a/hysop/operator/base/spatial_filtering.py b/hysop/operator/base/spatial_filtering.py index e8bd852e0..018f0d6a7 100644 --- a/hysop/operator/base/spatial_filtering.py +++ b/hysop/operator/base/spatial_filtering.py @@ -298,3 +298,58 @@ class RemeshLowpassFilterBase(LowpassFilterBase): self.fin, self.fout = fin, fout + +class SubgridLowpassFilterBase(LowpassFilterBase): + """ + Base implementation for lowpass spatial filtering: small grid -> coarse grid + using remeshing kernels. + """ + + @debug + def get_field_requirements(self): + requirements = super(SubgridLowpassFilterBase, self).get_field_requirements() + dim = self.Fin.dim + + Fin_topo, Fin_requirements = requirements.get_input_requirement(self.Fin) + try: + Fin_dx = Fin_topo.space_step + except AttributeError: + Fin_dx = Fin_topo.mesh.space_step + + Fout_topo, Fout_requirements = requirements.get_output_requirement(self.Fout) + try: + Fout_dx = Fout_topo.space_step + except AttributeError: + Fout_dx = Fout_topo.mesh.space_step + + ratio = Fout_dx / Fin_dx + msg='Destination grid is finer than source grid: {}'.format(ratio) + assert (ratio>=1.0).all(), msg + + iratio = ratio.astype(npw.int32) + msg='Grid ratio is not an integer on at least one axis: {}'.format(ratio) + assert (ratio==iratio).all(), msg + + self.iratio = iratio + + return requirements + + @debug + def discretize(self): + if self.discretized: + return + super(SubgridLowpassFilterBase, self).discretize() + dFin, dFout = self.dFin, self.dFout + + iratio = dFin.compute_resolution / dFout.compute_resolution + view = tuple(slice(None,None,r) for r in iratio) + + fin = dFin.compute_buffers[0][view] + fout = dFout.compute_buffers[0] + + msg='Something went wrong during slicing: fin.shape={}, fout.shape={}' + msg=msg.format(fin.shape, fout.shape) + assert (fin.shape == fout.shape), msg + + self.fin, self.fout = fin, fout + diff --git a/hysop/operator/multiresolution_filter.py b/hysop/operator/multiresolution_filter.py deleted file mode 100755 index 8838c207d..000000000 --- a/hysop/operator/multiresolution_filter.py +++ /dev/null @@ -1,22 +0,0 @@ -from hysop.constants import Implementation -from hysop.core.graph.computational_node_frontend import ComputationalGraphNodeFrontend -from hysop.backend.host.python.operator.multiresolution_filter import PythonMultiresolutionFilter - - -class MultiresolutionFilter(ComputationalGraphNodeFrontend): - - __implementations = { - Implementation.PYTHON: PythonMultiresolutionFilter - } - - @classmethod - def implementations(cls): - return cls.__implementations - - @classmethod - def default_implementation(cls): - return Implementation.PYTHON - - - def __init__(self, **kwds): - super(MultiresolutionFilter, self).__init__(**kwds) diff --git a/hysop/operator/spatial_filtering.py b/hysop/operator/spatial_filtering.py index 76b4176e0..02e51beea 100644 --- a/hysop/operator/spatial_filtering.py +++ b/hysop/operator/spatial_filtering.py @@ -14,30 +14,34 @@ from hysop.core.graph.node_generator import ComputationalGraphNodeGenerator from hysop.core.graph.computational_node_frontend import MultiComputationalGraphNodeFrontend FilteringMethod = EnumFactory.create('FilteringMethod', -['SPECTRAL', 'REMESH']) +['SPECTRAL', 'REMESH', 'SUBGRID']) class LowpassFilterFrontend(MultiComputationalGraphNodeFrontend): """ Interface for lowpass spatial filtering: small grid -> coarse grid Available implementations are: - *Python using remeshing kernels *Python/OpenCL using spectral filtering + *Python using remeshing kernels + *Python by just taking a subgrid (compatibility with deprecated MultiresolutionFilter) """ @classmethod def all_implementations(cls): from hysop.backend.host.python.operator.spatial_filtering import \ - PythonRemeshLowpassFilter, PythonSpectralLowpassFilter + PythonRemeshLowpassFilter, PythonSpectralLowpassFilter, PythonSubgridLowpassFilter from hysop.backend.device.opencl.operator.spatial_filtering import \ OpenClSpectralLowpassFilter ai = { - FilteringMethod.REMESH: { - Implementation.PYTHON: PythonRemeshLowpassFilter - }, FilteringMethod.SPECTRAL: { Implementation.PYTHON: PythonSpectralLowpassFilter, Implementation.OPENCL: OpenClSpectralLowpassFilter }, + FilteringMethod.REMESH: { + Implementation.PYTHON: PythonRemeshLowpassFilter + }, + FilteringMethod.SUBGRID: { + Implementation.PYTHON: PythonSubgridLowpassFilter + }, } return ai @@ -45,7 +49,8 @@ class LowpassFilterFrontend(MultiComputationalGraphNodeFrontend): def all_default_implementations(cls): adi = { FilteringMethod.REMESH: Implementation.PYTHON, - FilteringMethod.SPECTRAL: Implementation.PYTHON + FilteringMethod.SPECTRAL: Implementation.PYTHON, + FilteringMethod.SUBGRID: Implementation.PYTHON } return adi diff --git a/test_examples.sh b/test_examples.sh index acf7047f3..3c99a0ce1 100755 --- a/test_examples.sh +++ b/test_examples.sh @@ -7,29 +7,9 @@ if [ -z "$HYSOP_ROOT" ]; then echo "Setting HYSOP_ROOT to '$HYSOP_ROOT'" fi -export HYSOP_VERBOSE=1 -export HYSOP_DEBUG=0 -export HYSOP_PROFILE=0 -export HYSOP_KERNEL_DEBUG=0 -export MPLBACKEND='cairo' -python -c 'import hysop; print hysop' - -EXAMPLE_DIR="$HYSOP_ROOT/examples" -EXAMPLE_OPTIONS='-cp default -maxit 2' -python "$EXAMPLE_DIR/analytic/analytic.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/scalar_diffusion/scalar_diffusion.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/scalar_advection/scalar_advection.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/multiresolution/scalar_advection.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/shear_layer/shear_layer.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl python $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl opencl $EXAMPLE_OPTIONS -python -c "from hysop.f2hysop import scales2py as scales" && python "$EXAMPLE_DIR/taylor_green/taylor_green.py" -impl fortran $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_bubble.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_bubble_levelset.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_bubble_levelset_penalization.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/bubble/periodic_jet_levelset.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_periodic.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_symmetrized.py" $EXAMPLE_OPTIONS -python "$EXAMPLE_DIR/particles_above_salt/particles_above_salt_bc.py" $EXAMPLE_OPTIONS +export DO_TESTS=false +export DO_LONG_TESTS=false +export DO_EXAMPLES=true +$HYSOP_ROOT/ci/scripts/test.sh $HYSOP_ROOT $HYSOP_ROOT/hysop exit 0 -- GitLab