From 78ca7641a01390e1db5f188e90f3769be6a16b23 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Keck <jean-baptiste.keck@imag.fr>
Date: Fri, 9 Mar 2018 21:49:40 +0100
Subject: [PATCH] fix for local opencl ghost exchangers
---
.../device/opencl/opencl_array_backend.py | 12 +++---
.../opencl/opencl_copy_kernel_launchers.py | 7 ++--
hysop/fields/ghost_exchangers.py | 3 --
hysop/fields/tests/test_cartesian.py | 39 ++++++++++---------
4 files changed, 30 insertions(+), 31 deletions(-)
diff --git a/hysop/backend/device/opencl/opencl_array_backend.py b/hysop/backend/device/opencl/opencl_array_backend.py
index 91865e3d3..aa9879678 100644
--- a/hysop/backend/device/opencl/opencl_array_backend.py
+++ b/hysop/backend/device/opencl/opencl_array_backend.py
@@ -457,11 +457,11 @@ class OpenClArrayBackend(ArrayBackend):
if dst.nbytes != src.nbytes:
raise TypeError("'dst' has non-matching nbytes.")
if src.size:
- queue = first_not_None(queue, src.default_queue, dst.default_queue)
- evt = cl.enqueue_copy(queue=queue,
- dest=dst.base, src=src.base,
- dest_offset=dst.offset, src_offset=src.offset,
- byte_count=src.nbytes)
+ from hysop.backend.device.opencl.opencl_copy_kernel_launchers \
+ import OpenClCopyBufferRectLauncher
+ kl = OpenClCopyBufferRectLauncher.from_slices('buffer',
+ src=src, dst=dst)
+ evt = kl(queue=queue)
if async:
return evt
else:
@@ -1549,7 +1549,7 @@ class OpenClArrayBackend(ArrayBackend):
# only c-contiguous arrays are handled by pyopencl
a.handle.fill(value=value, queue=queue)
else:
- # else we do a nasty trick with a copy
+ # else we fill on cpu and copy it to device
b = a.backend.full(shape=a.shape, dtype=a.dtype, fill_value=value)
a.copy_from(b, queue=queue)
diff --git a/hysop/backend/device/opencl/opencl_copy_kernel_launchers.py b/hysop/backend/device/opencl/opencl_copy_kernel_launchers.py
index 7835d6845..eb99a74cc 100644
--- a/hysop/backend/device/opencl/opencl_copy_kernel_launchers.py
+++ b/hysop/backend/device/opencl/opencl_copy_kernel_launchers.py
@@ -385,7 +385,8 @@ class OpenClCopyBufferRectLauncher(OpenClCopyKernelLauncher):
assert pitches[-1] == dtype.itemsize
pitches = pitches[:-1]
region[-1] *= dtype.itemsize
- origin[-1] *= dtype.itemsize + start_offset
+ origin[-1] *= dtype.itemsize
+ origin[-1] += start_offset
return data, region, origin, pitches
@@ -414,7 +415,7 @@ class OpenClCopyBufferRectLauncher(OpenClCopyKernelLauncher):
msg0+='\n'
msg0 = msg0.format(src.shape, src.dtype, src_slices,
dst.shape, dst.dtype, dst_slices,
- '{}', '{}')
+ '{}', '{}')
src_slices, src_dtype, src_nelems, src_bytes, src_indices = \
cls._format_slices(src, src_slices)
@@ -464,7 +465,7 @@ class OpenClCopyBufferRectLauncher(OpenClCopyKernelLauncher):
_dst_origin[:copy_dims] = dst_origin[::-1]
_src_pitches[:copy_dims-1] = src_pitches[::-1]
_dst_pitches[:copy_dims-1] = dst_pitches[::-1]
-
+
return OpenClCopyBufferRectLauncher(varname=varname,
src=src_data, dst=dst_data, region=tuple(_region),
src_origin=tuple(_src_origin), dst_origin=tuple(_dst_origin),
diff --git a/hysop/fields/ghost_exchangers.py b/hysop/fields/ghost_exchangers.py
index 55ccad788..cfa2efc71 100644
--- a/hysop/fields/ghost_exchangers.py
+++ b/hysop/fields/ghost_exchangers.py
@@ -636,9 +636,6 @@ class CartesianDiscreteFieldGhostExchanger(GhostExchanger):
varname=varname+'_from_right_ghosts_buffer',
src=tmp, dst=buf, dst_slices=outer_right)
- print 'INNER LEFT', inner_left
- print 'OUTER_RIGHT', outer_right
-
# exchange all right inner ghosts to left outer ghosts
kl += OpenClCopyBufferRectLauncher.from_slices(
varname=varname+'_to_left_ghosts_buffer',
diff --git a/hysop/fields/tests/test_cartesian.py b/hysop/fields/tests/test_cartesian.py
index e22f3fb4d..715228a4c 100644
--- a/hysop/fields/tests/test_cartesian.py
+++ b/hysop/fields/tests/test_cartesian.py
@@ -156,6 +156,8 @@ def test_mpi_ghost_exchange(comm):
npts = (53,47,59,23)[:dim]
nghosts = (2,1,0,3)[:dim]
+ npts = (5,)*dim
+ nghosts = (1,)*dim
discretization = Discretization(npts, nghosts)
domain = Box(dim=dim)
@@ -177,12 +179,12 @@ def test_mpi_ghost_exchange(comm):
def ghost_base(i,d,rank,local_dir):
return (i+1)*17 + (d+1)*13 + (local_dir+1)*11 + (rank+1)*7
- def ghost_vals(buf,i,d,rank,local_dir):
- if (buf is None):
- return None
- base = np.full_like(buf, fill_value=ghost_base(i,d,rank,local_dir))
- I = np.ix_(*tuple(np.arange(buf.shape[direction], dtype=buf.dtype)
- for direction in xrange(buf.ndim)))
+ def ghost_vals(shape,dtype,i,d,rank,local_dir):
+ if (shape is None) or (len(shape)==0):
+ raise ValueError('Shape is None or an empty tuple: {}'.format(shape))
+ base = np.full(shape=shape, dtype=dtype, fill_value=ghost_base(i,d,rank,local_dir))
+ I = np.ix_(*tuple(np.arange(shape[direction], dtype=dtype) \
+ for direction in xrange(len(shape))))
return base + I[d]
for F in (F0, F1):
@@ -202,11 +204,12 @@ def test_mpi_ghost_exchange(comm):
sys.stdout.flush()
lghosts, rghosts, shape = dF.inner_ghost_slices[d]
_lghosts, _rghosts, shape = dF.outer_ghost_slices[d]
+
for (i,data) in enumerate(dF.data):
- data[lghosts] = ghost_vals(data[lghosts].get(), i,d,rank,0)
- data[rghosts] = ghost_vals(data[rghosts].get(), i,d,rank,1)
- data[_lghosts] = 0
- data[_rghosts] = 0
+ data[lghosts] = ghost_vals(shape, dtype, i,d,rank,0)
+ data[rghosts] = ghost_vals(shape, dtype, i,d,rank,1)
+ data[_lghosts] = +1
+ data[_rghosts] = -1
dF.exchange_ghosts(directions=d, exchange_method=exchange_method)
@@ -217,16 +220,14 @@ def test_mpi_ghost_exchange(comm):
left_rank, right_rank = rank, rank
for (i,data) in enumerate(dF.data):
- ldata = data[lghosts]
- rdata = data[rghosts]
+ ldata = np.atleast_1d(data[lghosts].get())
+ rdata = np.atleast_1d(data[rghosts].get())
if (ldata is not None):
- ldata = ldata.get()
- target_val = ghost_vals(ldata, i,d,left_rank,1)
- assert np.allclose(ldata, target_val), (rank, target_val)
+ target_vals = ghost_vals(ldata.shape, dtype,i,d,left_rank,1)
+ assert np.allclose(ldata, target_vals), (rank, target_vals)
if (rdata is not None):
- rdata = rdata.get()
- target_val = ghost_vals(rdata, i,d,right_rank,0)
- assert np.allclose(rdata, target_val), target_val
+ target_vals = ghost_vals(rdata.shape, dtype, i,d,right_rank,0)
+ assert np.allclose(rdata, target_vals), target_val
if rank==0:
print
@@ -382,4 +383,4 @@ if __name__ == '__main__':
test_serial_initialization_3d()
test_mpi_ghost_exchange(comm=comm)
- # test_mpi_ghost_accumulate(comm=comm)
+ test_mpi_ghost_accumulate(comm=comm)
--
GitLab