From 0d8c129c7bad0c6c1e5202833bb4a370d480d30c Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Keck <Jean-Baptiste.Keck@imag.fr>
Date: Mon, 19 Mar 2018 18:09:37 +0100
Subject: [PATCH] debug utility
---
ci/docker_images/ubuntu/bionic/Dockerfile | 11 +-
examples/shear_layer/shear_layer.py | 11 +-
hysop/backend/device/kernel_autotuner.py | 3 +-
.../device/opencl/operator/custom_symbolic.py | 26 +---
.../operator/directional/advection_dir.py | 2 +-
hysop/core/graph/graph.py | 14 +-
.../operator/base/custom_symbolic_operator.py | 6 -
hysop/operator/base/redistribute_operator.py | 7 -
hysop/tools/debug_utils.py | 132 ++++++++++++++++++
hysop/tools/io_utils.py | 3 +-
10 files changed, 172 insertions(+), 43 deletions(-)
create mode 100644 hysop/tools/debug_utils.py
diff --git a/ci/docker_images/ubuntu/bionic/Dockerfile b/ci/docker_images/ubuntu/bionic/Dockerfile
index 424381739..c5c854a08 100644
--- a/ci/docker_images/ubuntu/bionic/Dockerfile
+++ b/ci/docker_images/ubuntu/bionic/Dockerfile
@@ -101,17 +101,20 @@ RUN cd /tmp \
# python graphtools
RUN cd /tmp \
- && git clone https://github.com/antmd/graph-tool \
- && cd graph-tool \
+ && wget https://downloads.skewed.de/graph-tool/graph-tool-2.26.tar.bz2 \
+ && tar -xvjf graph-tool-2.26.tar.bz2 \
+ && cd graph-tool-2.26 \
&& ./autogen.sh \
&& mkdir pycairo   \
&& find /usr/ -name 'pycairo.h' -exec cp {} ./pycairo/pycairo.h \; \
- && for f in $(grep -Rl '[^:]placeholders::'); do echo '%s/[^:]\zs\zeplaceholders::/std::/g | w' | vim -e $f; done; \
&& CPPFLAGS=-I. ./configure \
&& CPPFLAGS=-I. make \
&& make install \
&& cd - \
- && rm -Rf /tmp/graphtool
+ && rm -Rf /tmp/graph-tool-2.26
+
+ #&& for f in $(grep -Rl '[^:]placeholders::'); do echo '%s/[^:]\zs\zeplaceholders::/std::/g | w' | vim -e $f; done; \
+ #CPPFLAGS=-I. ./configure \
RUN echo 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main' >> /etc/apt/sources.list \
&& echo 'deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main' >> /etc/apt/sources.list \
diff --git a/examples/shear_layer/shear_layer.py b/examples/shear_layer/shear_layer.py
index 1d76375bb..12e0a33fc 100644
--- a/examples/shear_layer/shear_layer.py
+++ b/examples/shear_layer/shear_layer.py
@@ -3,6 +3,8 @@
## See Brown 1995:
## Performance of under-resolved two dimensional incompressible flow simulations
+import matplotlib
+matplotlib.use('gtk3cairo')
import sympy as sm
import numpy as np
@@ -159,8 +161,15 @@ def compute(args):
dfields = problem.input_discrete_fields
dfields[vorti].initialize(formula=init_vorticity)
+ from hysop.tools.debug_utils import ImshowDebugger
+ dbg = ImshowDebugger(data={'Wz':(dfields[vorti],0)},
+ ntimes=1,
+ enable_on_op_apply=True)
+ dbg.synchronize_queue(cl_env.default_queue)
+ dbg('initialization')
+
# Finally solve the problem
- problem.solve(simu, dry_run=args.dry_run)
+ problem.solve(simu, dry_run=args.dry_run, dbg=dbg)
# Finalize
problem.finalize()
diff --git a/hysop/backend/device/kernel_autotuner.py b/hysop/backend/device/kernel_autotuner.py
index 303a40ea2..9ac783478 100644
--- a/hysop/backend/device/kernel_autotuner.py
+++ b/hysop/backend/device/kernel_autotuner.py
@@ -22,7 +22,7 @@ class KernelAutotuner(object):
FULL_RESULTS_KEY = '__FULL_RESULTS__'
DUMP_LAST_TUNED_KERNEL = False
- STORE_FULL_KERNEL_SOURCES = True
+ STORE_FULL_KERNEL_SOURCES = False
@staticmethod
def _hash_func():
@@ -144,7 +144,6 @@ class KernelAutotuner(object):
assert prg is None
assert kernel is None
- assert (cached_kernel_src is None)^(self.STORE_FULL_KERNEL_SOURCES), cached_kernel_src
global_work_size = npw.asintegerarray(global_work_size)
local_work_size = npw.asintegerarray(local_work_size)
diff --git a/hysop/backend/device/opencl/operator/custom_symbolic.py b/hysop/backend/device/opencl/operator/custom_symbolic.py
index 7eac15b15..57585e053 100644
--- a/hysop/backend/device/opencl/operator/custom_symbolic.py
+++ b/hysop/backend/device/opencl/operator/custom_symbolic.py
@@ -14,28 +14,16 @@ class OpenClCustomSymbolicOperator(CustomSymbolicOperatorBase, OpenClOperator):
def __init__(self, **kwds):
super(OpenClCustomSymbolicOperator, self).__init__(**kwds)
- @debug
- def get_work_properties(self):
- requests = super(OpenClCustomSymbolicOperator, self).get_work_properties()
-
- for sout in self.output_discrete_fields.values():
- full_outer_ghosts = sout.get_outer_ghost_slices()
- for direction in xrange(sout.dim):
- if sout.ghosts[direction]==0:
- continue
- dirlabel = DirectionLabels[sout.dim-direction-1]
- full_ghost_layer_shape = full_outer_ghosts[direction][-1]
- request = MemoryRequest.empty_like(a=sout, shape=full_ghost_layer_shape, nb_components=2)
- requests.push_mem_request(sout.name+'_ghost_layers_'+dirlabel, request)
- return requests
-
@debug
def setup(self, work):
super(OpenClCustomSymbolicOperator, self).setup(work)
- self._collect_kernels(work)
+ self._collect_kernels()
- def _collect_kernels(self, work):
- kl = self._collect_symbolic_kernel()
+ def _collect_kernels(self):
+ kl = OpenClKernelListLauncher(name='advec_remesh')
+ kl += self._collect_symbolic_kernel()
+ for sout in self.output_discrete_fields.values():
+ kl += sout.exchange_ghosts(build_launcher=True)
self.kl = kl
def _collect_symbolic_kernel(self):
@@ -60,8 +48,6 @@ class OpenClCustomSymbolicOperator(CustomSymbolicOperatorBase, OpenClOperator):
def apply(self, **kwds):
queue = self.cl_env.default_queue
evt = self.kl(queue=queue, **self._update_input_params())
- for sout in self.output_discrete_fields.values():
- evt = sout.exchange_ghosts(queue=queue, evt=evt)
@classmethod
def supports_mpi(cls):
diff --git a/hysop/backend/device/opencl/operator/directional/advection_dir.py b/hysop/backend/device/opencl/operator/directional/advection_dir.py
index 2235cccb7..1e10d22cb 100644
--- a/hysop/backend/device/opencl/operator/directional/advection_dir.py
+++ b/hysop/backend/device/opencl/operator/directional/advection_dir.py
@@ -159,7 +159,7 @@ class OpenClDirectionalAdvection(DirectionalAdvectionBase, OpenClDirectionalOper
return kl
@op_apply
- def apply(self, **kargs):
+ def apply(self, dbg=None, **kargs):
queue = self.cl_env.default_queue
dt = self.precision(self.dt() * self.dt_coeff)
diff --git a/hysop/core/graph/graph.py b/hysop/core/graph/graph.py
index 3c138f8d0..b0c680a60 100644
--- a/hysop/core/graph/graph.py
+++ b/hysop/core/graph/graph.py
@@ -126,5 +126,17 @@ def op_apply(f):
@profile
@ready
def apply(*args, **kwds):
- return f(*args, **kwds)
+ dbg = ('dbg' in kwds)
+ dbg &= (kwds['dbg'] is not None)
+ dbg &= (kwds['dbg'].enable_on_op_apply)
+ if dbg:
+ import inspect
+ msg=inspect.getsourcefile(f)
+ kwds['dbg']('pre '+msg, nostack=True)
+ ret = f(*args, **kwds)
+ kwds['dbg']('post '+msg, nostack=True)
+ return ret
+ else:
+ return f(*args, **kwds)
+ return ret
return apply
diff --git a/hysop/operator/base/custom_symbolic_operator.py b/hysop/operator/base/custom_symbolic_operator.py
index 981af0e8b..7cbbb2988 100644
--- a/hysop/operator/base/custom_symbolic_operator.py
+++ b/hysop/operator/base/custom_symbolic_operator.py
@@ -1229,12 +1229,6 @@ class CustomSymbolicOperatorBase(DirectionalOperatorBase):
input_dfields=self.input_discrete_fields,
output_dfields=self.output_discrete_fields)
- @debug
- def get_work_properties(self):
- """Extract work properties from discrete symbolic expressions."""
- requests = super(CustomSymbolicOperatorBase, self).get_work_properties()
- return requests
-
@debug
def setup(self, work):
"""Setup required work."""
diff --git a/hysop/operator/base/redistribute_operator.py b/hysop/operator/base/redistribute_operator.py
index 857118a64..fdeb3908a 100644
--- a/hysop/operator/base/redistribute_operator.py
+++ b/hysop/operator/base/redistribute_operator.py
@@ -93,13 +93,6 @@ class RedistributeOperatorBase(ComputationalGraphOperator):
self._vsource = {v: self.input_discrete_fields[v]}
self._vtarget = {v: self.output_discrete_fields[v]}
- @debug
- def get_work_properties(self):
- return None
- @debug
- def setup(self, work=None):
- super(RedistributeOperatorBase,self).setup(work=work)
-
@classmethod
def supports_multiple_field_topologies(cls):
return True
diff --git a/hysop/tools/debug_utils.py b/hysop/tools/debug_utils.py
new file mode 100644
index 000000000..6d32d8f06
--- /dev/null
+++ b/hysop/tools/debug_utils.py
@@ -0,0 +1,132 @@
+import traceback, inspect
+import numpy as np
+import matplotlib
+matplotlib.use('gtk3cairo')
+
+import matplotlib.pyplot as plt
+plt.ion()
+
+from hysop.fields.discrete_field import DiscreteFieldView
+from hysop.tools.types import check_instance, first_not_None
+
+class ImshowDebugger(object):
+ def __init__(self, data, ntimes=2, cmap='coolwarm',
+ enable_on_op_apply=False, **kwds):
+ check_instance(data, dict, keys=str)
+ ndata = len(data)
+ assert ndata >= 1, ndata
+ assert ntimes >= 1, ntimes
+
+ fig,_axes = plt.subplots(ndata, ntimes, **kwds)
+ _axes = np.asarray(_axes).reshape(ndata, ntimes)
+
+ axes = {}
+ for (i,k) in enumerate(data.keys()):
+ axes[k] = _axes[i]
+
+ imgs = {}
+ for (k,v) in data.iteritems():
+ v = self.get_data(v)
+ for j in xrange(ntimes):
+ axes[k][j].set_title('{} at t=Tn-{}'.format(k,j))
+ axes[k][j].set_xlim(0, v.shape[1]-1)
+ axes[k][j].set_ylim(0, v.shape[0]-1)
+ img = axes[k][j].imshow(self.normalize(v), cmap=cmap,
+ vmin=0.0, vmax=1.0, interpolation='bilinear')
+ imgs.setdefault(k,[]).append(img)
+
+ fig.canvas.mpl_connect('key_press_event', self.on_key_press)
+ fig.suptitle('HySoP Visual Debugger v1.0')
+
+ self.fig = fig
+ self.axes = axes
+ self.data = data
+ self.imgs = imgs
+ self.ntimes = ntimes
+ self.enable_on_op_apply = enable_on_op_apply
+
+ self.cl_queues = []
+ self.running = True
+
+ mng = plt.get_current_fig_manager()
+ mng.window.maximize()
+
+ plt.draw()
+ fig.show()
+
+ def get_data(self, data):
+ if isinstance(data, tuple):
+ (field, component) = data
+ check_instance(field, DiscreteFieldView)
+ check_instance(component, int)
+ data = field.data[component].get()[field.compute_slices].handle
+ check_instance(data, np.ndarray)
+ return data
+
+ def normalize(self, data):
+ check_instance(data, np.ndarray)
+ assert data.ndim == 2
+ dmin, dmax = np.min(data), np.max(data)
+ data = data.astype(np.float32)
+ if (dmax-dmin)<1e-4:
+ data = np.clip(data, 0.0, 1.0)
+ else:
+ data = (data - dmin)/(dmax - dmin)
+ return data
+
+ def update(self):
+ for queue in self.cl_queues:
+ queue.flush()
+ queue.finish()
+ imgs = self.imgs
+ ntimes = self.ntimes
+ for (k,data) in self.data.iteritems():
+ data = self.get_data(data)
+ for j in xrange(ntimes-1,0,-1):
+ imgs[k][j].set_data(imgs[k][j-1].get_array())
+ imgs[k][0].set_array(self.normalize(data))
+ plt.draw()
+
+ def _break(self, msg=None, nostack=False):
+ if not self.running:
+ return
+ msg=first_not_None(msg, '')
+ if not nostack:
+ _file,_line = inspect.stack()[2][1:3]
+ msg='{}::{} {}'.format(_file, _line, msg)
+ print msg
+ self.fig.suptitle('Hysop Visual Debugger v1.0\n{}'.format(msg))
+ self.update()
+ self.blocking = True
+ while self.blocking:
+ plt.pause(0.01)
+
+ def __call__(self, msg=None, nostack=False):
+ self._break(msg=msg, nostack=nostack)
+
+ def on_key_press(self, event):
+ key = event.key
+ if (key=='n'):
+ self.blocking = False
+ elif key == 'q':
+ plt.close(self.fig)
+ self.blocking = False
+ self.running = False
+
+ def synchronize_queue(self, queue):
+ self.cl_queues.append(queue)
+
+if __name__ == '__main__':
+ import random
+ shape0 = (100,100)
+ shape1 = (200,100)
+ img0 = np.zeros(shape=shape0, dtype=np.float32)
+ img1 = np.ones(shape=shape1, dtype=np.float32)
+ dbg = ImshowDebugger({'F0':img0, 'F1':img1}, ntimes=3)
+ dbg('init')
+ i=0
+ while dbg.running:
+ img0[...] = np.random.rand(*shape0).astype(np.float32)
+ img1[...] = np.random.rand(*shape1).astype(np.float32)
+ dbg('msg{}'.format(str(i)))
+ i+=1
diff --git a/hysop/tools/io_utils.py b/hysop/tools/io_utils.py
index 549e22b1e..ce02db2fe 100755
--- a/hysop/tools/io_utils.py
+++ b/hysop/tools/io_utils.py
@@ -10,7 +10,6 @@
"""
import os, h5py, psutil, warnings, tempfile
import subprocess32 as subprocess
-import scitools.filetable as ft
from collections import namedtuple
from inspect import getouterframes, currentframe
from re import findall
@@ -335,12 +334,14 @@ class Writer(object):
def _fullwrite(self):
"""open, write and close"""
+ import scitools.filetable as ft
self._file = open(self.io_params.filename, 'a')
ft.write(self._file, self.buffer)
self._file.close()
def _partialwrite(self):
"""just write, no open, nor close"""
+ import scitools.filetable as ft
ft.write(self._file, self.buffer)
def finalize(self):
--
GitLab