From 0f7c6a745770ae96a93406630210d85d54fd6718 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Keck <jean-baptiste.keck@imag.fr>
Date: Sun, 18 Mar 2018 15:07:05 +0100
Subject: [PATCH] better field requirements reports
---
.../device/opencl/opencl_array_backend.py | 18 ++++
hysop/backend/host/host_array_backend.py | 14 +++
hysop/core/arrays/array_backend.py | 4 +
hysop/core/graph/computational_graph.py | 90 ++++++++++++++-----
hysop/core/memory/memory_request.py | 2 +-
hysop/domain/box.py | 4 +-
hysop/fields/field_requirements.py | 11 +--
hysop/tools/string_utils.py | 3 +-
hysop/topology/cartesian_topology.py | 4 +-
hysop/topology/topology.py | 6 +-
10 files changed, 119 insertions(+), 37 deletions(-)
diff --git a/hysop/backend/device/opencl/opencl_array_backend.py b/hysop/backend/device/opencl/opencl_array_backend.py
index 6c7bbc479..c2f74efb0 100644
--- a/hysop/backend/device/opencl/opencl_array_backend.py
+++ b/hysop/backend/device/opencl/opencl_array_backend.py
@@ -301,6 +301,24 @@ class OpenClArrayBackend(ArrayBackend):
return not (self == other)
def __hash__(self):
return id(self._context) ^ id(self._default_queue) ^ id(self.allocator) ^ id(self.host_array_backend)
+
+
+ __backends = {}
+
+ @classmethod
+ def get_or_create(cls, cl_env, queue, allocator,
+ host_array_backend=None):
+ from hysop.core.arrays.all import HostArrayBackend, \
+ default_host_array_backend
+ host_array_backend = first_not_None(host_array_backend, default_host_array_backend)
+ key = (cl_env, queue, allocator, host_array_backend)
+ if key in cls.__backends:
+ return cls.__backends[key]
+ else:
+ obj = cls(cl_env=cl_env, queue=queue, allocator=allocator,
+ host_array_backend=host_array_backend)
+ cls.__backends[key] = obj
+ return obj
def __init__(self, cl_env=None, queue=None, allocator=None,
host_array_backend=None):
diff --git a/hysop/backend/host/host_array_backend.py b/hysop/backend/host/host_array_backend.py
index 98f349a0f..b15be2594 100644
--- a/hysop/backend/host/host_array_backend.py
+++ b/hysop/backend/host/host_array_backend.py
@@ -66,6 +66,20 @@ class HostArrayBackend(ArrayBackend):
Host array backend.
"""
+ __backends = {}
+
+ @classmethod
+ def get_or_create(cls, allocator):
+ from hysop.backend.host.host_allocator import default_host_allocator
+ allocator = first_not_None(allocator, default_host_allocator)
+ key = (allocator,)
+ if key in cls.__backends:
+ return cls.__backends[key]
+ else:
+ obj = cls(allocator=allocator)
+ cls.__backends[key] = obj
+ return obj
+
def __init__(self, allocator, **kwds):
check_instance(allocator, AllocatorBase)
assert allocator.is_on_host(), 'allocator does not allocate buffers on host.'
diff --git a/hysop/core/arrays/array_backend.py b/hysop/core/arrays/array_backend.py
index 8f7cc8234..72c7491d9 100644
--- a/hysop/core/arrays/array_backend.py
+++ b/hysop/core/arrays/array_backend.py
@@ -83,6 +83,10 @@ class ArrayBackend(TaggedObject):
"""
If set to true, prints all wrapped calls and arguments conversion.
"""
+
+ @classmethod
+ def get_or_create(cls, **kwds):
+ cls._not_implemented_yet('get_or_create')
@staticmethod
def get_alignment_and_size(shape, dtype, min_alignment=None):
diff --git a/hysop/core/graph/computational_graph.py b/hysop/core/graph/computational_graph.py
index c47877b82..9f33a49e8 100644
--- a/hysop/core/graph/computational_graph.py
+++ b/hysop/core/graph/computational_graph.py
@@ -65,47 +65,91 @@ class ComputationalGraph(ComputationalGraphNode):
self._profiler += node._profiler
def field_requirements_report(self, requirements):
- inputs = {}
- sinputs = ''
+ inputs, outputs = {}, {}
+ sinputs, soutputs = {}, {}
def sorted_reqs(reqs):
return sorted(reqs, key=lambda x: \
'{}::{}'.format(x.operator.name, x.field.name))
for field, mreqs in requirements.input_field_requirements.iteritems():
for td, reqs in mreqs.requirements.iteritems():
for req in reqs:
- if self.__FORCE_REPORTS__ or __DEBUG__ or (__VERBOSE__ and not req.is_default()):
- inputs.setdefault(td, {}).setdefault(field, []).append(req)
+ inputs.setdefault(td, {}).setdefault(field, []).append(req)
for td, td_reqs in inputs.iteritems():
- sinputs += '\n {}'.format(td)
+ sin = sinputs.setdefault(td, [])
for field, reqs in td_reqs.iteritems():
for req in sorted_reqs(reqs):
- sinputs += '\n *{}'.format(req)
-
- outputs = {}
- soutputs = ''
+ opname = getattr(req.operator, 'name', 'UnknownOperator')
+ fname = getattr(req.field, 'name', 'UnknownField')
+ min_ghosts=req.ghost_str(req.min_ghosts)
+ max_ghosts=req.ghost_str(req.max_ghosts+1)
+ ghosts = '{}<=ghosts<{}'.format(min_ghosts, max_ghosts)
+ can_split=req.can_split.view(npw.int8)
+ basis='{}'.format(','.join(str(basis)[:3] for basis in req.basis)) \
+ if req.basis else 'ANY'
+ tstates='{}'.format(','.join(str(ts) for ts in req.tstates)) \
+ if req.tstates else 'ANY'
+ sin.append( (opname, fname, ghosts, basis, tstates) )
for field, mreqs in requirements.output_field_requirements.iteritems():
for td, reqs in mreqs.requirements.iteritems():
for req in reqs:
- if self.__FORCE_REPORTS__ or __DEBUG__ or (__VERBOSE__ and not req.is_default()):
- outputs.setdefault(td, {}).setdefault(field, []).append(req)
+ outputs.setdefault(td, {}).setdefault(field, []).append(req)
for td, td_reqs in outputs.iteritems():
- soutputs += '\n {}'.format(td)
+ sin = soutputs.setdefault(td, [])
for field, reqs in td_reqs.iteritems():
for req in sorted_reqs(reqs):
- soutputs += '\n *{}'.format(req)
-
- ss = '== ComputationalGraph {} field requirements report =='.format(self.name)
+ opname = getattr(req.operator, 'name', 'UnknownOperator')
+ fname = getattr(req.field, 'name', 'UnknownField')
+ min_ghosts=req.ghost_str(req.min_ghosts)
+ max_ghosts=req.ghost_str(req.max_ghosts+1)
+ ghosts = '{}<=ghosts<{}'.format(min_ghosts, max_ghosts)
+ can_split=req.can_split.view(npw.int8)
+ basis='{}'.format(','.join(str(basis)[:3] for basis in req.basis)) \
+ if req.basis else 'ANY'
+ tstates='{}'.format(','.join(str(ts) for ts in req.tstates)) \
+ if req.tstates else 'ANY'
+ sin.append( (opname, fname, ghosts, basis, tstates) )
+
+ titles = [[('OPERATOR', 'FIELD', 'GHOSTS', 'BASIS', 'TSTATES')]]
+ name_size = max(len(s[0]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ field_size = max(len(s[1]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ ghosts_size = max(len(s[2].decode('utf-8').replace(u'\u221e', u' ')) \
+ for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ basis_size = max(len(s[3]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ tstates_size = max(len(s[4]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+
+ template = '\n {:<{name_size}} {:^{field_size}} {:^{ghosts_size}} {:^{basis_size}} {:^{tstates_size}}'
+
+ ss= '>INPUTS:'
if sinputs:
- ss+= '\n>INPUTS:{}'.format(sinputs)
+ for (td, sreqs) in sinputs.iteritems():
+ ss+='\n {}'.format(td)
+ ss+= template.format(*titles[0][0],
+ name_size=name_size, field_size=field_size, ghosts_size=ghosts_size,
+ basis_size=basis_size, tstates_size=tstates_size)
+ for (opname, fname, ghosts, basis, tstates) in sreqs:
+ ss+=template.format(
+ opname, fname, ghosts, basis, tstates,
+ name_size=name_size, field_size=field_size, ghosts_size=ghosts_size,
+ basis_size=basis_size, tstates_size=tstates_size)
else:
- ss+= '\n>INPUTS: None'
+ ss+=' None'
+ ss+= '\n>OUTPUTS:'
if soutputs:
- ss+= '\n>OUTPUTS:{}'.format(soutputs)
+ for (td, sreqs) in soutputs.iteritems():
+ ss+='\n {}'.format(td)
+ ss+= template.format(*titles[0][0],
+ name_size=name_size, field_size=field_size, ghosts_size=ghosts_size,
+ basis_size=basis_size, tstates_size=tstates_size)
+ for (opname, fname, ghosts, basis, tstates) in sreqs:
+ ss+=template.format(
+ opname, fname, ghosts, basis, tstates,
+ name_size=name_size, field_size=field_size, ghosts_size=ghosts_size,
+ basis_size=basis_size, tstates_size=tstates_size)
else:
- ss+= '\n>OUTPUTS: None'
- l = len('== ComputationalGraph {} field requirements report =='.format(self.name))
- ss += '\n'+l*'='+'\n'
- return ss
+ ss+=' None'
+
+ title = ' ComputationalGraph {} field requirements report '.format(self.name)
+ return '\n{}\n'.format(framed_str(title=title, msg=ss))
def domain_report(self):
domains = self.get_domains()
@@ -133,7 +177,7 @@ class ComputationalGraph(ComputationalGraphNode):
ops.setdefault(domain, []).append( (op.name, inputs, outputs, type(op).__name__) )
if (None in domains):
- opeators = domains[None]
+ operators = domains[None]
for op in sorted(operators, key=lambda x: x.name):
pinputs = ','.join( sorted([p for p in op.input_params]))
poutputs =','.join( sorted([p for p in op.output_params]))
diff --git a/hysop/core/memory/memory_request.py b/hysop/core/memory/memory_request.py
index ae40fe091..067711285 100644
--- a/hysop/core/memory/memory_request.py
+++ b/hysop/core/memory/memory_request.py
@@ -354,7 +354,7 @@ class MultipleOperatorMemoryRequests(object):
def __str__(self):
s=''
- for backend, backend_requests in self._all_requests_per_backend.iteritems():
+ for (backend, backend_requests) in self._all_requests_per_backend.iteritems():
kind = backend.kind
if kind == Backend.OPENCL:
precision = ' on device {}'.format(backend.device.name)
diff --git a/hysop/domain/box.py b/hysop/domain/box.py
index 83a1359b9..d6b97d937 100644
--- a/hysop/domain/box.py
+++ b/hysop/domain/box.py
@@ -76,8 +76,8 @@ class BoxView(DomainView):
"""
Return a short description of this Box as a string.
"""
- return 'Box(tag={}, O=[{}], L=[{}], BC=[{}], current_task={})'.format(
- self.tag,
+ return '{} (O=[{}], L=[{}], BC=[{}], current_task={})'.format(
+ self.full_tag,
','.join(('{:1.1f}'.format(val) for val in self.origin)),
','.join(('{:1.1f}'.format(val) for val in self.length)),
','.join(('{}/{}'.format(str(lb)[:3],str(rb)[:3]) for (lb,rb) in \
diff --git a/hysop/fields/field_requirements.py b/hysop/fields/field_requirements.py
index 07dd797dc..2c1214bdd 100644
--- a/hysop/fields/field_requirements.py
+++ b/hysop/fields/field_requirements.py
@@ -107,17 +107,18 @@ class DiscreteFieldRequirements(object):
return id(self.operator) ^ id(self.variables) ^ id(self.field) ^ \
hash((to_tuple(self.min_ghosts), to_tuple(self.max_ghosts),
self.basis, self.tstates))
+
+ def ghost_str(self, array):
+ inf = u'+\u221e'
+ vals = [u''+str(x) if np.isfinite(x) else inf for x in array]
+ return u'[{}]'.format(u','.join(vals)).encode('utf-8').strip()
def __str__(self):
- def arraystr(array):
- inf = u'+\u221e'
- vals = [u''+str(x) if np.isfinite(x) else inf for x in array]
- return u'[{}]'.format(u','.join(vals)).encode('utf-8').strip()
return '{:15s} {:>10s}<=ghosts<{:<10s} can_split={} basis={} tstates={}'.format(
'{}::{}'.format(getattr(self.operator, 'name', 'UnknownOperator'),
getattr(self.field, 'name', 'UnknownField')),
- arraystr(self.min_ghosts), arraystr(self.max_ghosts+1),
+ self.ghost_str(self.min_ghosts), self.ghost_str(self.max_ghosts+1),
self.can_split.view(np.int8),
'[{}]'.format(','.join(str(basis)[:3] for basis in self.basis)) \
if self.basis else 'ANY',
diff --git a/hysop/tools/string_utils.py b/hysop/tools/string_utils.py
index 6aa9c2d97..e9c1214ae 100644
--- a/hysop/tools/string_utils.py
+++ b/hysop/tools/string_utils.py
@@ -33,7 +33,8 @@ def framed_str(title, msg, c='=', at_border=2):
Format a message to fit between two separation lines
containing a title.
"""
- length = max(len(m) for m in msg.split('\n'))
+ clean = lambda s: re.sub(r'[^\x00-\x7f]','',s)
+ length = max(len(clean(m)) for m in msg.split('\n'))
title = c*at_border + title + c*at_border
header = title + c*max(0, length-len(title))
footer = c*len(header)
diff --git a/hysop/topology/cartesian_topology.py b/hysop/topology/cartesian_topology.py
index e276861dc..f18248f40 100644
--- a/hysop/topology/cartesian_topology.py
+++ b/hysop/topology/cartesian_topology.py
@@ -363,7 +363,9 @@ class CartesianTopologyView(TopologyView):
s='CartesianTopology[tag={}, domain={}, task_id={}, pcoords={}, pshape={}, '
s+='distr.={}, periods={}, shape={}, ghosts={}]'
s = s.format(
- self.tag, self.domain.domain.full_tag, self.task_id,
+ self.tag,
+ self.domain.domain.full_tag,
+ self.task_id,
self.proc_coords, self.proc_shape,
'[{}]'.format(','.join('T' if per else 'F' for per in self.is_distributed)),
'[{}]'.format(','.join('T' if per else 'F' for per in self.is_periodic)),
diff --git a/hysop/topology/topology.py b/hysop/topology/topology.py
index c69ebe237..9f34cacd7 100644
--- a/hysop/topology/topology.py
+++ b/hysop/topology/topology.py
@@ -356,15 +356,13 @@ class Topology(RegisteredObject):
assert (cl_env is None)
assert (queue is None)
from hysop.core.arrays.all import HostArrayBackend
- from hysop.backend.host.host_allocator import default_host_allocator
- allocator = allocator or default_host_allocator
- backend = HostArrayBackend(allocator)
+ backend = HostArrayBackend.get_or_create(allocator)
elif (backend == Backend.OPENCL):
from hysop.backend.device.opencl.opencl_tools import get_or_create_opencl_env
from hysop.core.arrays.all import OpenClArrayBackend
cl_env = cl_env or get_or_create_opencl_env(mpi_params)
assert cl_env.mpi_params == mpi_params
- backend = OpenClArrayBackend(cl_env=cl_env, queue=queue, allocator=allocator)
+ backend = OpenClArrayBackend.get_or_create(cl_env=cl_env, queue=queue, allocator=allocator)
assert backend.cl_env.mpi_params == mpi_params
else:
msg = 'Unsupported backend {}.'.format(backend)
--
GitLab