diff --git a/hysop/core/graph/computational_graph.py b/hysop/core/graph/computational_graph.py
index 759176cee60b8ad6143b5aacbcc684d4f1a0d740..233f4e307c9dc889b6639c7a662f5480891bfd45 100644
--- a/hysop/core/graph/computational_graph.py
+++ b/hysop/core/graph/computational_graph.py
@@ -1,14 +1,14 @@
# coding: utf-8
from hysop import __DEBUG__, __VERBOSE__, vprint, dprint
-from hysop.tools.decorators import debug
+from hysop.tools.decorators import debug
from hysop.tools.types import to_list, to_set, to_tuple, first_not_None, check_instance
from hysop.tools.string_utils import framed_str, strlen, multiline_split
from hysop.tools.numpywrappers import npw
from hysop.core.graph.graph import not_implemented, initialized, discretized, \
- ready, graph_built, not_initialized
+ ready, graph_built, not_initialized
from hysop.core.graph.graph import Graph, ComputationalGraphNodeData, gt
-from hysop.core.graph.computational_node import ComputationalGraphNode
+from hysop.core.graph.computational_node import ComputationalGraphNode
from hysop.core.graph.computational_operator import ComputationalGraphOperator
from hysop.core.graph.node_generator import ComputationalGraphNodeGenerator
from hysop.core.graph.node_requirements import NodeRequirements, OperatorRequirements
@@ -31,8 +31,8 @@ class ComputationalGraph(ComputationalGraphNode):
@debug
def __init__(self, candidate_input_tensors=None,
- candidate_output_tensors=None,
- **kwds):
+ candidate_output_tensors=None,
+ **kwds):
"""
Parameters
----------
@@ -50,22 +50,22 @@ class ComputationalGraph(ComputationalGraphNode):
"""
if ('input_fields' in kwds.keys()) or ('output_fields' in kwds.keys()):
- msg='input_fields or output_fields parameters should not be used in {}, they are \
+ msg = 'input_fields or output_fields parameters should not be used in {}, they are \
deduced during graph construction (building step).'.format(cls)
raise ValueError(msg)
if ('input_params' in kwds.keys()) or ('output_params' in kwds.keys()):
- msg='input_params or output_params parameters should not be used in {}, they are \
+ msg = 'input_params or output_params parameters should not be used in {}, they are \
deduced during graph construction (building step).'.format(cls)
raise ValueError(msg)
- super(ComputationalGraph,self).__init__(input_fields=None, output_fields=None,
- **kwds)
+ super(ComputationalGraph, self).__init__(input_fields=None, output_fields=None,
+ **kwds)
self.nodes = []
self.graph = None
self.graph_built = False
self.graph_is_rendering = False
- self.candidate_input_tensors = set(first_not_None(candidate_input_tensors, ()))
+ self.candidate_input_tensors = set(first_not_None(candidate_input_tensors, ()))
self.candidate_output_tensors = set(first_not_None(candidate_output_tensors, ()))
@graph_built
@@ -73,20 +73,19 @@ class ComputationalGraph(ComputationalGraphNode):
"""Collect profiling informations of profiled attributes."""
for node in self.nodes:
self._profiler += node._profiler
-
-
+
def node_requirements_report(self, requirements):
- values = [(u'OPERATOR', u'TOPOLOGY', u'TSTATE', u'GHOSTS', u'MEMORY ORDER',
- u'NODE.MRO[0]', u'NODE.MRO[1]', u'NODE.MRO[2]')]
+ values = [(u'OPERATOR', u'TOPOLOGY', u'TSTATE', u'GHOSTS', u'MEMORY ORDER',
+ u'NODE.MRO[0]', u'NODE.MRO[1]', u'NODE.MRO[2]')]
for node in self.nodes:
reqs = node.get_node_requirements()
if not isinstance(reqs, OperatorRequirements):
continue
- opname = node.pretty_name.decode('utf-8')
- optypes = type(node).__mro__
+ opname = node.pretty_name.decode('utf-8')
+ optypes = type(node).__mro__
n = len(optypes)
- optypes = tuple(_.__name__ for _ in optypes[:min(3,n)]) + (u'',)*(3-n)
- vals = (opname,
+ optypes = tuple(_.__name__ for _ in optypes[:min(3, n)]) + (u'',)*(3-n)
+ vals = (opname,
reqs.enforce_unique_topology_shape, reqs.enforce_unique_transposition_state,
reqs.enforce_unique_ghosts, reqs.enforce_unique_memory_order,
) + optypes
@@ -94,45 +93,45 @@ class ComputationalGraph(ComputationalGraphNode):
values.append(vals)
template = u'\n {:<{name_size}} {:^{topology_size}} {:^{tstates_size}} {:^{ghosts_size}} {:^{order_size}} {:<{type_size0}} {:<{type_size1}} {:<{type_size2}}'
- name_size = max(strlen(s[0]) for s in values)
+ name_size = max(strlen(s[0]) for s in values)
topology_size = max(strlen(s[1]) for s in values)
- tstates_size = max(strlen(s[2]) for s in values)
- ghosts_size = max(strlen(s[3]) for s in values)
- order_size = max(strlen(s[4]) for s in values)
- type_size0 = max(strlen(s[5]) for s in values)
- type_size1 = max(strlen(s[6]) for s in values)
- type_size2 = max(strlen(s[7]) for s in values)
-
- ss=u''
+ tstates_size = max(strlen(s[2]) for s in values)
+ ghosts_size = max(strlen(s[3]) for s in values)
+ order_size = max(strlen(s[4]) for s in values)
+ type_size0 = max(strlen(s[5]) for s in values)
+ type_size1 = max(strlen(s[6]) for s in values)
+ type_size2 = max(strlen(s[7]) for s in values)
+
+ ss = u''
for (opname, enforce_unique_topology_shape, enforce_unique_transposition_state,
enforce_unique_ghosts, enforce_unique_memory_order, optype0, optype1, optype2) in values:
- ss+=template.format(
- opname,
- enforce_unique_topology_shape,
- enforce_unique_transposition_state,
- enforce_unique_ghosts,
- enforce_unique_memory_order,
- optype0, optype1, optype2,
- name_size=name_size,
- topology_size=topology_size,
- tstates_size=tstates_size,
- ghosts_size=ghosts_size,
- order_size=order_size,
- type_size0=type_size0,
- type_size1=type_size1,
- type_size2=type_size2)
-
+ ss += template.format(
+ opname,
+ enforce_unique_topology_shape,
+ enforce_unique_transposition_state,
+ enforce_unique_ghosts,
+ enforce_unique_memory_order,
+ optype0, optype1, optype2,
+ name_size=name_size,
+ topology_size=topology_size,
+ tstates_size=tstates_size,
+ ghosts_size=ghosts_size,
+ order_size=order_size,
+ type_size0=type_size0,
+ type_size1=type_size1,
+ type_size2=type_size2)
+
title = u'ComputationalGraph {} node requirements report '.format(
self.pretty_name.decode('utf-8'))
return u'\n{}\n'.format(framed_str(title=title, msg=ss[1:])).encode('utf-8')
-
def field_requirements_report(self, requirements):
inputs, outputs = {}, {}
sinputs, soutputs = {}, {}
+
def sorted_reqs(reqs):
- return sorted(reqs, key=lambda x: \
- '{}::{}'.format(x.field.name, x.operator.name))
+ return sorted(reqs, key=lambda x:
+ '{}::{}'.format(x.field.name, x.operator.name))
for field, mreqs in requirements.input_field_requirements.iteritems():
for td, reqs in mreqs.requirements.iteritems():
for req in reqs:
@@ -142,17 +141,18 @@ class ComputationalGraph(ComputationalGraphNode):
for field, reqs in td_reqs.iteritems():
for req in sorted_reqs(reqs):
opname = getattr(req.operator, 'pretty_name', 'UnknownOperator').decode('utf-8')
- fname = getattr(req.field, 'pretty_name', 'UnknownField').decode('utf-8')
- min_ghosts=req.ghost_str(req.min_ghosts)
- max_ghosts=req.ghost_str(req.max_ghosts+1)
- discr = str(req.operator.input_fields[field].grid_resolution)
+ fname = getattr(req.field, 'pretty_name', 'UnknownField').decode('utf-8')
+ min_ghosts = req.ghost_str(req.min_ghosts)
+ max_ghosts = req.ghost_str(req.max_ghosts+1)
+ discr = str(req.operator.input_fields[field].grid_resolution)
ghosts = u'{}<=ghosts<{}'.format(min_ghosts, max_ghosts)
- can_split=req.can_split.view(npw.int8)
- memory_order=u'{}'.format(req.memory_order) if req.memory_order else u'ANY'
- can_split=u'[{}]'.format(u','.join('1' if cs else '0' for cs in req.can_split))
- tstates=u'{}'.format(u','.join(str(ts) for ts in req.tstates)) \
- if req.tstates else 'ANY'
- sin.append( (opname, fname, discr, ghosts, memory_order, can_split, tstates) )
+ can_split = req.can_split.view(npw.int8)
+ memory_order = u'{}'.format(req.memory_order) if req.memory_order else u'ANY'
+ can_split = u'[{}]'.format(
+ u','.join('1' if cs else '0' for cs in req.can_split))
+ tstates = u'{}'.format(u','.join(str(ts) for ts in req.tstates)) \
+ if req.tstates else 'ANY'
+ sin.append((opname, fname, discr, ghosts, memory_order, can_split, tstates))
for field, mreqs in requirements.output_field_requirements.iteritems():
for td, reqs in mreqs.requirements.iteritems():
for req in reqs:
@@ -162,71 +162,75 @@ class ComputationalGraph(ComputationalGraphNode):
for field, reqs in td_reqs.iteritems():
for req in sorted_reqs(reqs):
opname = getattr(req.operator, 'pretty_name', 'UnknownOperator').decode('utf-8')
- fname = getattr(req.field, 'pretty_name', 'UnknownField').decode('utf-8')
- min_ghosts=req.ghost_str(req.min_ghosts)
- max_ghosts=req.ghost_str(req.max_ghosts+1)
- discr = str(req.operator.output_fields[field].grid_resolution)
+ fname = getattr(req.field, 'pretty_name', 'UnknownField').decode('utf-8')
+ min_ghosts = req.ghost_str(req.min_ghosts)
+ max_ghosts = req.ghost_str(req.max_ghosts+1)
+ discr = str(req.operator.output_fields[field].grid_resolution)
ghosts = u'{}<=ghosts<{}'.format(min_ghosts, max_ghosts)
- can_split=req.can_split.view(npw.int8)
- memory_order=u'{}'.format(req.memory_order) if req.memory_order else u'ANY'
- can_split=u'[{}]'.format(u','.join('1' if cs else '0' for cs in req.can_split))
- tstates=u'{}'.format(u','.join(str(ts) for ts in req.tstates)) \
- if req.tstates else u'ANY'
- sout.append( (opname, fname, discr, ghosts, memory_order, can_split, tstates) )
-
- titles = [[(u'OPERATOR', u'FIELD', u'DISCRETIZATION', u'GHOSTS', u'MEMORY ORDER', u'CAN_SPLIT', u'TSTATES')]]
- name_size = max(len(s[0]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
- field_size = max(len(s[1]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
- discr_size = max(len(s[2]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
- ghosts_size = max(len(s[3]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
- order_size = max(len(s[4]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
- cansplit_size = max(len(s[5]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
- tstates_size = max(len(s[6]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ can_split = req.can_split.view(npw.int8)
+ memory_order = u'{}'.format(req.memory_order) if req.memory_order else u'ANY'
+ can_split = u'[{}]'.format(
+ u','.join('1' if cs else '0' for cs in req.can_split))
+ tstates = u'{}'.format(u','.join(str(ts) for ts in req.tstates)) \
+ if req.tstates else u'ANY'
+ sout.append((opname, fname, discr, ghosts, memory_order, can_split, tstates))
+
+ titles = [[(u'OPERATOR', u'FIELD', u'DISCRETIZATION', u'GHOSTS',
+ u'MEMORY ORDER', u'CAN_SPLIT', u'TSTATES')]]
+ name_size = max(len(s[0]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ field_size = max(len(s[1]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ discr_size = max(len(s[2]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ ghosts_size = max(len(s[3]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ order_size = max(len(s[4]) for ss in sinputs.values()+soutputs.values()+titles for s in ss)
+ cansplit_size = max(len(s[5]) for ss in sinputs.values() +
+ soutputs.values()+titles for s in ss)
+ tstates_size = max(len(s[6]) for ss in sinputs.values() +
+ soutputs.values()+titles for s in ss)
template = u'\n {:<{name_size}} {:^{field_size}} {:^{discr_size}} {:^{ghosts_size}} {:^{order_size}} {:^{cansplit_size}} {:^{tstates_size}}'
- ss= u'>INPUTS:'
+ ss = u'>INPUTS:'
if sinputs:
for (td, sreqs) in sinputs.iteritems():
if isinstance(td, Topology):
- ss+=u'\n {}'.format(td.short_description())
+ ss += u'\n {}'.format(td.short_description())
else:
- ss+=u'\n {}'.format(td)
- ss+= template.format(*titles[0][0],
- name_size=name_size, field_size=field_size,
+ ss += u'\n {}'.format(td)
+ ss += template.format(*titles[0][0],
+ name_size=name_size, field_size=field_size,
+ discr_size=discr_size, ghosts_size=ghosts_size,
+ order_size=order_size, cansplit_size=cansplit_size,
+ tstates_size=tstates_size)
+ for (opname, fname, discr, ghosts, order, can_split, tstates) in sreqs:
+ ss += template.format(
+ opname, fname, discr, ghosts, order, can_split, tstates,
+ name_size=name_size, field_size=field_size,
discr_size=discr_size, ghosts_size=ghosts_size,
order_size=order_size, cansplit_size=cansplit_size,
tstates_size=tstates_size)
- for (opname, fname, discr, ghosts, order, can_split, tstates) in sreqs:
- ss+=template.format(
- opname, fname, discr, ghosts, order, can_split, tstates,
- name_size=name_size, field_size=field_size,
- discr_size=discr_size, ghosts_size=ghosts_size,
- order_size=order_size, cansplit_size=cansplit_size,
- tstates_size=tstates_size)
else:
- ss+=u' None'
- ss+= u'\n>OUTPUTS:'
+ ss += u' None'
+ ss += u'\n>OUTPUTS:'
if soutputs:
for (td, sreqs) in soutputs.iteritems():
if isinstance(td, Topology):
- ss+=u'\n {}'.format(td.short_description())
+ ss += u'\n {}'.format(td.short_description())
else:
- ss+=u'\n {}'.format(td)
- ss+= template.format(*titles[0][0],
- name_size=name_size, field_size=field_size,
+ ss += u'\n {}'.format(td)
+ ss += template.format(*titles[0][0],
+ name_size=name_size, field_size=field_size,
+ discr_size=discr_size, ghosts_size=ghosts_size,
+ order_size=order_size, cansplit_size=cansplit_size,
+ tstates_size=tstates_size)
+ for (opname, fname, discr, ghosts, order, can_split, tstates) in sreqs:
+ ss += template.format(
+ opname, fname, discr, ghosts, order, can_split, tstates,
+ name_size=name_size, field_size=field_size,
discr_size=discr_size, ghosts_size=ghosts_size,
order_size=order_size, cansplit_size=cansplit_size,
tstates_size=tstates_size)
- for (opname, fname, discr, ghosts, order, can_split, tstates) in sreqs:
- ss+=template.format(
- opname, fname, discr, ghosts, order, can_split, tstates,
- name_size=name_size, field_size=field_size,
- discr_size=discr_size, ghosts_size=ghosts_size,
- order_size=order_size, cansplit_size=cansplit_size,
- tstates_size=tstates_size)
else:
- ss+=u' None'
+ ss += u' None'
title = u'ComputationalGraph {} field requirements report '.format(
self.pretty_name.decode('utf-8'))
@@ -234,36 +238,42 @@ class ComputationalGraph(ComputationalGraphNode):
def domain_report(self):
domains = self.get_domains()
- ops={}
+ ops = {}
- maxlen = (None, 40, None, 40, None)
- split_sep = (None, ',', None, ',', None)
+ maxlen = (None, 40, None, 40, None)
+ split_sep = (None, ',', None, ',', None)
newline_prefix = (None, ' ', '', ' ', None)
- replace = ('', '', '-', '', '')
+ replace = ('', '', '-', '', '')
- for (domain,operators) in domains.iteritems():
+ for (domain, operators) in domains.iteritems():
if (domain is None):
continue
for op in sorted(operators, key=lambda x: x.pretty_name):
- finputs = u','.join( sorted([f.pretty_name.decode('utf-8') for f in op.iter_input_fields() if f.domain is domain]))
- foutputs =u','.join( sorted([f.pretty_name.decode('utf-8') for f in op.iter_output_fields() if f.domain is domain]))
- pinputs = u','.join( sorted([p.pretty_name.decode('utf-8') for p in op.input_params.values()]))
- poutputs =u','.join( sorted([p.pretty_name.decode('utf-8') for p in op.output_params.values()]))
- infields = u'[{}]'.format(finputs) if finputs else u''
+ finputs = u','.join(sorted([f.pretty_name.decode('utf-8')
+ for f in op.iter_input_fields() if f.domain is domain]))
+ foutputs = u','.join(sorted([f.pretty_name.decode('utf-8')
+ for f in op.iter_output_fields() if f.domain is domain]))
+ pinputs = u','.join(sorted([p.pretty_name.decode('utf-8')
+ for p in op.input_params.values()]))
+ poutputs = u','.join(sorted([p.pretty_name.decode('utf-8')
+ for p in op.output_params.values()]))
+ infields = u'[{}]'.format(finputs) if finputs else u''
outfields = u'[{}]'.format(foutputs) if foutputs else u''
- inparams = u'[{}]'.format(pinputs) if pinputs else u''
+ inparams = u'[{}]'.format(pinputs) if pinputs else u''
outparams = u'[{}]'.format(poutputs) if poutputs else u''
- inputs = u'{}{}{}'.format(infields, u'x' if infields and inparams else u'', inparams)
- outputs = u'{}{}{}'.format(outfields, u'x' if outfields and outparams else u'', outparams)
+ inputs = u'{}{}{}'.format(
+ infields, u'x' if infields and inparams else u'', inparams)
+ outputs = u'{}{}{}'.format(
+ outfields, u'x' if outfields and outparams else u'', outparams)
if inputs == u'':
- inputs=u'no inputs'
+ inputs = u'no inputs'
if outputs == u'':
- outputs=u'no outputs'
+ outputs = u'no outputs'
- opname = op.pretty_name.decode('utf-8')
- optype = type(op).__name__
+ opname = op.pretty_name.decode('utf-8')
+ optype = type(op).__name__
strdata = (opname, inputs, '->', outputs, optype)
op_data = ops.setdefault(domain, [])
@@ -272,71 +282,72 @@ class ComputationalGraph(ComputationalGraphNode):
if (None in domains):
operators = domains[None]
for op in sorted(operators, key=lambda x: x.pretty_name):
- pinputs = u','.join( sorted([p.pretty_name.decode('utf-8') for p in op.input_params.values()]))
- poutputs =u','.join( sorted([p.pretty_name.decode('utf-8') for p in op.output_params.values()]))
- inparams = u'[{}]'.format(pinputs) if pinputs else ''
+ pinputs = u','.join(sorted([p.pretty_name.decode('utf-8')
+ for p in op.input_params.values()]))
+ poutputs = u','.join(sorted([p.pretty_name.decode('utf-8')
+ for p in op.output_params.values()]))
+ inparams = u'[{}]'.format(pinputs) if pinputs else ''
outparams = u'[{}]'.format(poutputs) if poutputs else ''
- inputs=u'{}'.format(inparams)
- outputs=u'{}'.format(outparams)
+ inputs = u'{}'.format(inparams)
+ outputs = u'{}'.format(outparams)
if inputs == '':
- inputs=u'no inputs'
+ inputs = u'no inputs'
if outputs == '':
- outputs=u'no outputs'
- opname = op.pretty_name.decode('utf-8')
- optype = type(op).__name__
+ outputs = u'no outputs'
+ opname = op.pretty_name.decode('utf-8')
+ optype = type(op).__name__
strdata = (opname, inputs, '->', outputs, optype)
op_data = ops.setdefault(None, [])
op_data += multiline_split(strdata, maxlen, split_sep, replace, newline_prefix)
- name_size = max(strlen(s[0]) for ss in ops.values() for s in ss)
- in_size = max(strlen(s[1]) for ss in ops.values() for s in ss)
+ name_size = max(strlen(s[0]) for ss in ops.values() for s in ss)
+ in_size = max(strlen(s[1]) for ss in ops.values() for s in ss)
arrow_size = max(strlen(s[2]) for ss in ops.values() for s in ss)
- out_size = max(strlen(s[3]) for ss in ops.values() for s in ss)
- type_size = max(strlen(s[4]) for ss in ops.values() for s in ss)
+ out_size = max(strlen(s[3]) for ss in ops.values() for s in ss)
+ type_size = max(strlen(s[4]) for ss in ops.values() for s in ss)
ss = u''
- for (domain,dops) in ops.iteritems():
+ for (domain, dops) in ops.iteritems():
if (domain is None):
continue
ss += u'\n>{}'.format(domain.short_description())
ss += u'\n {:<{name_size}} {:<{in_size}} {:<{arrow_size}} {:<{out_size}} {:<{type_size}}'.format(
- 'OPERATOR', 'INPUTS', '', 'OUTPUTS', 'OPERATOR TYPE',
- name_size=name_size, in_size=in_size,
- arrow_size=arrow_size,
- out_size=out_size, type_size=type_size)
+ 'OPERATOR', 'INPUTS', '', 'OUTPUTS', 'OPERATOR TYPE',
+ name_size=name_size, in_size=in_size,
+ arrow_size=arrow_size,
+ out_size=out_size, type_size=type_size)
for (opname, inputs, arrow, outputs, optype) in dops:
ss += u'\n {:<{name_size}} {:<{in_size}} {:<{arrow_size}} {:<{out_size}} {:<{type_size}}'.format(
- opname, inputs, arrow, outputs, optype,
- name_size=name_size, in_size=in_size,
- arrow_size=arrow_size,
- out_size=out_size, type_size=type_size)
+ opname, inputs, arrow, outputs, optype,
+ name_size=name_size, in_size=in_size,
+ arrow_size=arrow_size,
+ out_size=out_size, type_size=type_size)
if (None in domains):
ss += u'\n>Domainless operators:'
ss += u'\n {:<{name_size}} {:<{in_size}} {:<{arrow_size}} {:<{out_size}} {:<{type_size}}'.format(
- 'OPERATOR', 'INPUTS', '', 'OUTPUTS', 'OPERATOR TYPE',
- name_size=name_size, in_size=in_size,
- arrow_size=arrow_size,
- out_size=out_size, type_size=type_size)
+ 'OPERATOR', 'INPUTS', '', 'OUTPUTS', 'OPERATOR TYPE',
+ name_size=name_size, in_size=in_size,
+ arrow_size=arrow_size,
+ out_size=out_size, type_size=type_size)
for (opname, inputs, arrow, outputs, optype) in ops[None]:
ss += u'\n {:<{name_size}} {:<{in_size}} {:<{arrow_size}} {:<{out_size}} {:<{type_size}}'.format(
- opname, inputs, arrow, outputs, optype,
- name_size=name_size, in_size=in_size,
- arrow_size=arrow_size,
- out_size=out_size, type_size=type_size)
+ opname, inputs, arrow, outputs, optype,
+ name_size=name_size, in_size=in_size,
+ arrow_size=arrow_size,
+ out_size=out_size, type_size=type_size)
- title=u'ComputationalGraph {} domain and operator report '.format(
+ title = u'ComputationalGraph {} domain and operator report '.format(
self.pretty_name.decode('utf-8'))
return u'\n{}\n'.format(framed_str(title=title, msg=ss[1:])).encode('utf-8')
-
def topology_report(self):
- ss=''
- for (backend,topologies) in self.get_topologies().iteritems():
+ ss = ''
+ for (backend, topologies) in self.get_topologies().iteritems():
ss += u'\n {}:'.format(backend.short_description())
ss += u'\n *'+'\n *'.join(t.short_description()
- for t in sorted(topologies, key=lambda x: x.id))
+ for t in sorted(topologies, key=lambda x: x.id))
title = u'ComputationalGraph {} topology report '.format(
self.pretty_name.decode('utf-8'))
return u'\n{}\n'.format(framed_str(title=title, msg=ss[1:]))
@@ -349,9 +360,9 @@ class ComputationalGraph(ComputationalGraphNode):
topologies = {}
for field in self.fields:
field_topologies = {}
- for (i,vid) in enumerate(self.sorted_nodes):
+ for (i, vid) in enumerate(self.sorted_nodes):
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
if field in op.input_fields:
topo = op.input_fields[field]
field_topologies.setdefault(topo, []).append(op)
@@ -366,17 +377,17 @@ class ComputationalGraph(ComputationalGraphNode):
n0 = len(str(topo.backend.kind).lower())
n1 = len(str(topo.tag))
for i in xrange(nentries):
- sops=u', '.join(pnames[nbyline*i:nbyline*(i+1)])
- if (i!=nentries-1) or (len(pnames)%nbyline!=0):
- sops+=','
- if (i==0):
+ sops = u', '.join(pnames[nbyline*i:nbyline*(i+1)])
+ if (i != nentries-1) or (len(pnames) % nbyline != 0):
+ sops += ','
+ if (i == 0):
entries = (str(topo.backend.kind).lower(), topo.tag, sops)
else:
entries = ('', '-'*n1, sops)
topologies.setdefault(field, []).append(entries)
- if (len(pnames)%nbyline != 0):
- sops=u', '.join(pnames[nbyline*nentries:])
- if (nentries==0):
+ if (len(pnames) % nbyline != 0):
+ sops = u', '.join(pnames[nbyline*nentries:])
+ if (nentries == 0):
entries = (str(topo.backend.kind).lower(), topo.tag, sops)
else:
entries = ('', '-'*n1, sops)
@@ -384,7 +395,7 @@ class ComputationalGraph(ComputationalGraphNode):
titles = [[(u'BACKEND', u'TOPOLOGY', u'OPERATORS')]]
backend_size = max(len(s[0]) for ss in topologies.values()+titles for s in ss)
- topo_size = max(len(s[1]) for ss in topologies.values()+titles for s in ss)
+ topo_size = max(len(s[1]) for ss in topologies.values()+titles for s in ss)
template = u'\n {:<{backend_size}} {:<{topo_size}} {}'
sizes = {'backend_size': backend_size,
'topo_size': topo_size}
@@ -403,17 +414,17 @@ class ComputationalGraph(ComputationalGraphNode):
return ss.encode('utf-8')
def operator_report(self):
- maxlen = (None, None, 40, None, 40, None)
- split_sep = (None, None, ',', None, ',', None)
+ maxlen = (None, None, 40, None, 40, None)
+ split_sep = (None, None, ',', None, ',', None)
newline_prefix = (None, None, ' ', '', ' ', None)
- replace = ('--', '', '', '-', '', '')
+ replace = ('--', '', '', '-', '', '')
reduced_graph = self.reduced_graph
operators = reduced_graph.vertex_properties['operators']
ops = []
- for (i,vid) in enumerate(self.sorted_nodes):
+ for (i, vid) in enumerate(self.sorted_nodes):
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
handled_inputs, handled_outputs = (), ()
finputs, foutputs = [], []
@@ -431,60 +442,61 @@ class ComputationalGraph(ComputationalGraphNode):
foutputs.append(u'{}.{}'.format(f.pretty_name.decode('utf-8'),
t0.pretty_tag.decode('utf-8')))
handled_outputs += f.fields
- finputs += [u'{}.{}'.format(f.pretty_name.decode('utf-8'),
- t.pretty_tag.decode('utf-8'))
- for (f,t) in op.input_fields.iteritems()
- if f not in handled_inputs]
- foutputs += [u'{}.{}'.format(f.pretty_name.decode('utf-8'),
- t.pretty_tag.decode('utf-8'))
- for (f,t) in op.output_fields.iteritems()
- if f not in handled_outputs]
+ finputs += [u'{}.{}'.format(f.pretty_name.decode('utf-8'),
+ t.pretty_tag.decode('utf-8'))
+ for (f, t) in op.input_fields.iteritems()
+ if f not in handled_inputs]
+ foutputs += [u'{}.{}'.format(f.pretty_name.decode('utf-8'),
+ t.pretty_tag.decode('utf-8'))
+ for (f, t) in op.output_fields.iteritems()
+ if f not in handled_outputs]
finputs = u','.join(sorted(finputs))
foutputs = u','.join(sorted(foutputs))
- pinputs = u','.join( sorted([p.pretty_name.decode('utf-8')
- for p in op.input_params.values()]))
- poutputs = u','.join( sorted([p.pretty_name.decode('utf-8')
- for p in op.output_params.values()]))
+ pinputs = u','.join(sorted([p.pretty_name.decode('utf-8')
+ for p in op.input_params.values()]))
+ poutputs = u','.join(sorted([p.pretty_name.decode('utf-8')
+ for p in op.output_params.values()]))
- infields = u'[{}]'.format(finputs) if finputs else u''
+ infields = u'[{}]'.format(finputs) if finputs else u''
outfields = u'[{}]'.format(foutputs) if foutputs else u''
- inparams = u'[{}]'.format(pinputs) if pinputs else u''
+ inparams = u'[{}]'.format(pinputs) if pinputs else u''
outparams = u'[{}]'.format(poutputs) if poutputs else u''
- inputs = u'{}{}{}'.format(infields, u'x' if infields and inparams else u'', inparams)
- outputs = u'{}{}{}'.format(outfields, u'x' if outfields and outparams else u'', outparams)
+ inputs = u'{}{}{}'.format(infields, u'x' if infields and inparams else u'', inparams)
+ outputs = u'{}{}{}'.format(
+ outfields, u'x' if outfields and outparams else u'', outparams)
if inputs == '':
- inputs=u'no inputs'
+ inputs = u'no inputs'
if outputs == '':
- outputs=u'no outputs'
+ outputs = u'no outputs'
- opname = op.pretty_name.decode('utf-8')
- optype = type(op).__name__
+ opname = op.pretty_name.decode('utf-8')
+ optype = type(op).__name__
strdata = (str(i), opname, inputs, '->', outputs, optype)
ops += multiline_split(strdata, maxlen, split_sep, replace, newline_prefix)
- isize = max(strlen(s[0]) for s in ops)
- name_size = max(strlen(s[1]) for s in ops)
- in_size = max(strlen(s[2]) for s in ops)
+ isize = max(strlen(s[0]) for s in ops)
+ name_size = max(strlen(s[1]) for s in ops)
+ in_size = max(strlen(s[2]) for s in ops)
arrow_size = max(strlen(s[3]) for s in ops)
- out_size = max(strlen(s[4]) for s in ops)
- type_size = max(strlen(s[5]) for s in ops)
+ out_size = max(strlen(s[4]) for s in ops)
+ type_size = max(strlen(s[5]) for s in ops)
ss = u' {:<{isize}} {:<{name_size}} {:<{in_size}} {:<{arrow_size}} {:<{out_size}} {:<{type_size}}'.format(
- 'ID', 'OPERATOR', 'INPUTS', '', 'OUTPUTS', 'OPERATOR TYPE',
- isize=isize,
- name_size=name_size, in_size=in_size,
- arrow_size=arrow_size,
- out_size=out_size, type_size=type_size)
+ 'ID', 'OPERATOR', 'INPUTS', '', 'OUTPUTS', 'OPERATOR TYPE',
+ isize=isize,
+ name_size=name_size, in_size=in_size,
+ arrow_size=arrow_size,
+ out_size=out_size, type_size=type_size)
for (i, opname, inputs, arrow, outputs, optype) in ops:
ss += u'\n {:>{isize}} {:<{name_size}} {:<{in_size}} {:<{arrow_size}} {:<{out_size}} {:<{type_size}}'.format(
- i, opname, inputs, arrow, outputs, optype,
- isize=isize,
- name_size=name_size, in_size=in_size,
- arrow_size=arrow_size,
- out_size=out_size, type_size=type_size)
+ i, opname, inputs, arrow, outputs, optype,
+ isize=isize,
+ name_size=name_size, in_size=in_size,
+ arrow_size=arrow_size,
+ out_size=out_size, type_size=type_size)
title = u'ComputationalGraph {} discrete operator report '.format(
self.pretty_name.decode('utf-8'))
@@ -540,10 +552,11 @@ class ComputationalGraph(ComputationalGraphNode):
def available_methods(self):
avail_methods = {}
if not self.nodes:
- msg=u'No nodes present in ComputationalGraph {}.'.format(self.pretty_name.decode('utf-8'))
+ msg = u'No nodes present in ComputationalGraph {}.'.format(
+ self.pretty_name.decode('utf-8'))
raise RuntimeError(msg.encode('utf-8'))
for node in self.nodes:
- for (k,v) in node.available_methods().iteritems():
+ for (k, v) in node.available_methods().iteritems():
v = to_set(v)
if (k in avail_methods):
avail_methods[k].update(v)
@@ -556,10 +569,10 @@ class ComputationalGraph(ComputationalGraphNode):
@debug
def initialize(self,
- is_root=True,
- topgraph_method=None,
- outputs_are_inputs=False,
- **kwds):
+ is_root=True,
+ topgraph_method=None,
+ outputs_are_inputs=False,
+ **kwds):
if self.initialized:
return
self.is_root = is_root
@@ -567,7 +580,7 @@ class ComputationalGraph(ComputationalGraphNode):
if is_root:
self.pre_initialize(**kwds)
- msg=u'ComputationalGraph {} is empty.'
+ msg = u'ComputationalGraph {} is empty.'
assert len(self.nodes) > 0, msg.format(self.pretty_name.decode('utf-8')).encode('utf-8')
for node in self.nodes:
@@ -605,7 +618,7 @@ class ComputationalGraph(ComputationalGraphNode):
self.handle_topologies(self.input_topology_states, self.output_topology_states)
self.check()
- self.initialized=True
+ self.initialized = True
if is_root:
self.post_initialize(**kwds)
@@ -613,10 +626,10 @@ class ComputationalGraph(ComputationalGraphNode):
def check(self):
super(ComputationalGraph, self).check()
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
+ operators = reduced_graph.vertex_properties['operators']
for vid in self.sorted_nodes:
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
op.check()
@debug
@@ -633,7 +646,7 @@ class ComputationalGraph(ComputationalGraphNode):
@debug
def handle_topologies(self, input_topology_states, output_topology_states):
- from hysop.problem import Problem
+ from hysop.problem import Problem
# do not call super method
for node in self.nodes:
assert isinstance(node, ComputationalGraphOperator) or isinstance(node, Problem)
@@ -642,7 +655,7 @@ class ComputationalGraph(ComputationalGraphNode):
assert (node._field_requirements is not None)
if not isinstance(node, Problem):
node.handle_topologies(input_topology_states[node], output_topology_states[node])
- node.input_topology_states = input_topology_states[node]
+ node.input_topology_states = input_topology_states[node]
node.output_topology_states = output_topology_states[node]
self.topology_handled = True
@@ -655,12 +668,12 @@ class ComputationalGraph(ComputationalGraphNode):
builder = GraphBuilder(node=self)
builder.configure(current_level=current_level,
- outputs_are_inputs=outputs_are_inputs, **kwds)
+ outputs_are_inputs=outputs_are_inputs, **kwds)
builder.build_graph()
input_fields = builder.input_fields
output_fields = builder.output_fields
- candidate_input_tensors = self.candidate_input_tensors
+ candidate_input_tensors = self.candidate_input_tensors
candidate_output_tensors = self.candidate_output_tensors
input_tensor_fields = ()
@@ -680,15 +693,15 @@ class ComputationalGraph(ComputationalGraphNode):
input_params=builder.input_params,
output_params=builder.output_params)
- self.graph = builder.graph
+ self.graph = builder.graph
self.reduced_graph = builder.reduced_graph
- self.sorted_nodes = builder.sorted_nodes
- self.nodes = builder.nodes
- self.input_topology_states = builder.op_input_topology_states
+ self.sorted_nodes = builder.sorted_nodes
+ self.nodes = builder.nodes
+ self.input_topology_states = builder.op_input_topology_states
self.output_topology_states = builder.op_output_topology_states
self.initial_input_topology_states = builder.input_topology_states
- self.final_output_topology_states = builder.output_topology_states
+ self.final_output_topology_states = builder.output_topology_states
self.level = current_level
@@ -710,13 +723,13 @@ class ComputationalGraph(ComputationalGraphNode):
if (visu_rank is None) or (main_rank != visu_rank):
return
- graph = self.reduced_graph
- edge_text = graph.edge_properties['var_names']
- vertex_text = graph.vertex_properties['op_pnames']
- vertex_info = graph.vertex_properties['op_info']
+ graph = self.reduced_graph
+ edge_text = graph.edge_properties['var_names']
+ vertex_text = graph.vertex_properties['op_pnames']
+ vertex_info = graph.vertex_properties['op_info']
if 'command_queues' in graph.vp:
command_queues = graph.vertex_properties['command_queues']
- active_ops = graph.vertex_properties['active_ops']
+ active_ops = graph.vertex_properties['active_ops']
else:
command_queues = None
active_ops = None
@@ -728,16 +741,16 @@ class ComputationalGraph(ComputationalGraphNode):
pos_layout = sfdp_layout(graph)
- win = GraphWindow(graph, pos_layout, geometry=(800,600),
- vertex_text = vertex_text,
- edge_text = edge_text,
- vertex_font_size = vertex_font_size,
- edge_font_size = edge_font_size,
- vertex_color = active_ops,
- vertex_fill_color = command_queues,
- display_props=vertex_info,
- display_props_size=14,
- max_render_time=50)
+ win = GraphWindow(graph, pos_layout, geometry=(800, 600),
+ vertex_text=vertex_text,
+ edge_text=edge_text,
+ vertex_font_size=vertex_font_size,
+ edge_font_size=edge_font_size,
+ vertex_color=active_ops,
+ vertex_fill_color=command_queues,
+ display_props=vertex_info,
+ display_props_size=14,
+ max_render_time=50)
def update_window():
win.graph.regenerate_surface()
@@ -763,23 +776,24 @@ class ComputationalGraph(ComputationalGraphNode):
if self.discretized:
return
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
+ operators = reduced_graph.vertex_properties['operators']
for vid in self.sorted_nodes:
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
if not op.discretized:
op.discretize()
if self.is_root:
input_discrete_fields = {}
- for (field,topo) in self.input_fields.iteritems():
+ for (field, topo) in self.input_fields.iteritems():
istate = self.initial_input_topology_states[field][1]
- istate = istate.copy(is_read_only=False) # problem inputs are writeable for initialization
+ # problem inputs are writeable for initialization
+ istate = istate.copy(is_read_only=False)
dfield = field.discretize(topo, istate)
input_discrete_fields[field] = dfield
output_discrete_fields = {}
- for field,topo in self.output_fields.iteritems():
+ for field, topo in self.output_fields.iteritems():
ostate = self.final_output_topology_states[field][1]
dfield = field.discretize(topo, ostate)
output_discrete_fields[field] = dfield
@@ -805,20 +819,20 @@ class ComputationalGraph(ComputationalGraphNode):
output_discrete_tensor_fields[tfield] = tdfield
discrete_fields = tuple(set(input_discrete_fields.values() +
- output_discrete_fields.values()))
+ output_discrete_fields.values()))
discrete_tensor_fields = tuple(set(input_discrete_tensor_fields.values() +
output_discrete_tensor_fields.values()))
else:
- input_discrete_fields = None
+ input_discrete_fields = None
output_discrete_fields = None
input_discrete_tensor_fields = None
output_discrete_tensor_fields = None
discrete_fields = None
discrete_tensor_fields = None
- self.input_discrete_fields = input_discrete_fields
+ self.input_discrete_fields = input_discrete_fields
self.output_discrete_fields = output_discrete_fields
self.input_discrete_tensor_fields = input_discrete_tensor_fields
self.output_discrete_tensor_fields = output_discrete_tensor_fields
@@ -827,24 +841,23 @@ class ComputationalGraph(ComputationalGraphNode):
self.discretized = True
-
@debug
@discretized
def get_work_properties(self):
requests = MultipleOperatorMemoryRequests()
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
+ operators = reduced_graph.vertex_properties['operators']
for vid in self.sorted_nodes:
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
if op not in requests.operators():
wp = op.get_work_properties()
requests += op.get_work_properties()
- if __DEBUG__ or (__VERBOSE__ and self.level==0) or self.__FORCE_REPORTS__:
+ if __DEBUG__ or (__VERBOSE__ and self.level == 0) or self.__FORCE_REPORTS__:
srequests = requests.sreport()
ss = (srequests if (srequests != u'') else u' *no extra work requested*')
- title= u'ComputationalGraph {} work properties report '.format(
+ title = u'ComputationalGraph {} work properties report '.format(
self.pretty_name.decode('utf-8'))
vprint(u'\n{}\n'.format(framed_str(title=title, msg=ss)).encode('utf-8'))
return requests
@@ -858,10 +871,10 @@ class ComputationalGraph(ComputationalGraphNode):
work = self.get_work_properties()
work.allocate(allow_subbuffers=allow_subbuffers)
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
+ operators = reduced_graph.vertex_properties['operators']
for vid in self.sorted_nodes:
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
if not op.ready:
op.setup(work=work)
self.ready = True
@@ -883,9 +896,9 @@ class ComputationalGraph(ComputationalGraphNode):
@debug
@ready
def apply(self, **kwds):
- drawing = self.graph_is_rendering
+ drawing = self.graph_is_rendering
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
+ operators = reduced_graph.vertex_properties['operators']
if drawing:
active_ops = reduced_graph.vertex_properties['active_ops']
@@ -909,10 +922,10 @@ class ComputationalGraph(ComputationalGraphNode):
@ready
def finalize(self, **kwds):
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
+ operators = reduced_graph.vertex_properties['operators']
for vid in self.sorted_nodes:
vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ op = operators[vertex]
if op.ready:
op.finalize(**kwds)
self.ready = False
@@ -920,9 +933,11 @@ class ComputationalGraph(ComputationalGraphNode):
@classmethod
def supports_multiple_field_topologies(cls):
return True
+
@classmethod
def supports_multiple_topologies(cls):
return True
+
@classmethod
def supports_mpi(cls):
return True
diff --git a/hysop/core/graph/computational_node.py b/hysop/core/graph/computational_node.py
index c20dba7c9e810a753d6be7ed493fb9f48c2834e6..9bfc396f41f5968e9512b82474afe861cae987bc 100644
--- a/hysop/core/graph/computational_node.py
+++ b/hysop/core/graph/computational_node.py
@@ -6,59 +6,61 @@ Base for directionally splitted advection solvers (pure-python and GPU version).
from abc import ABCMeta, abstractmethod
from hysop import dprint
-from hysop.deps import copy, warnings
-from hysop.tools.types import InstanceOf, to_set, check_instance, first_not_None
-from hysop.tools.io_utils import IOParams
+from hysop.deps import copy, warnings
+from hysop.tools.types import InstanceOf, to_set, check_instance, first_not_None
+from hysop.tools.io_utils import IOParams
from hysop.parameters.parameter import Parameter
from hysop.fields.continuous_field import Field, ScalarField, TensorField
from hysop.core.graph.node_requirements import NodeRequirements
-from hysop.core.graph.graph import not_implemented, wraps,\
- not_initialized, initialized, discretized, ready
+from hysop.core.graph.graph import not_implemented, wraps,\
+ not_initialized, initialized, discretized, ready
from hysop.core.graph.continuous import OperatorBase
from hysop.topology.topology import Topology, TopologyView
from hysop.tools.decorators import debug
from hysop.tools.warning import HysopWarning
from hysop.topology.cartesian_descriptor import get_topo_descriptor_discretization
-
+
def base_initialized(f):
assert callable(f)
@wraps(f)
- def _check(*args,**kwds):
+ def _check(*args, **kwds):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self._base_initialized:
- reason='this self._init_base() has not been called yet.'
+ reason = 'this self._init_base() has not been called yet.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kwds)
+ return f(*args, **kwds)
return _check
+
def topology_handled(f):
assert callable(f)
@wraps(f)
- def _check(*args,**kwds):
+ def _check(*args, **kwds):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self.topology_handled:
- reason='this self.handle_topologies() has not been called yet.'
+ reason = 'this self.handle_topologies() has not been called yet.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kwds)
+ return f(*args, **kwds)
return _check
+
class ComputationalGraphNode(OperatorBase):
"""
Interface of an abstract computational graph node.
"""
__metaclass__ = ABCMeta
-
+
@debug
- def __init__(self, input_fields=None, output_fields=None,
- input_params=None, output_params=None,
- input_tensor_fields=None, output_tensor_fields=None,
- name=None, pretty_name=None, method=None, **kwds):
+ def __init__(self, input_fields=None, output_fields=None,
+ input_params=None, output_params=None,
+ input_tensor_fields=None, output_tensor_fields=None,
+ name=None, pretty_name=None, method=None, to_be_skipped_func=None, **kwds):
"""
Initialize a ComputationalGraphNode.
@@ -87,16 +89,16 @@ class ComputationalGraphNode(OperatorBase):
Pretty name of this node (string), optional, defaults to name.
method: dict, optional
user method specification for this graph node, optional, defaults to None.
- kwds:
+ kwds:
arguments for base classes (mpi_params and io_params).
Attributes
----------
- name: str
+ name: str
name of this node (used for printing and display purpose).
- pretty_name: str
+ pretty_name: str
Pretty name of this node (used for printing and display purpose).
- input_fields: dict
+ input_fields: dict
input fields as a dictionnary (see Notes).
output_fields: dict
output fields as a dictionnary (see Notes).
@@ -108,54 +110,54 @@ class ComputationalGraphNode(OperatorBase):
flag set after discretize() has been called.
ready: bool
flag set after setup() has been called.
-
+
method : dict(MethodKey, MethodValue)
method, set after initialize() has been called.
input_field_requirements : dict(Field, DiscreteFieldRequirements)
input constraints, set after initialize() has been called.
output_field_requirements = {}
output constraints, set after initialize() has been called.
-
+
Notes
-----
- For the input and output fields, the keys of the dicts have to be of
+ For the input and output fields, the keys of the dicts have to be of
type :class:`hysop.fields.continuous_field.Field`.
- and the values should consist of
+ and the values should consist of
:class:`hysop.topology.topology_descriptor.TopologyDescriptors` instances
ie. an already defined topology or a topology descriptor.
VectorFields and TensorFields are expanded to ScalarFields.
- For input and output parameters, the keys of the dicts can be arbitrary names that
+ For input and output parameters, the keys of the dicts can be arbitrary names that
can be used to retrieve the parameters
Giving the following keywords as inputs (in **kwds) will throw a ValueError:
input_vars, output_vars, variables, iwork, rwork, work, backend
- About the method parameter:
+ About the method parameter:
One can not directly use the method parameter after this call.
User method is put into attribute base_method awaiting the initialization step.
See ComputationalGraphNode.handle_method() to see how method is handled.
"""
should_init = (input_fields is not None) or (output_fields is not None) \
- or (input_params is not None) or (output_params is not None)
-
+ or (input_params is not None) or (output_params is not None)
+
# Check extra args
cls = self.__class__
for _ in ('variables', 'input_vars', 'output_vars'):
if _ in kwds.keys():
- msg='The \'{}\' parameter should not be used in {}, use input_fields and '
- msg +='output_fields instead.'
+ msg = 'The \'{}\' parameter should not be used in {}, use input_fields and '
+ msg += 'output_fields instead.'
msg = msg.format(_, cls)
raise ValueError(msg)
if ('iwork' in kwds) or ('rwork' in kwds) or ('work' in kwds):
- msg='work, rwork or iwork parameters can not be used before the full description \
+ msg = 'work, rwork or iwork parameters can not be used before the full description \
of the graph in class {}.'.format(cls)
raise ValueError(msg)
if ('backend' in kwds):
- msg='{} is not a ComputationalGraphNodeFrontend thus no backend can be specified.'
- msg=msg.format(cls)
+ msg = '{} is not a ComputationalGraphNodeFrontend thus no backend can be specified.'
+ msg = msg.format(cls)
raise ValueError(msg)
# Expand input and output TensorFields to ScalarFields
@@ -166,12 +168,12 @@ class ComputationalGraphNode(OperatorBase):
for tfield in input_tensor_fields:
for field in tfield:
if (field not in input_fields):
- msg='Input fields and input tensor fields mismatch.'
+ msg = 'Input fields and input tensor fields mismatch.'
raise RuntimeError(msg)
elif (input_fields is not None):
input_tensor_fields = tuple(filter(lambda x: x.is_tensor, input_fields.keys()))
- input_fields = {sfield: topod for (tfield,topod) in input_fields.iteritems()
- for sfield in tfield.fields }
+ input_fields = {sfield: topod for (tfield, topod) in input_fields.iteritems()
+ for sfield in tfield.fields}
else:
input_tensor_fields = ()
@@ -181,86 +183,86 @@ class ComputationalGraphNode(OperatorBase):
for tfield in output_tensor_fields:
for field in tfield:
if (field not in output_fields):
- msg='Output fields and output tensor fields mismatch.'
+ msg = 'Output fields and output tensor fields mismatch.'
raise RuntimeError(msg)
elif (output_fields is not None):
output_tensor_fields = tuple(filter(lambda x: x.is_tensor, output_fields.keys()))
- output_fields = {sfield: topod for (tfield,topod) in output_fields.iteritems()
- for sfield in tfield.fields }
+ output_fields = {sfield: topod for (tfield, topod) in output_fields.iteritems()
+ for sfield in tfield.fields}
else:
output_tensor_fields = ()
# Check input values
- input_fields = first_not_None(input_fields, {})
+ input_fields = first_not_None(input_fields, {})
output_fields = first_not_None(output_fields, {})
- input_params = first_not_None(input_params, {})
+ input_params = first_not_None(input_params, {})
output_params = first_not_None(output_params, {})
- method = first_not_None(method, {})
- name = first_not_None(name, self.__class__.__name__)
- pretty_name = first_not_None(pretty_name, name)
-
+ method = first_not_None(method, {})
+ name = first_not_None(name, self.__class__.__name__)
+ pretty_name = first_not_None(pretty_name, name)
+
if isinstance(pretty_name, unicode):
pretty_name = pretty_name.encode('utf-8')
-
+
if not isinstance(name, str):
- msg='name is not a string but a {}.'
+ msg = 'name is not a string but a {}.'
raise ValueError(msg.format(name.__class__))
- if not isinstance(pretty_name, (str,unicode)):
- msg='pretty_name is not a string but a {}.'
+ if not isinstance(pretty_name, (str, unicode)):
+ msg = 'pretty_name is not a string but a {}.'
raise ValueError(msg.format(name.__class__))
if not isinstance(input_fields, dict):
- msg='input_fields is not a dict but a {}.'
+ msg = 'input_fields is not a dict but a {}.'
raise ValueError(msg.format(input_fields.__class__))
if not isinstance(output_fields, dict):
- msg='output_fields is not a dict but a {}.'
+ msg = 'output_fields is not a dict but a {}.'
raise ValueError(msg.format(output_fields.__class__))
if not isinstance(input_params, dict):
input_params = to_set(input_params)
- input_params = { p.name:p for p in input_params }
+ input_params = {p.name: p for p in input_params}
if not isinstance(output_params, dict):
output_params = to_set(output_params)
- output_params = { p.name:p for p in output_params }
-
- self.name = name
- self.pretty_name = pretty_name
-
- self.input_fields = input_fields
- self.output_fields = output_fields
-
- self.input_params = input_params
- self.output_params = output_params
-
- self.input_tensor_fields = input_tensor_fields
+ output_params = {p.name: p for p in output_params}
+
+ self.name = name
+ self.pretty_name = pretty_name
+
+ self.input_fields = input_fields
+ self.output_fields = output_fields
+
+ self.input_params = input_params
+ self.output_params = output_params
+
+ self.input_tensor_fields = input_tensor_fields
self.output_tensor_fields = output_tensor_fields
self.base_method = method
- self.initialized = False
+ self.initialized = False
self.topology_handled = False
- self.discretized = False
- self.ready = False
+ self.discretized = False
+ self.ready = False
- self.input_discrete_fields = None
+ self.input_discrete_fields = None
self.output_discrete_fields = None
- self.discrete_fields = None
- self.input_discrete_tensor_fields = None
+ self.discrete_fields = None
+ self.input_discrete_tensor_fields = None
self.output_discrete_tensor_fields = None
- self.discrete_tensor_fields = None
+ self.discrete_tensor_fields = None
if not hasattr(self, '_field_requirements'):
self._field_requirements = None
# graph builder hints to build I/O operators.
- self._input_fields_to_dump = []
+ self._input_fields_to_dump = []
self._output_fields_to_dump = []
- self._input_params_to_dump = []
+ self._input_params_to_dump = []
self._output_params_to_dump = []
self._base_initialized = False
self.__kwds = kwds
-
+
if should_init:
- self._init_base(input_fields, output_fields,
+ self._init_base(input_fields, output_fields,
input_tensor_fields, output_tensor_fields,
input_params, output_params)
else:
@@ -272,6 +274,14 @@ class ComputationalGraphNode(OperatorBase):
self.io_params = io_params
self._set_io()
+ # Default function for skipping operator's apply
+ def to_be_skipped(*args, **kwargs):
+ return False
+ if to_be_skipped_func is None:
+ self.to_be_skipped = to_be_skipped
+ else:
+ self.to_be_skipped = to_be_skipped_func
+
def _get_is_domainless(self):
"""Return True if this node has no input nor output fields."""
return (not self.input_fields) and (not self.output_fields)
@@ -290,7 +300,7 @@ class ComputationalGraphNode(OperatorBase):
else:
scalar_fields += (field,)
return (scalar_fields, tensor_fields)
-
+
@debug
def _setup_method(self, topgraph_method):
"""
@@ -299,13 +309,13 @@ class ComputationalGraphNode(OperatorBase):
"""
cls = type(self)
if topgraph_method:
- base_method = self.base_method
+ base_method = self.base_method
avail_methods = self.available_methods()
extra_keys = set(topgraph_method.keys())\
- .intersection(avail_methods.keys())\
- .difference(self.base_method.keys())
+ .intersection(avail_methods.keys())\
+ .difference(self.base_method.keys())
- method = self.base_method.copy()
+ method = self.base_method.copy()
for k in extra_keys:
method[k] = topgraph_method[k]
else:
@@ -315,23 +325,23 @@ class ComputationalGraphNode(OperatorBase):
return method
@debug
- def _init_base(self, input_fields, output_fields,
- input_tensor_fields, output_tensor_fields,
- input_params, output_params):
+ def _init_base(self, input_fields, output_fields,
+ input_tensor_fields, output_tensor_fields,
+ input_params, output_params):
"""
Initialize base class and check everything.
"""
# Merge scalar and tensor fields
all_input_fields = tuple(input_tensor_fields)
for ofield in input_fields.keys():
- if not any (ofield in tf for tf in input_tensor_fields):
+ if not any(ofield in tf for tf in input_tensor_fields):
all_input_fields += (ofield,)
all_output_fields = tuple(output_tensor_fields)
for ofield in output_fields.keys():
- if not any (ofield in tf for tf in output_tensor_fields):
+ if not any(ofield in tf for tf in output_tensor_fields):
all_output_fields += (ofield,)
-
+
assert not self._base_initialized
check_instance(input_fields, dict, keys=ScalarField)
check_instance(output_fields, dict, keys=ScalarField)
@@ -342,41 +352,41 @@ class ComputationalGraphNode(OperatorBase):
check_instance(all_input_fields, tuple, values=Field)
check_instance(all_output_fields, tuple, values=Field)
- self.input_fields = input_fields
+ self.input_fields = input_fields
self.output_fields = output_fields
- self.input_params = input_params
+ self.input_params = input_params
self.output_params = output_params
self.input_tensor_fields = input_tensor_fields
self.output_tensor_fields = output_tensor_fields
-
+
ifields = set(self.input_fields.keys())
ofields = set(self.output_fields.keys())
- fields = tuple(ifields.union(ofields))
-
+ fields = tuple(ifields.union(ofields))
+
itfields = set(self.input_tensor_fields)
otfields = set(self.output_tensor_fields)
- tfields = tuple(itfields.union(otfields))
-
+ tfields = tuple(itfields.union(otfields))
+
iparams = set(self.input_params.values())
oparams = set(self.output_params.values())
parameters = tuple(iparams.union(oparams))
-
+
if 'mpi_params' in self.__kwds:
mpi_params = self.__kwds['mpi_params']
for topo in set(self.input_fields.values() + self.output_fields.values()):
if isinstance(topo, Topology) and (topo.mpi_params != mpi_params):
- msg='MPI parameters mismatch between already specified topology mpi_params '
- msg+='and operator MPI paramaters in operator {}.'.format(self.name)
- msg+='\n *operator: {}'.format(mpi_params)
- msg+='\n *field: {}'.format(topo.mpi_params)
- msg+='\n'
+ msg = 'MPI parameters mismatch between already specified topology mpi_params '
+ msg += 'and operator MPI paramaters in operator {}.'.format(self.name)
+ msg += '\n *operator: {}'.format(mpi_params)
+ msg += '\n *field: {}'.format(topo.mpi_params)
+ msg += '\n'
raise RuntimeError(msg)
- super(ComputationalGraphNode, self).__init__(name=self.name,
- fields=fields,
- tensor_fields=tfields,
- parameters=parameters,
- **self.__kwds)
+ super(ComputationalGraphNode, self).__init__(name=self.name,
+ fields=fields,
+ tensor_fields=tfields,
+ parameters=parameters,
+ **self.__kwds)
self._base_initialized = True
self.all_input_fields = all_input_fields
self.all_output_fields = all_output_fields
@@ -392,30 +402,30 @@ class ComputationalGraphNode(OperatorBase):
method.update(user_method)
available_methods = self.available_methods()
- for (k,v) in method.iteritems():
+ for (k, v) in method.iteritems():
if k not in available_methods.keys():
- msg='{} is not an available method key for computational node {}.'
+ msg = '{} is not an available method key for computational node {}.'
msg = msg.format(k, self.name)
warnings.warn(msg, HysopWarning)
continue
-
+
available = to_set(available_methods[k])
- instances = set(x for x in available if isinstance(x,InstanceOf))
+ instances = set(x for x in available if isinstance(x, InstanceOf))
available = available.difference(instances)
-
- good=False
+
+ good = False
for instance in instances:
if instance.match_instance(v):
- good=True
+ good = True
break
good = good or (v in available)
if (not good):
- msg='{} is not an available method value for key {},'.format(v, k.__name__)
- msg+='\n possible values are {}.'.format(available_methods[k])
+ msg = '{} is not an available method value for key {},'.format(v, k.__name__)
+ msg += '\n possible values are {}.'.format(available_methods[k])
raise ValueError(msg)
return method
-
+
@debug
@base_initialized
def check(self):
@@ -426,7 +436,7 @@ class ComputationalGraphNode(OperatorBase):
self._check_variables()
self._check_topologies()
self._check_support()
-
+
@debug
@base_initialized
def _check_variables(self):
@@ -435,14 +445,14 @@ class ComputationalGraphNode(OperatorBase):
Called automatically in ComputationalGraphNode.check()
"""
for variables in [self.input_fields, self.output_fields]:
- for (k,v) in variables.iteritems():
- if not isinstance(k,Field):
- msg = 'Given key is not a continuous Field (got a {}).'
+ for (k, v) in variables.iteritems():
+ if not isinstance(k, Field):
+ msg = 'Given key is not a continuous Field (got a {}).'
raise TypeError(msg.format(k.__class__))
if not isinstance(v, TopologyView):
- msg='Expected a Topology instance but got a {}.'.format(v.__class__)
- msg+='\nAll topologies are expected to be set after '
- msg+='ComputationalGraph.get_field_requirements() has been called.'
+ msg = 'Expected a Topology instance but got a {}.'.format(v.__class__)
+ msg += '\nAll topologies are expected to be set after '
+ msg += 'ComputationalGraph.get_field_requirements() has been called.'
raise TypeError(msg)
@debug
@@ -457,11 +467,11 @@ class ComputationalGraphNode(OperatorBase):
_multi_topo_fields (list of field that have at least two different topologies)
Called automatically in ComputationalGraphNode.check()
"""
- is_distributed = (self.mpi_params.size > 1)
- has_multiple_topologies = False
+ is_distributed = (self.mpi_params.size > 1)
+ has_multiple_topologies = False
has_multiple_field_topologies = False
multi_topo_fields = set()
-
+
topos = (self.input_fields.values()+self.output_fields.values())
if topos:
topo_ref = topos[0].topology
@@ -476,16 +486,16 @@ class ComputationalGraphNode(OperatorBase):
multi_topo_fields.add(ifield)
has_multiple_field_topologies = True
- self._is_distributed = is_distributed
- self._has_multiple_topologies = has_multiple_topologies
+ self._is_distributed = is_distributed
+ self._has_multiple_topologies = has_multiple_topologies
self._has_multiple_field_topologies = has_multiple_field_topologies
- self._multi_topo_fields = multi_topo_fields
-
+ self._multi_topo_fields = multi_topo_fields
+
@debug
@base_initialized
def _check_support(self):
"""
- Check input and output variables topologies against the supported topologies of
+ Check input and output variables topologies against the supported topologies of
this node.
See ComputationalGraphNode.supports_multiple_topologies()
@@ -496,29 +506,29 @@ class ComputationalGraphNode(OperatorBase):
cls = self.__class__
if (self._has_multiple_field_topologies) and \
(not cls.supports_multiple_field_topologies()):
- msg='Graph operator \'{}\' does not support multiple topologies yet.'
- msg+= '\nTopology mismatch for continuous variable(s) {} between '
- msg+= 'input and output variables.'
- msg=msg.format(self.name, [f.name for f in self._multi_topo_fields])
+ msg = 'Graph operator \'{}\' does not support multiple topologies yet.'
+ msg += '\nTopology mismatch for continuous variable(s) {} between '
+ msg += 'input and output variables.'
+ msg = msg.format(self.name, [f.name for f in self._multi_topo_fields])
raise NotImplementedError(msg)
if (self._has_multiple_topologies) and \
(not cls.supports_multiple_topologies()):
- msg='Graph operator {} does not support multiple field topologies yet.'
- msg=msg.format(self.node_tag)
- msg+='\n>Input topologies:'
+ msg = 'Graph operator {} does not support multiple field topologies yet.'
+ msg = msg.format(self.node_tag)
+ msg += '\n>Input topologies:'
for (field, topo) in self.input_fields.iteritems():
- msg+='\n *{} -> {}'.format(field.short_description(), topo.short_description())
- msg+='\n>Output topologies:'
+ msg += '\n *{} -> {}'.format(field.short_description(), topo.short_description())
+ msg += '\n>Output topologies:'
for (field, topo) in self.output_fields.iteritems():
- msg+='\n *{} -> {}'.format(field.short_description(), topo.short_description())
+ msg += '\n *{} -> {}'.format(field.short_description(), topo.short_description())
raise NotImplementedError(msg)
if (self._is_distributed) and (not cls.supports_mpi()):
- msg='\nMPI multi-process has not been implemented in graph operator \'{}\' yet!\n'
- msg=msg.format(type(self))
+ msg = '\nMPI multi-process has not been implemented in graph operator \'{}\' yet!\n'
+ msg = msg.format(type(self))
raise NotImplementedError(msg)
-
-## ComputationalGraphNode interface
+
+# ComputationalGraphNode interface
@base_initialized
def get_topologies(self):
"""
@@ -533,7 +543,7 @@ class ComputationalGraphNode(OperatorBase):
def get_domains(self):
"""
Returns all the domains used in this operator.
- Domains are keys and values are operators that have variables
+ Domains are keys and values are operators that have variables
defined on this domain.
If this node has no domain (ie. no input or output variables),
if fills the 'None' domain.
@@ -551,7 +561,7 @@ class ComputationalGraphNode(OperatorBase):
Returns all the backends used in this operator as a set.
"""
return self.get_topologies().keys()
-
+
@abstractmethod
def available_methods(self):
"""
@@ -562,7 +572,7 @@ class ComputationalGraphNode(OperatorBase):
class types. This is used to check user method input.
"""
pass
-
+
@abstractmethod
def default_method(self):
"""
@@ -572,20 +582,20 @@ class ComputationalGraphNode(OperatorBase):
a default value for this key will be extracted from the default one.
"""
pass
-
+
@debug
def handle_method(self, method):
"""
Method automatically called during initialization.
This allow to extract method values after method preprocessing.
Method preprocessing means:
- 1) complete user input with compatible top graph user inputs
+ 1) complete user input with compatible top graph user inputs
2) complete the resulting dictionnary with the node default_method
3) check method against available_methods.
The result of this process is fed as argument of this function.
"""
- self.method = {k:v for (k,v) in method.iteritems()}
-
+ self.method = {k: v for (k, v) in method.iteritems()}
+
@abstractmethod
@debug
def get_field_requirements(self):
@@ -594,7 +604,7 @@ class ComputationalGraphNode(OperatorBase):
Topology requirements are:
1) min and max ghosts for each input and output variables
2) allowed splitting directions for cartesian topologies
- 3) required local and global transposition state, if any.
+ 3) required local and global transposition state, if any.
and more
They are stored in self.input_field_requirements and
self.output_field_requirements.
@@ -609,7 +619,7 @@ class ComputationalGraphNode(OperatorBase):
def get_node_requirements(self):
"""Called after get_field_requirements to get global node requirements."""
return NodeRequirements(self)
-
+
@debug
def get_and_set_field_requirements(self):
"""
@@ -620,10 +630,10 @@ class ComputationalGraphNode(OperatorBase):
assert (field_requirements is not None)
self._field_requirements = field_requirements
- node_requirements = self.get_node_requirements()
+ node_requirements = self.get_node_requirements()
assert isinstance(node_requirements, NodeRequirements)
self._node_requirements = field_requirements
-
+
node_requirements.check_and_update_reqs(field_requirements)
return field_requirements
@@ -634,9 +644,9 @@ class ComputationalGraphNode(OperatorBase):
"""
freqs = self._field_requirements
if (freqs is None):
- msg='{}.get_and_set_field_requirements() has not been called yet '
- msg+='on node {}.'
- msg=msg.format(type(self).__name__, self.name)
+ msg = '{}.get_and_set_field_requirements() has not been called yet '
+ msg += 'on node {}.'
+ msg = msg.format(type(self).__name__, self.name)
raise RuntimeError(msg)
return self._field_requirements.input_field_requirements
@@ -646,22 +656,22 @@ class ComputationalGraphNode(OperatorBase):
"""
freqs = self._field_requirements
if (freqs is None):
- msg='{}.get_and_set_field_requirements() has not been called yet '
- msg+='on node {}.'
- msg=msg.format(type(self).__name__, self.name)
+ msg = '{}.get_and_set_field_requirements() has not been called yet '
+ msg += 'on node {}.'
+ msg = msg.format(type(self).__name__, self.name)
raise RuntimeError(msg)
return freqs.output_field_requirements
-
- input_field_requirements = property(get_input_field_requirements)
- output_field_requirements = property(get_output_field_requirements)
-
+
+ input_field_requirements = property(get_input_field_requirements)
+ output_field_requirements = property(get_output_field_requirements)
+
@debug
def handle_topologies(self, input_topology_states,
- output_topology_states):
+ output_topology_states):
"""
Called after all topologies have been set up.
- Topologies are available as values of self.input_fields
+ Topologies are available as values of self.input_fields
and self.output_fields and are mapped by continuous Field.
In addition input_topology_states are passed as argument
@@ -683,22 +693,22 @@ class ComputationalGraphNode(OperatorBase):
for tfield in tfields:
if field in tfield:
return variables[tfield]
- msg='Could not find any topology descriptor corresponding to field {}.'
- msg=msg.format(field.short_description())
+ msg = 'Could not find any topology descriptor corresponding to field {}.'
+ msg = msg.format(field.short_description())
raise KeyError(msg)
-
+
@classmethod
def get_topo_discretization(cls, variables, field):
topo = cls.get_topo_descriptor(variables=variables, field=field)
return get_topo_descriptor_discretization(topo)
-
+
@classmethod
def supports_multiple_topologies(cls):
"""
Should return True if this node supports multiple topologies.
"""
return True
-
+
@classmethod
def supports_multiple_field_topologies(cls):
"""
@@ -707,7 +717,7 @@ class ComputationalGraphNode(OperatorBase):
This is usefull in Redistribute like operators.
If this returns True this implies supports_multiple_topologies().
It also implies that self.variables[field] may return a set of topologies.
- In this case one can recover input and output topologies by using
+ In this case one can recover input and output topologies by using
self.input_fields[field] and self.output_fields[field].
In addition one can find such fields by using the list self.multi_topo_fields
which is set after ComputationalGraphNode.initialize() has been called.
@@ -720,7 +730,7 @@ class ComputationalGraphNode(OperatorBase):
Return True if this operator was implemented to support multiple mpi processes.
"""
return False
-
+
@debug
def pre_initialize(self, **kwds):
"""
@@ -731,7 +741,7 @@ class ComputationalGraphNode(OperatorBase):
"""
pass
- @debug
+ @debug
def post_initialize(self, **kwds):
"""
Function called after initialization,
@@ -744,11 +754,11 @@ class ComputationalGraphNode(OperatorBase):
def initialize(self, topgraph_method=None, **kwds):
"""
Initialize this node.
-
+
Initialization step sets the following variables:
- *self.method,
- *self.input_field_requirements
- *self.output_field_requirements
+ *self.method,
+ *self.input_field_requirements
+ *self.output_field_requirements
*self.initialized
It returns self.method.
@@ -759,25 +769,25 @@ class ComputationalGraphNode(OperatorBase):
self.get_field_requirements()
self._initialized = True
self.post_initialize()
-
+
See ComputationalGraphNode.handle_method() to see how user method is handled.
See ComputationalGraphNode.get_field_requirements() to see how topology requirements
are handled.
-
+
After this method has been handled by all operators, initialization collects min and max
ghosts required by each operators which will be usefull in the discretiezation step
to automatically build topologies or check against user supplied topologies.
- This function also sets the self.initialized flag to True (just before post
+ This function also sets the self.initialized flag to True (just before post
initialization).
Once this flag is set one may call ComputationalGraphNode.discretize().
"""
if self.initialized:
return
-
+
method = self._setup_method(topgraph_method)
self.handle_method(method)
- self.initialized=True
+ self.initialized = True
return method
@@ -787,11 +797,11 @@ class ComputationalGraphNode(OperatorBase):
"""
Discretize this operator.
By default this just sets the self.discretized flag to True.
- Once this flag is set one may call ComputationalGraphNode.get_work_properties() and
+ Once this flag is set one may call ComputationalGraphNode.get_work_properties() and
ComputationalGraphNode.setup().
"""
self.discretized = True
-
+
@discretized
def get_input_discrete_field(self, field):
"""
@@ -802,24 +812,24 @@ class ComputationalGraphNode(OperatorBase):
check_instance(field, Field)
if (self.input_discrete_fields is None):
- msg='{}(name={}) \n => Discretization did not set self.input_discrete_fields.'
- msg=msg.format(self.full_tag, self.name)
+ msg = '{}(name={}) \n => Discretization did not set self.input_discrete_fields.'
+ msg = msg.format(self.full_tag, self.name)
raise RuntimeError(msg)
if (self.input_discrete_tensor_fields is None):
- msg='{}(name={}) \n => Discretization did not set self.input_discrete_tensor_fields.'
- msg=msg.format(self.full_tag, self.name)
+ msg = '{}(name={}) \n => Discretization did not set self.input_discrete_tensor_fields.'
+ msg = msg.format(self.full_tag, self.name)
raise RuntimeError(msg)
if field.is_tensor:
if (field not in self.input_tensor_fields):
- msg="{} is not a registered input TensorField for graph node:\n{}"
- msg=msg.format(field.short_description(), self.long_description())
+ msg = "{} is not a registered input TensorField for graph node:\n{}"
+ msg = msg.format(field.short_description(), self.long_description())
raise RuntimeError(msg)
return self.input_discrete_tensor_fields[field]
else:
if (field not in self.input_fields):
- msg="{} is not a registered input ScalarField for graph node:\n{}"
- msg=msg.format(field.short_description(), self.long_description())
+ msg = "{} is not a registered input ScalarField for graph node:\n{}"
+ msg = msg.format(field.short_description(), self.long_description())
raise RuntimeError(msg)
return self.input_discrete_fields[field]
@@ -833,31 +843,31 @@ class ComputationalGraphNode(OperatorBase):
check_instance(field, Field)
if (self.output_discrete_fields is None):
- msg='{}(name={}) \n => Discretization did not set self.output_discrete_fields.'
- msg=msg.format(self.full_tag, self.name)
+ msg = '{}(name={}) \n => Discretization did not set self.output_discrete_fields.'
+ msg = msg.format(self.full_tag, self.name)
raise RuntimeError(msg)
if (self.output_discrete_tensor_fields is None):
- msg='{}(name={}) \n => Discretization did not set self.output_discrete_tensor_fields.'
- msg=msg.format(self.full_tag, self.name)
+ msg = '{}(name={}) \n => Discretization did not set self.output_discrete_tensor_fields.'
+ msg = msg.format(self.full_tag, self.name)
raise RuntimeError(msg)
if field.is_tensor:
if (field not in self.output_tensor_fields):
- msg="{} is not a registered output TensorField for graph node:\n{}"
- msg=msg.format(field.short_description(), self.long_description())
+ msg = "{} is not a registered output TensorField for graph node:\n{}"
+ msg = msg.format(field.short_description(), self.long_description())
raise RuntimeError(msg)
return self.output_discrete_tensor_fields[field]
else:
if (field not in self.output_fields):
- msg="{} is not a registered output ScalarField for graph node:\n{}"
- msg=msg.format(field.short_description(), self.long_description())
+ msg = "{} is not a registered output ScalarField for graph node:\n{}"
+ msg = msg.format(field.short_description(), self.long_description())
raise RuntimeError(msg)
return self.output_discrete_fields[field]
-
+
@base_initialized
- def iter_input_fields(self, with_scalars=True,
- with_tensors=True,
- as_scalars=False):
+ def iter_input_fields(self, with_scalars=True,
+ with_tensors=True,
+ as_scalars=False):
"""
Iterate over all input fields.
By default iterate over all tensors and scalars unless
@@ -867,8 +877,8 @@ class ComputationalGraphNode(OperatorBase):
"""
assert with_scalars or with_tensors, 'iterating over nothing'
input_scalar_fields_from_tensors = set(field
- for tfield in self.input_tensor_fields
- for field in tfield.fields)
+ for tfield in self.input_tensor_fields
+ for field in tfield.fields)
if with_tensors and (not as_scalars):
for tfield in self.input_tensor_fields:
@@ -885,9 +895,9 @@ class ComputationalGraphNode(OperatorBase):
yield field
@base_initialized
- def iter_output_fields(self, with_scalars=True,
- with_tensors=True,
- as_scalars=False):
+ def iter_output_fields(self, with_scalars=True,
+ with_tensors=True,
+ as_scalars=False):
"""
Iterate over all output fields.
By default iterate over all tensors and scalars unless
@@ -897,8 +907,8 @@ class ComputationalGraphNode(OperatorBase):
"""
assert with_scalars or with_tensors, 'iterating over nothing'
output_scalar_fields_from_tensors = set(field
- for tfield in self.output_tensor_fields
- for field in tfield.fields)
+ for tfield in self.output_tensor_fields
+ for field in tfield.fields)
if with_tensors and (not as_scalars):
for tfield in self.output_tensor_fields:
@@ -915,9 +925,9 @@ class ComputationalGraphNode(OperatorBase):
yield field
@discretized
- def iter_input_discrete_fields(self, with_scalars=True,
- with_tensors=True,
- as_scalars=False):
+ def iter_input_discrete_fields(self, with_scalars=True,
+ with_tensors=True,
+ as_scalars=False):
"""
Iterate over all input (field, discrete_field) pairs.
By default iterate over all tensors and scalars unless
@@ -927,8 +937,8 @@ class ComputationalGraphNode(OperatorBase):
"""
assert with_scalars or with_tensors, 'iterating over nothing'
input_scalar_fields_from_tensors = set(field
- for tfield in self.input_tensor_fields
- for field in tfield.fields)
+ for tfield in self.input_tensor_fields
+ for field in tfield.fields)
if with_tensors and (not as_scalars):
for (tfield, tdfield) in self.input_discrete_tensor_fields.iteritems():
@@ -945,9 +955,9 @@ class ComputationalGraphNode(OperatorBase):
yield (field, dfield)
@discretized
- def iter_output_discrete_fields(self, with_scalars=True,
- with_tensors=True,
- as_scalars=False):
+ def iter_output_discrete_fields(self, with_scalars=True,
+ with_tensors=True,
+ as_scalars=False):
"""
Iterate over all output (field, discrete_field) pairs.
By default iterate over all tensors and scalars unless
@@ -957,8 +967,8 @@ class ComputationalGraphNode(OperatorBase):
"""
assert with_scalars or with_tensors, 'iterating over nothing'
output_scalar_fields_from_tensors = set(field
- for tfield in self.output_tensor_fields
- for field in tfield.fields)
+ for tfield in self.output_tensor_fields
+ for field in tfield.fields)
if with_tensors and (not as_scalars):
for (tfield, tdfield) in self.output_discrete_tensor_fields.iteritems():
@@ -980,20 +990,20 @@ class ComputationalGraphNode(OperatorBase):
"""
Returns extra memory requirements of this node.
This allows operators to request for temporary buffers
- that will be shared between operators in a graph to reduce
+ that will be shared between operators in a graph to reduce
the memory footprint and the number of allocations.
By default this returns None, meanning that this node requires
no extra buffers.
"""
return None
-
+
@debug
@discretized
def setup(self, work):
"""
Setup temporary buffer that have been requested in get_work_properties().
This function may be used to execute post allocation routines.
- This sets self.ready flag to True.
+ This sets self.ready flag to True.
Once this flag is set one may call ComputationalGraphNode.apply() and
ComputationalGraphNode.finalize().
"""
@@ -1007,7 +1017,7 @@ class ComputationalGraphNode(OperatorBase):
Applies this node (operator, computational graph operator...).
"""
pass
-
+
@debug
@ready
def finalize(self, **kwds):
@@ -1017,13 +1027,13 @@ class ComputationalGraphNode(OperatorBase):
"""
self.ready = False
- def dump_inputs(self, fields=None, io_params=None,
- filename=None, frequency=None, fileformat=None, io_leader=None,
- **op_kwds):
+ def dump_inputs(self, fields=None, io_params=None,
+ filename=None, frequency=None, fileformat=None, io_leader=None,
+ **op_kwds):
"""
Tell this operator to dump some of its inputs before
apply is called.
-
+
Target folder, file, dump frequency and other io pameters
are passed trough io_params or as keywords.
"""
@@ -1031,46 +1041,46 @@ class ComputationalGraphNode(OperatorBase):
if (fields is not None):
if isinstance(fields, Field):
fields = (fields,)
- check_instance(fields, (set,list,tuple), values=Field)
+ check_instance(fields, (set, list, tuple), values=Field)
if self._base_initialized:
for field in fields:
- if ((field not in self.input_fields) and \
- (field not in self.input_tensor_fields)):
- msg='Field {} is not an input field of operator {}.'
- msg=msg.format(field.name, self.name)
+ if ((field not in self.input_fields) and
+ (field not in self.input_tensor_fields)):
+ msg = 'Field {} is not an input field of operator {}.'
+ msg = msg.format(field.name, self.name)
raise RuntimeError(msg)
else:
assert self._base_initialized, self.name
fields = self.all_input_fields
- fields = list(sorted(fields, key=lambda f:f.name))
+ fields = list(sorted(fields, key=lambda f: f.name))
if (io_params is None):
- io_params = self.io_params
-
+ io_params = self.io_params
+
if (io_params is None):
- msg='io_params was never set for operator {}.'.format(self.name)
+ msg = 'io_params was never set for operator {}.'.format(self.name)
raise RuntimeError(msg)
-
- frequency = first_not_None(frequency, io_params.frequency)
+
+ frequency = first_not_None(frequency, io_params.frequency)
fileformat = first_not_None(fileformat, io_params.fileformat)
- io_leader = first_not_None(io_leader, io_params.io_leader)
-
+ io_leader = first_not_None(io_leader, io_params.io_leader)
+
if (filename is not None):
pass
elif (fields is None):
- filename='{}_in'.format(io_params.filename)
+ filename = '{}_in'.format(io_params.filename)
else:
- filename='{}_{}'.format(io_params.filename,
- '_'.join('{}in'.format(f.name) for f in fields))
+ filename = '{}_{}'.format(io_params.filename,
+ '_'.join('{}in'.format(f.name) for f in fields))
+
+ io_params = IOParams(filename=filename, frequency=frequency,
+ fileformat=fileformat, io_leader=io_leader)
- io_params = IOParams(filename=filename, frequency=frequency,
- fileformat=fileformat, io_leader=io_leader)
-
self._input_fields_to_dump.append((fields, io_params, op_kwds))
-
- def dump_outputs(self, fields=None, io_params=None,
- filename=None, frequency=None, fileformat=None, io_leader=None,
- **op_kwds):
+
+ def dump_outputs(self, fields=None, io_params=None,
+ filename=None, frequency=None, fileformat=None, io_leader=None,
+ **op_kwds):
"""
Tell this operator to dump some of its outputs after
apply is called.
@@ -1083,40 +1093,40 @@ class ComputationalGraphNode(OperatorBase):
if (fields is not None):
if isinstance(fields, Field):
fields = (fields,)
- check_instance(fields, (set,list,tuple), values=Field)
+ check_instance(fields, (set, list, tuple), values=Field)
if self._base_initialized:
for field in fields:
- if ((field not in self.output_fields) and \
- (field not in self.output_tensor_fields)):
- msg='Field {} is not an output field of operator {}.'
- msg=msg.format(field.name, self.name)
+ if ((field not in self.output_fields) and
+ (field not in self.output_tensor_fields)):
+ msg = 'Field {} is not an output field of operator {}.'
+ msg = msg.format(field.name, self.name)
raise RuntimeError(msg)
else:
fields = self.all_output_fields
- fields = list(sorted(fields, key=lambda f:f.name))
-
+ fields = list(sorted(fields, key=lambda f: f.name))
+
if (io_params is None):
- io_params = self.io_params
-
+ io_params = self.io_params
+
if (io_params is None):
- msg='io_params was never set for operator {}.'.format(self.name)
+ msg = 'io_params was never set for operator {}.'.format(self.name)
raise RuntimeError(msg)
-
- frequency = first_not_None(frequency, io_params.frequency)
+
+ frequency = first_not_None(frequency, io_params.frequency)
fileformat = first_not_None(fileformat, io_params.fileformat)
- io_leader = first_not_None(io_leader, io_params.io_leader)
-
+ io_leader = first_not_None(io_leader, io_params.io_leader)
+
if (filename is not None):
pass
elif (fields is None):
- filename='{}_out'.format(io_params.filename)
+ filename = '{}_out'.format(io_params.filename)
else:
- filename='{}_{}'.format(io_params.filename,
- '_'.join('{}out'.format(f.name) for f in fields))
+ filename = '{}_{}'.format(io_params.filename,
+ '_'.join('{}out'.format(f.name) for f in fields))
+
+ io_params = IOParams(filename=filename, frequency=frequency,
+ fileformat=fileformat, io_leader=io_leader)
- io_params = IOParams(filename=filename, frequency=frequency,
- fileformat=fileformat, io_leader=io_leader)
-
self._output_fields_to_dump.append((fields, io_params, op_kwds))
@property
@@ -1127,45 +1137,45 @@ class ComputationalGraphNode(OperatorBase):
some operators may have the same name.
"""
return '{}::{}'.format(self.__class__.__name__, self.name)
-
+
def long_description(self):
sep = '\n *'
- ss='{}[name={}, pname:{}]'
- ss=ss.format(self.full_tag, self.name, self.pretty_name)
- ss+='\n INPUT FIELDS:{}'
+ ss = '{}[name={}, pname:{}]'
+ ss = ss.format(self.full_tag, self.name, self.pretty_name)
+ ss += '\n INPUT FIELDS:{}'
if self.input_fields:
- ss = ss.format(sep + sep.join(f.short_description()
- for f in self.input_fields.keys()))
+ ss = ss.format(sep + sep.join(f.short_description()
+ for f in self.input_fields.keys()))
else:
ss = ss.format(' None')
- ss+='\n OUTPUT FIELDS:{}'
+ ss += '\n OUTPUT FIELDS:{}'
if self.output_fields:
- ss = ss.format(sep + sep.join(f.short_description()
- for f in self.output_fields.keys()))
+ ss = ss.format(sep + sep.join(f.short_description()
+ for f in self.output_fields.keys()))
else:
ss = ss.format(' None')
- ss+='\n INPUT TENSOR FIELDS:{}'
+ ss += '\n INPUT TENSOR FIELDS:{}'
if self.input_tensor_fields:
- ss = ss.format(sep + sep.join(f.short_description()
- for f in self.input_tensor_fields))
+ ss = ss.format(sep + sep.join(f.short_description()
+ for f in self.input_tensor_fields))
else:
ss = ss.format(' None')
- ss+='\n OUTPUT TENSOR FIELDS:{}'
+ ss += '\n OUTPUT TENSOR FIELDS:{}'
if self.output_tensor_fields:
- ss = ss.format(sep + sep.join(f.short_description()
- for f in self.output_tensor_fields))
+ ss = ss.format(sep + sep.join(f.short_description()
+ for f in self.output_tensor_fields))
else:
ss = ss.format(' None')
- ss+='\n INPUT PARAMS:{}'
+ ss += '\n INPUT PARAMS:{}'
if self.input_params:
- ss = ss.format(sep + sep.join(f.short_description()
- for f in self.input_params.values()))
+ ss = ss.format(sep + sep.join(f.short_description()
+ for f in self.input_params.values()))
else:
ss = ss.format(' None')
- ss+='\n OUTPUT PARAMS:{}'
+ ss += '\n OUTPUT PARAMS:{}'
if self.output_params:
- ss = ss.format(sep + sep.join(f.short_description()
- for f in self.output_params.values()))
+ ss = ss.format(sep + sep.join(f.short_description()
+ for f in self.output_params.values()))
else:
ss = ss.format(' None')
return ss
diff --git a/hysop/core/graph/graph.py b/hysop/core/graph/graph.py
index 6cd4c57577f0adc5a4fe03547fee9fb7009e5894..49b5795f5cc4f73e5bbd66295a1bc2928c4bd210 100644
--- a/hysop/core/graph/graph.py
+++ b/hysop/core/graph/graph.py
@@ -1,103 +1,107 @@
-
import inspect
import graph_tool as gt
-from graph_tool import Graph, GraphView
-from graph_tool import topology, stats, search
+from graph_tool import Graph, GraphView
+from graph_tool import topology, stats, search
from hysop.tools.decorators import not_implemented, debug, wraps, profile
-
+from hysop import vprint
+
+
class ComputationalGraphNodeData(object):
"""
Simple class to hold some node data.
"""
+
def __init__(self, current_level, node_id):
self.current_level = current_level
- self.node_id = node_id
- self.apply_kargs = [] #list of dictionnary, last one has priority
+ self.node_id = node_id
+ self.apply_kargs = [] # list of dictionnary, last one has priority
+
def __str__(self):
return '(lvl={},id={})'.format(self.current_level, self.node_id)
+
if __debug__:
# python in debug mode, all decorators do check their target attribute
def not_initialized(f):
assert callable(f)
@wraps(f)
- def _not_initialized(*args,**kargs):
- return f(*args,**kargs)
+ def _not_initialized(*args, **kargs):
+ return f(*args, **kargs)
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if self.initialized:
- reason='this node has already been initialized.'
+ reason = 'this node has already been initialized.'
raise RuntimeError(msg.format(reason))
return _not_initialized
def initialized(f):
assert callable(f)
@wraps(f)
- def _initialized(*args,**kargs):
+ def _initialized(*args, **kargs):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self.initialized:
- reason='this node has not been initialized yet.'
+ reason = 'this node has not been initialized yet.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kargs)
+ return f(*args, **kargs)
return _initialized
def discretized(f):
assert callable(f)
@wraps(f)
- def _discretized(*args,**kargs):
+ def _discretized(*args, **kargs):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self.discretized:
- reason='this node has not been discretized yet.'
+ reason = 'this node has not been discretized yet.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kargs)
+ return f(*args, **kargs)
return _discretized
def ready(f):
assert callable(f)
@wraps(f)
- def _ready(*args,**kargs):
+ def _ready(*args, **kargs):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__, self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self.ready:
- reason='this node has not been set up.'
+ reason = 'this node has not been set up.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kargs)
+ return f(*args, **kargs)
return _ready
def graph_built(f):
assert callable(f)
@wraps(f)
- def _graph_built(*args,**kargs):
+ def _graph_built(*args, **kargs):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self.graph_built:
reason = 'the graph has not been built yet.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kargs)
+ return f(*args, **kargs)
return _graph_built
def generated(f):
assert callable(f)
@wraps(f)
- def _generated(*args,**kargs):
+ def _generated(*args, **kargs):
self = args[0]
msg = 'Cannot call {}.{}() on node \'{}\' because {}'\
- .format(self.__class__.__name__,f.__name__,self.name,'{}')
+ .format(self.__class__.__name__, f.__name__, self.name, '{}')
if not self.generated:
- reason='this node has not been generated yet.'
+ reason = 'this node has not been generated yet.'
raise RuntimeError(msg.format(reason))
- return f(*args,**kargs)
+ return f(*args, **kargs)
return _generated
-else: # not __debug__
+else: # not __debug__
# python optimized, no checks
def not_initialized(f):
return f
@@ -124,46 +128,50 @@ def op_apply(f):
@ready
def apply(*args, **kwds):
#print u'APPLY {}'.format(args[0].name)
- dbg = ('dbg' in kwds)
+ dbg = ('dbg' in kwds)
dbg = dbg and (kwds['dbg'] is not None)
dbg = dbg and (kwds['dbg'].enable_on_op_apply)
- debug_dump = ('debug_dumper' in kwds)
+ debug_dump = ('debug_dumper' in kwds)
debug_dump = debug_dump and (kwds['debug_dumper'] is not None)
debug_dump = debug_dump and (kwds['debug_dumper'].enable_on_op_apply)
+ op = args[0]
if debug_dump:
assert 'simulation' in kwds
- op = args[0]
simu = kwds['simulation']
it = simu.current_iteration
t = simu.t()
_file = inspect.getsourcefile(f)
_, _line = inspect.getsourcelines(f)
- description='{}:{}'.format(_file, _line)
+ description = '{}:{}'.format(_file, _line)
for param in sorted(op.input_params.values(), key=lambda x: x.name):
tag = 'pre_{}_{}'.format(op.name, param.name)
- kwds['debug_dumper'](it, t, tag,
- (param._value,), description=description)
+ kwds['debug_dumper'](it, t, tag,
+ (param._value,), description=description)
for dfield in sorted(op.input_discrete_fields.values(), key=lambda x: x.name):
tag = 'pre_{}_{}'.format(op.name, dfield.name)
- kwds['debug_dumper'](it, t, tag,
- tuple(df.sdata.get().handle[df.compute_slices] for df in dfield.dfields), description=description)
+ kwds['debug_dumper'](it, t, tag,
+ tuple(df.sdata.get().handle[df.compute_slices] for df in dfield.dfields), description=description)
ret = f(*args, **kwds)
for param in sorted(op.output_params.values(), key=lambda x: x.name):
tag = 'post_{}_{}'.format(op.name, param.name)
- kwds['debug_dumper'](it, t, tag,
- (param._value,), description=description)
+ kwds['debug_dumper'](it, t, tag,
+ (param._value,), description=description)
for dfield in sorted(op.output_discrete_fields.values(), key=lambda x: x.name):
tag = 'post_{}_{}'.format(op.name, dfield.name)
- kwds['debug_dumper'](it, t, tag,
- tuple(df.sdata.get().handle[df.compute_slices] for df in dfield.dfields), description=description)
+ kwds['debug_dumper'](it, t, tag,
+ tuple(df.sdata.get().handle[df.compute_slices] for df in dfield.dfields), description=description)
return ret
elif dbg:
- msg=inspect.getsourcefile(f)
+ msg = inspect.getsourcefile(f)
kwds['dbg']('pre '+msg, nostack=True)
ret = f(*args, **kwds)
kwds['dbg']('post '+msg, nostack=True)
return ret
else:
- return f(*args, **kwds)
+ if not op.to_be_skipped():
+ return f(*args, **kwds)
+ else:
+ vprint("Skip {}".format(op.name))
+ return
return ret
return apply