From a481ede34ce4e1db10b31977c606bdc3a42cf00f Mon Sep 17 00:00:00 2001
From: JM Etancelin <jean-matthieu.etancelin@univ-pau.fr>
Date: Tue, 7 Jul 2020 10:53:58 +0200
Subject: [PATCH] Fix profiling with tasks. move operator skip test out
op_apply decorator to skip profiling as well as computation
---
hysop/core/graph/computational_graph.py | 43 +++++++++++++------------
hysop/core/graph/graph.py | 6 +---
2 files changed, 23 insertions(+), 26 deletions(-)
diff --git a/hysop/core/graph/computational_graph.py b/hysop/core/graph/computational_graph.py
index 6aa71de02..e8c678a9e 100644
--- a/hysop/core/graph/computational_graph.py
+++ b/hysop/core/graph/computational_graph.py
@@ -513,39 +513,39 @@ class ComputationalGraph(ComputationalGraphNode):
return ''
# isize = tasksize = name_size = type_size = 8
- from hysop import main_rank as rk
+ rk = self._profiler.get_comm().Get_rank()
maxlen = (None, None, None, None, None, None)
split_sep = (None, None, None, None, None, None)
newline_prefix = (None, None, None, None, None, None)
replace = ('--', '', '', '', '', '')
reduced_graph = self.reduced_graph
- operators = reduced_graph.vertex_properties['operators']
profnames, proftimes, profcalls = \
self._profiler.all_names[rk], self._profiler.all_times[rk], self._profiler.all_calls[rk]
ops = []
- for (i, vid) in enumerate(self.sorted_nodes):
- vertex = reduced_graph.vertex(vid)
- op = operators[vertex]
+ for (i, op) in enumerate(self.nodes):
opname = op.pretty_name.decode('utf-8')
optype = type(op).__name__
procdata, taskdata = [], []
for _ in self._profiler.summary.values():
if _.get_name().find(op.name) >= 0:
for __ in _.summary.values():
- procdata.append("{0} {1} {2:.5g} {3:.5g}".format(
- __.fname, __.nb_calls, __.total_time, __.total_time/__.nb_calls))
- nn = _.get_name() + '.' + __.fname
- idx = [k for k in enumerate(profnames) if k[1].find(nn) >= 0][0][0]
- taskvals = proftimes[:, idx][proftimes[:, idx] != 0.]
- taskdata.append("{0:.5g} {1:.5g} {2:.5g} {3}".format(
- taskvals[-1]/(len(taskvals)-1),
- npw.min(taskvals[:-1]), npw.max(taskvals[:-1]), len(taskvals)-1))
-
- strdata = (str(i), str(op.mpi_params.task_id), opname, optype, procdata[0], taskdata[0])
- ops += multiline_split(strdata, maxlen, split_sep, replace, newline_prefix)
- for p, t in zip(procdata[1:], taskdata[1:]):
- ops += multiline_split(('', '', '', '', p, t), maxlen,
- split_sep, replace, newline_prefix)
+ try:
+ procdata.append("{0} {1} {2:.5g} {3:.5g}".format(
+ __.fname, __.nb_calls, __.total_time, __.total_time/__.nb_calls))
+ nn = _.get_name() + '.' + __.fname
+ idx = [k for k in enumerate(profnames) if k[1].find(nn) >= 0][0][0]
+ taskvals = proftimes[:, idx][proftimes[:, idx] != 0.]
+ taskdata.append("{0:.5g} {1:.5g} {2:.5g} {3}".format(
+ taskvals[-1]/(len(taskvals)-1),
+ npw.min(taskvals[:-1]), npw.max(taskvals[:-1]), len(taskvals)-1))
+ except AttributeError:
+ pass
+ if procdata:
+ strdata = (str(i), str(op.mpi_params.task_id), opname, optype, procdata[0], taskdata[0])
+ ops += multiline_split(strdata, maxlen, split_sep, replace, newline_prefix)
+ for p, t in zip(procdata[1:], taskdata[1:]):
+ ops += multiline_split(('', '', '', '', p, t), maxlen,
+ split_sep, replace, newline_prefix)
isize = max(strlen(s[0]) for s in ops)
tasksize = max(max(strlen(s[1]) for s in ops), 6)
@@ -1011,8 +1011,9 @@ class ComputationalGraph(ComputationalGraphNode):
@ready
def apply(self, **kwds):
for node in self.nodes:
- dprint('{}.apply()'.format(node.name))
- node.apply(**kwds)
+ if not node.to_be_skipped(node, **kwds):
+ dprint('{}.apply()'.format(node.name))
+ node.apply(**kwds)
@debug
@ready
diff --git a/hysop/core/graph/graph.py b/hysop/core/graph/graph.py
index 6f42e0620..f5ca3f7d6 100644
--- a/hysop/core/graph/graph.py
+++ b/hysop/core/graph/graph.py
@@ -414,10 +414,6 @@ def op_apply(f):
kwds['dbg']('post '+msg, nostack=True)
return ret
else:
- if not op.to_be_skipped(*args, **kwds):
- return f(*args, **kwds)
- else:
- dprint("Skip {}".format(op.name))
- return
+ return f(*args, **kwds)
return ret
return apply
--
GitLab