From 4fea5a665a1a7f3565112c78bf399a5f433c4138 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franck=20P=C3=A9rignon?= <franck.perignon@imag.fr>
Date: Wed, 5 Feb 2014 12:49:02 +0000
Subject: [PATCH] Add parent comm in Bridges to deal with (future) intra/inter
comm context
---
HySoP/hysop/mpi/topology.py | 97 +++++++++++++++++++++++--------------
1 file changed, 61 insertions(+), 36 deletions(-)
diff --git a/HySoP/hysop/mpi/topology.py b/HySoP/hysop/mpi/topology.py
index 3386a2705..82be9a6a5 100644
--- a/HySoP/hysop/mpi/topology.py
+++ b/HySoP/hysop/mpi/topology.py
@@ -25,7 +25,7 @@ To get more details try :
from parmepy.constants import ORDER, np, PARMES_INDEX, debug, PARMES_INTEGER
from parmepy.mpi.mesh import SubMesh
from itertools import count
-from parmepy.mpi.main_var import main_comm, main_size, main_rank, MPI
+from parmepy.mpi.main_var import main_comm, MPI
class Cartesian(object):
@@ -74,6 +74,8 @@ class Cartesian(object):
# to 0 if shape[:] = 1
## (Source) Communicator used to build the topology
self._comm_origin = comm
+ ## number of process in comm_origin
+ self._origin_size = self._comm_origin.Get_size()
## True if everything has been properly set for this topology
self.isUpToDate = False
## Grid of mpi process layout
@@ -82,7 +84,7 @@ class Cartesian(object):
if shape is None:
# If shape is not provided, computation of the "optimal" processus
# distribution for each direction of the grid topology.
- shape = np.asarray(MPI.Compute_dims(main_size, self.dim),
+ shape = np.asarray(MPI.Compute_dims(self._origin_size, self.dim),
dtype=PARMES_INDEX)
if cutdir is not None:
@@ -113,7 +115,7 @@ class Cartesian(object):
self.shape = shape
# Special care for the 1 process case:
- if main_size == 1:
+ if self._origin_size == 1:
self.dim = 1
self.cutdir[-1] = True
@@ -210,6 +212,12 @@ class Cartesian(object):
# print "A similar topology already exist in the domain's list.\
# You'd rather destroy this one and use topo of id", isNew
+ def getParent(self):
+ """
+ returns the communicator used to build this topology
+ """
+ return self._comm_origin
+
@debug
def setUp(self):
"""Topology set up.
@@ -361,8 +369,8 @@ class Cartesian(object):
s += str(self.dim) + 'D cartesian Topology of size ' \
+ str(self.shape) + '\n'
s += 'Process of coordinates ' + str(self.proc_coords[:])
- s += ' and of ranks (topo/world) ' + str(self.rank) + '/'
- s += str(main_rank) + '.\n'
+ s += ' and of ranks (topo/origin) ' + str(self.rank) + '/'
+ s += str(self._comm_origin.Get_rank()) + '.\n'
s += 'Neighbours coordinates : \n'
s += str(self.neighbours) + '\n'
s += 'Ghost layer : ' + str(self.ghosts[:]) + '\n'
@@ -434,11 +442,12 @@ class Bridge(object):
"""
@debug
- def __init__(self, topoFrom, topoTo):
+ def __init__(self, topoFrom, topoTo, parentComm=None):
"""
Bridge constructor.
@param source topology
@param targeted topology
+ @param parent communicator, if None, use topoFrom.getParent()
"""
## Source topology
self.topoFrom = topoFrom
@@ -475,9 +484,25 @@ class Bridge(object):
self.hasLocalInter = True
+ ## Parent communicator
+ ## Todo : define some proper conditions for compatibility
+ ## between topoFrom, topoTo and parent:
+ ## - same size
+ ## - same domain
+ ## - common processus ...
+ ## At the time we check that both topo have
+ ## the same comm_origin.
+ if parentComm is None:
+ self._parentComm = self.topoFrom.getParent()
+ else:
+ self._parentComm = parentComm
+ self._parentRank = self._parentComm.Get_rank()
+ self._parentSize = self._parentComm.Get_size()
if self.topoFrom == self.topoTo:
return
+ assert(self.topoTo.getParent() is self._parentComm)
+
# Both topologies must have the
# same number of mpi processes.
assert topoFrom.size == topoTo.size
@@ -489,29 +514,32 @@ class Bridge(object):
start1 = topoFrom.mesh.global_start
end1 = topoFrom.mesh.global_end
iglob1 = np.zeros((size1, dom.dimension * 2), dtype=np.int32)
- iglob1[main_rank, 0::2] = start1
- iglob1[main_rank, 1::2] = end1
+ iglob1[self._parentRank, 0::2] = start1
+ iglob1[self._parentRank, 1::2] = end1
size2 = topoTo.size
start2 = topoTo.mesh.global_start
end2 = topoTo.mesh.global_end
iglob2 = np.zeros((size2, dom.dimension * 2), dtype=np.int32)
- iglob2[main_rank, 0::2] = start2
- iglob2[main_rank, 1::2] = end2
+ iglob2[self._parentRank, 0::2] = start2
+ iglob2[self._parentRank, 1::2] = end2
# 2 Distribute these indices.
- main_comm.Allgather([iglob1[main_rank, :], MPI.INT], [iglob1, MPI.INT])
- main_comm.Allgather([iglob2[main_rank, :], MPI.INT], [iglob2, MPI.INT])
+ self._parentComm.Allgather([iglob1[self._parentRank, :], MPI.INT],
+ [iglob1, MPI.INT])
+ self._parentComm.Allgather([iglob2[self._parentRank, :], MPI.INT],
+ [iglob2, MPI.INT])
# Connectivity :
sTo = np.zeros((0, 1 + dom.dimension * 2), dtype=np.int)
indexFrom = []
for dim in range(dom.dimension):
- indexFrom.append(range(iglob1[main_rank, dim * 2],
- iglob1[main_rank, dim * 2 + 1] + 1))
+ indexFrom.append(range(iglob1[self._parentRank, dim * 2],
+ iglob1[self._parentRank, dim * 2 + 1] + 1))
line = np.zeros((1 + 2 * dom.dimension), dtype=np.int32)
- listRanks = [i for i in range(main_size) if i != main_rank]
+ listRanks = [i for i in range(self._parentSize)
+ if i != self._parentRank]
for i in listRanks:
line[0] = i
@@ -533,8 +561,8 @@ class Bridge(object):
localFrom = np.zeros((2 * dom.dimension), dtype=np.int32)
for dim in range(dom.dimension):
- indexTo = range(iglob2[main_rank, dim * 2],
- iglob2[main_rank, dim * 2 + 1] + 1)
+ indexTo = range(iglob2[self._parentRank, dim * 2],
+ iglob2[self._parentRank, dim * 2 + 1] + 1)
interRow = [k for k in indexFrom[dim] if k in indexTo]
interRow.sort()
@@ -567,15 +595,15 @@ class Bridge(object):
dom.dimension * 2), dtype=np.int)
globalConnectivity[:, 1:] = sTo.copy(order='A')
- globalConnectivity[:, 0] = main_rank
+ globalConnectivity[:, 0] = self._parentRank
- if main_rank == 0:
- for i in range(1, main_size):
- temp = main_comm.recv(source=i)
+ if self._parentRank == 0:
+ for i in range(1, self._parentSize):
+ temp = self._parentComm.recv(source=i)
globalConnectivity = np.vstack((globalConnectivity, temp))
else:
- main_comm.ssend(globalConnectivity, dest=0)
+ self._parentComm.ssend(globalConnectivity, dest=0)
## recvFrom[:,0] rank of the sending process in main_comm
## recvFrom[:,1:] local indices (start,end, ...)
@@ -586,8 +614,8 @@ class Bridge(object):
## an array that will be saved at positions of indices 2:5
## in the first dir and 1:3 in second dir
rFrom = np.empty((), dtype=np.int32)
- if main_rank == 0:
- for fromRank in range(main_size):
+ if self._parentRank == 0:
+ for fromRank in range(self._parentSize):
cond1 = globalConnectivity[:, 1] == fromRank
cond2 = globalConnectivity[:, 0] != fromRank
cond = cond1 & cond2
@@ -595,13 +623,13 @@ class Bridge(object):
cond1 = np.ones((sol.shape[1]), dtype=np.bool)
cond1[1] = False
sol = np.compress(cond1, sol, axis=1)
- if(fromRank != main_rank):
- main_comm.ssend(sol, dest=fromRank)
+ if(fromRank != self._parentRank):
+ self._parentComm.ssend(sol, dest=fromRank)
else:
rFrom = sol.copy()
else:
- rFrom = main_comm.recv(source=0)
+ rFrom = self._parentComm.recv(source=0)
# Final setup for recvFrom and sendTo : shift from
# global indices to local ones.
@@ -636,14 +664,11 @@ class Bridge(object):
""" TopologyBridge info display """
s = '======== Bridge from topology ' + str(self.topoFrom.getId())
s += ' to topology ' + str(self.topoTo.getId()) + ' ========\n'
- s += '[' + str(main_rank) + '] sendTo :' + str(self.sendTo) + '\n'
- s += '[' + str(main_rank) + '] recvFrom :' + str(self.recvFrom) + '\n'
- s += '[' + str(main_rank) + '] iTo :' + str(self.ito) + '\n'
- s += '[' + str(main_rank) + '] iFrom :' + str(self.ifrom) + '\n'
+ s += '[' + str(self._parentRank) + '] sendTo :'
+ s += str(self.sendTo) + '\n'
+ s += '[' + str(self._parentRank) + '] recvFrom :'
+ s += str(self.recvFrom) + '\n'
+ s += '[' + str(self._parentRank) + '] iTo :' + str(self.ito) + '\n'
+ s += '[' + str(self._parentRank) + '] iFrom :' + str(self.ifrom) + '\n'
s += '=================================\n'
return s
-
-if __name__ == "__main__":
- print "This module defines the following classes:"
- print "- Cartesian Topology: ", Cartesian.__doc__
- print "- Bridge between topologies: ", Bridge.__doc__
--
GitLab