Skip to content
Snippets Groups Projects
Commit 4fea5a66 authored by Franck Pérignon's avatar Franck Pérignon
Browse files

Add parent comm in Bridges to deal with (future) intra/inter comm context

parent 0727d4f2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
HySoP/hysop/mpi/topology.py
+ 61
− 36
View file @ 4fea5a66
...@@ -25,7 +25,7 @@ To get more details try : ...@@ -25,7 +25,7 @@ To get more details try :
from parmepy.constants import ORDER, np, PARMES_INDEX, debug, PARMES_INTEGER from parmepy.constants import ORDER, np, PARMES_INDEX, debug, PARMES_INTEGER
from parmepy.mpi.mesh import SubMesh from parmepy.mpi.mesh import SubMesh
from itertools import count from itertools import count
from parmepy.mpi.main_var import main_comm, main_size, main_rank, MPI from parmepy.mpi.main_var import main_comm, MPI
class Cartesian(object): class Cartesian(object):
...@@ -74,6 +74,8 @@ class Cartesian(object): ...@@ -74,6 +74,8 @@ class Cartesian(object):
# to 0 if shape[:] = 1 # to 0 if shape[:] = 1
## (Source) Communicator used to build the topology ## (Source) Communicator used to build the topology
self._comm_origin = comm self._comm_origin = comm
## number of process in comm_origin
self._origin_size = self._comm_origin.Get_size()
## True if everything has been properly set for this topology ## True if everything has been properly set for this topology
self.isUpToDate = False self.isUpToDate = False
## Grid of mpi process layout ## Grid of mpi process layout
...@@ -82,7 +84,7 @@ class Cartesian(object): ...@@ -82,7 +84,7 @@ class Cartesian(object):
if shape is None: if shape is None:
# If shape is not provided, computation of the "optimal" processus # If shape is not provided, computation of the "optimal" processus
# distribution for each direction of the grid topology. # distribution for each direction of the grid topology.
shape = np.asarray(MPI.Compute_dims(main_size, self.dim), shape = np.asarray(MPI.Compute_dims(self._origin_size, self.dim),
dtype=PARMES_INDEX) dtype=PARMES_INDEX)
if cutdir is not None: if cutdir is not None:
...@@ -113,7 +115,7 @@ class Cartesian(object): ...@@ -113,7 +115,7 @@ class Cartesian(object):
self.shape = shape self.shape = shape
# Special care for the 1 process case: # Special care for the 1 process case:
if main_size == 1: if self._origin_size == 1:
self.dim = 1 self.dim = 1
self.cutdir[-1] = True self.cutdir[-1] = True
...@@ -210,6 +212,12 @@ class Cartesian(object): ...@@ -210,6 +212,12 @@ class Cartesian(object):
# print "A similar topology already exist in the domain's list.\ # print "A similar topology already exist in the domain's list.\
# You'd rather destroy this one and use topo of id", isNew # You'd rather destroy this one and use topo of id", isNew
def getParent(self):
"""
returns the communicator used to build this topology
"""
return self._comm_origin
@debug @debug
def setUp(self): def setUp(self):
"""Topology set up. """Topology set up.
...@@ -361,8 +369,8 @@ class Cartesian(object): ...@@ -361,8 +369,8 @@ class Cartesian(object):
s += str(self.dim) + 'D cartesian Topology of size ' \ s += str(self.dim) + 'D cartesian Topology of size ' \
+ str(self.shape) + '\n' + str(self.shape) + '\n'
s += 'Process of coordinates ' + str(self.proc_coords[:]) s += 'Process of coordinates ' + str(self.proc_coords[:])
s += ' and of ranks (topo/world) ' + str(self.rank) + '/' s += ' and of ranks (topo/origin) ' + str(self.rank) + '/'
s += str(main_rank) + '.\n' s += str(self._comm_origin.Get_rank()) + '.\n'
s += 'Neighbours coordinates : \n' s += 'Neighbours coordinates : \n'
s += str(self.neighbours) + '\n' s += str(self.neighbours) + '\n'
s += 'Ghost layer : ' + str(self.ghosts[:]) + '\n' s += 'Ghost layer : ' + str(self.ghosts[:]) + '\n'
...@@ -434,11 +442,12 @@ class Bridge(object): ...@@ -434,11 +442,12 @@ class Bridge(object):
""" """
@debug @debug
def __init__(self, topoFrom, topoTo): def __init__(self, topoFrom, topoTo, parentComm=None):
""" """
Bridge constructor. Bridge constructor.
@param source topology @param source topology
@param targeted topology @param targeted topology
@param parent communicator, if None, use topoFrom.getParent()
""" """
## Source topology ## Source topology
self.topoFrom = topoFrom self.topoFrom = topoFrom
...@@ -475,9 +484,25 @@ class Bridge(object): ...@@ -475,9 +484,25 @@ class Bridge(object):
self.hasLocalInter = True self.hasLocalInter = True
## Parent communicator
## Todo : define some proper conditions for compatibility
## between topoFrom, topoTo and parent:
## - same size
## - same domain
## - common processus ...
## At the time we check that both topo have
## the same comm_origin.
if parentComm is None:
self._parentComm = self.topoFrom.getParent()
else:
self._parentComm = parentComm
self._parentRank = self._parentComm.Get_rank()
self._parentSize = self._parentComm.Get_size()
if self.topoFrom == self.topoTo: if self.topoFrom == self.topoTo:
return return
assert(self.topoTo.getParent() is self._parentComm)
# Both topologies must have the # Both topologies must have the
# same number of mpi processes. # same number of mpi processes.
assert topoFrom.size == topoTo.size assert topoFrom.size == topoTo.size
...@@ -489,29 +514,32 @@ class Bridge(object): ...@@ -489,29 +514,32 @@ class Bridge(object):
start1 = topoFrom.mesh.global_start start1 = topoFrom.mesh.global_start
end1 = topoFrom.mesh.global_end end1 = topoFrom.mesh.global_end
iglob1 = np.zeros((size1, dom.dimension * 2), dtype=np.int32) iglob1 = np.zeros((size1, dom.dimension * 2), dtype=np.int32)
iglob1[main_rank, 0::2] = start1 iglob1[self._parentRank, 0::2] = start1
iglob1[main_rank, 1::2] = end1 iglob1[self._parentRank, 1::2] = end1
size2 = topoTo.size size2 = topoTo.size
start2 = topoTo.mesh.global_start start2 = topoTo.mesh.global_start
end2 = topoTo.mesh.global_end end2 = topoTo.mesh.global_end
iglob2 = np.zeros((size2, dom.dimension * 2), dtype=np.int32) iglob2 = np.zeros((size2, dom.dimension * 2), dtype=np.int32)
iglob2[main_rank, 0::2] = start2 iglob2[self._parentRank, 0::2] = start2
iglob2[main_rank, 1::2] = end2 iglob2[self._parentRank, 1::2] = end2
# 2 Distribute these indices. # 2 Distribute these indices.
main_comm.Allgather([iglob1[main_rank, :], MPI.INT], [iglob1, MPI.INT]) self._parentComm.Allgather([iglob1[self._parentRank, :], MPI.INT],
main_comm.Allgather([iglob2[main_rank, :], MPI.INT], [iglob2, MPI.INT]) [iglob1, MPI.INT])
self._parentComm.Allgather([iglob2[self._parentRank, :], MPI.INT],
[iglob2, MPI.INT])
# Connectivity : # Connectivity :
sTo = np.zeros((0, 1 + dom.dimension * 2), dtype=np.int) sTo = np.zeros((0, 1 + dom.dimension * 2), dtype=np.int)
indexFrom = [] indexFrom = []
for dim in range(dom.dimension): for dim in range(dom.dimension):
indexFrom.append(range(iglob1[main_rank, dim * 2], indexFrom.append(range(iglob1[self._parentRank, dim * 2],
iglob1[main_rank, dim * 2 + 1] + 1)) iglob1[self._parentRank, dim * 2 + 1] + 1))
line = np.zeros((1 + 2 * dom.dimension), dtype=np.int32) line = np.zeros((1 + 2 * dom.dimension), dtype=np.int32)
listRanks = [i for i in range(main_size) if i != main_rank] listRanks = [i for i in range(self._parentSize)
if i != self._parentRank]
for i in listRanks: for i in listRanks:
line[0] = i line[0] = i
...@@ -533,8 +561,8 @@ class Bridge(object): ...@@ -533,8 +561,8 @@ class Bridge(object):
localFrom = np.zeros((2 * dom.dimension), dtype=np.int32) localFrom = np.zeros((2 * dom.dimension), dtype=np.int32)
for dim in range(dom.dimension): for dim in range(dom.dimension):
indexTo = range(iglob2[main_rank, dim * 2], indexTo = range(iglob2[self._parentRank, dim * 2],
iglob2[main_rank, dim * 2 + 1] + 1) iglob2[self._parentRank, dim * 2 + 1] + 1)
interRow = [k for k in indexFrom[dim] if k in indexTo] interRow = [k for k in indexFrom[dim] if k in indexTo]
interRow.sort() interRow.sort()
...@@ -567,15 +595,15 @@ class Bridge(object): ...@@ -567,15 +595,15 @@ class Bridge(object):
dom.dimension * 2), dtype=np.int) dom.dimension * 2), dtype=np.int)
globalConnectivity[:, 1:] = sTo.copy(order='A') globalConnectivity[:, 1:] = sTo.copy(order='A')
globalConnectivity[:, 0] = main_rank globalConnectivity[:, 0] = self._parentRank
if main_rank == 0: if self._parentRank == 0:
for i in range(1, main_size): for i in range(1, self._parentSize):
temp = main_comm.recv(source=i) temp = self._parentComm.recv(source=i)
globalConnectivity = np.vstack((globalConnectivity, temp)) globalConnectivity = np.vstack((globalConnectivity, temp))
else: else:
main_comm.ssend(globalConnectivity, dest=0) self._parentComm.ssend(globalConnectivity, dest=0)
## recvFrom[:,0] rank of the sending process in main_comm ## recvFrom[:,0] rank of the sending process in main_comm
## recvFrom[:,1:] local indices (start,end, ...) ## recvFrom[:,1:] local indices (start,end, ...)
...@@ -586,8 +614,8 @@ class Bridge(object): ...@@ -586,8 +614,8 @@ class Bridge(object):
## an array that will be saved at positions of indices 2:5 ## an array that will be saved at positions of indices 2:5
## in the first dir and 1:3 in second dir ## in the first dir and 1:3 in second dir
rFrom = np.empty((), dtype=np.int32) rFrom = np.empty((), dtype=np.int32)
if main_rank == 0: if self._parentRank == 0:
for fromRank in range(main_size): for fromRank in range(self._parentSize):
cond1 = globalConnectivity[:, 1] == fromRank cond1 = globalConnectivity[:, 1] == fromRank
cond2 = globalConnectivity[:, 0] != fromRank cond2 = globalConnectivity[:, 0] != fromRank
cond = cond1 & cond2 cond = cond1 & cond2
...@@ -595,13 +623,13 @@ class Bridge(object): ...@@ -595,13 +623,13 @@ class Bridge(object):
cond1 = np.ones((sol.shape[1]), dtype=np.bool) cond1 = np.ones((sol.shape[1]), dtype=np.bool)
cond1[1] = False cond1[1] = False
sol = np.compress(cond1, sol, axis=1) sol = np.compress(cond1, sol, axis=1)
if(fromRank != main_rank): if(fromRank != self._parentRank):
main_comm.ssend(sol, dest=fromRank) self._parentComm.ssend(sol, dest=fromRank)
else: else:
rFrom = sol.copy() rFrom = sol.copy()
else: else:
rFrom = main_comm.recv(source=0) rFrom = self._parentComm.recv(source=0)
# Final setup for recvFrom and sendTo : shift from # Final setup for recvFrom and sendTo : shift from
# global indices to local ones. # global indices to local ones.
...@@ -636,14 +664,11 @@ class Bridge(object): ...@@ -636,14 +664,11 @@ class Bridge(object):
""" TopologyBridge info display """ """ TopologyBridge info display """
s = '======== Bridge from topology ' + str(self.topoFrom.getId()) s = '======== Bridge from topology ' + str(self.topoFrom.getId())
s += ' to topology ' + str(self.topoTo.getId()) + ' ========\n' s += ' to topology ' + str(self.topoTo.getId()) + ' ========\n'
s += '[' + str(main_rank) + '] sendTo :' + str(self.sendTo) + '\n' s += '[' + str(self._parentRank) + '] sendTo :'
s += '[' + str(main_rank) + '] recvFrom :' + str(self.recvFrom) + '\n' s += str(self.sendTo) + '\n'
s += '[' + str(main_rank) + '] iTo :' + str(self.ito) + '\n' s += '[' + str(self._parentRank) + '] recvFrom :'
s += '[' + str(main_rank) + '] iFrom :' + str(self.ifrom) + '\n' s += str(self.recvFrom) + '\n'
s += '[' + str(self._parentRank) + '] iTo :' + str(self.ito) + '\n'
s += '[' + str(self._parentRank) + '] iFrom :' + str(self.ifrom) + '\n'
s += '=================================\n' s += '=================================\n'
return s return s
if __name__ == "__main__":
print "This module defines the following classes:"
print "- Cartesian Topology: ", Cartesian.__doc__
print "- Bridge between topologies: ", Bridge.__doc__
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment