From 673c4ca06a8f2c0eadcbe7c08215d4f2693df133 Mon Sep 17 00:00:00 2001
From: Eric Dalissier <Eric.Dalissier@imag.fr>
Date: Thu, 31 Jan 2013 16:55:33 +0000
Subject: [PATCH] Add of datatype on Up and Down direction change of tabSort
(table which choose the send and reciev proc)
---
HySoP/hysop/domain/topology.py | 15 +++-
HySoP/hysop/tools/cpu_data_transfer.py | 111 ++++++++++++++++++-------
2 files changed, 94 insertions(+), 32 deletions(-)
diff --git a/HySoP/hysop/domain/topology.py b/HySoP/hysop/domain/topology.py
index 12fd26e3c..e7e5011a5 100644
--- a/HySoP/hysop/domain/topology.py
+++ b/HySoP/hysop/domain/topology.py
@@ -4,6 +4,7 @@ MPI Topologies
"""
import mpi4py.MPI as MPI
from ..constants import *
+import sys
class CartesianTopology(object):
@@ -26,7 +27,6 @@ class CartesianTopology(object):
if(dims is None):
assert(dim is not None)
self.dims = np.asarray(MPI.Compute_dims(nbprocs, dim), dtype=PARMES_INTEGER)
- print 'dims = ' , self.dims
# Topology dimension
self.dim = dim
else:
@@ -44,6 +44,8 @@ class CartesianTopology(object):
self.size = self.topo.Get_size()
# Rank of the current process, in the new topology
self.rank = self.topo.Get_rank()
+ if (self.rank == 0) :
+ print 'dims = ' , self.dims
# Coordinates of the current process
self.coords = self.topo.Get_coords(self.rank)
# Neighbours
@@ -55,6 +57,15 @@ class CartesianTopology(object):
self.south, self.north = self.topo.Shift(ZDIR, 1)
self.tabSort = np.array([[self.up,0] , [self.down,1] , [self.east,2], [self.west,3],
[self.north,4],[self.south,5]])
+ tabSort1 = self.tabSort[self.tabSort[:,0]<=self.rank]
+ tabSort2 = self.tabSort[self.tabSort[:,0]>self.rank]
+ if (np.asarray(tabSort1[:,0].shape)>0):
+ tabSort1 = tabSort1[tabSort1[:,0].argsort()]
+ if (np.asarray(tabSort2[:,0].shape)>0):
+# print 'tabSort2', self.rank, tabSort2 , tabSort2[:,0].shape
+ tabSort2 = tabSort2[np.invert(tabSort2[:,0].argsort() )]
+ tabSort2 = tabSort2[tabSort2[:,0].argsort()]
+ self.tabSort = np.concatenate((tabSort1, tabSort2))
self.tabSort=self.tabSort[self.tabSort[:,0].argsort()]
# ghost layer (default = 0)
if(ghosts is None):
@@ -145,6 +156,8 @@ class LocalMesh(object):
coord_start = self.start * self.size + dom_origin - ghosts* self.size
self.origin = coord_start
coord_end = (self.end +1) * self.size + dom_origin - ghosts* self.size
+# print 'Print TOPO'
+# print 'resolution', self.resolution , 'shape', (coord_end-coord_start)/self.size, 'coord_end',coord_end, 'coord_start',coord_start
if self.start.size == 3:
self.coords = np.ogrid[coord_start[0]:coord_end[0]:self.size[0],
coord_start[1]:coord_end[1]:self.size[1],
diff --git a/HySoP/hysop/tools/cpu_data_transfer.py b/HySoP/hysop/tools/cpu_data_transfer.py
index 3c79a29b9..2952c9801 100644
--- a/HySoP/hysop/tools/cpu_data_transfer.py
+++ b/HySoP/hysop/tools/cpu_data_transfer.py
@@ -22,7 +22,7 @@ class Synchronize(object):
self.topo = topology
self.total_time = 0.
## problem le topology.mesh.resolution peut etre different du shape du field rentre
-# self.size= topology.mesh.resolution
+ self.size= topology.mesh.resolution
def apply(self, *listFields):
@@ -46,8 +46,9 @@ class Synchronize(object):
self.comm = MPI.COMM_WORLD
for f in listFields:
- self.size= f.data[0].shape
+# self.size= f.data[0].shape
# print 'size array:' ,self.size, 'size topo :', self.topo.mesh.resolution
+# print 'rank', self.topo.rank, 'tab',self.topo.tabSort
for i in xrange(self.topo.tabSort[:,0].shape[0]) :
# print 'rang, destinataire, orientation', self.topo.rank, self.topo.tabSort[i,0], self.topo.tabSort[i,1]
if ( self.topo.rank == self.topo.tabSort[i,0]):
@@ -87,39 +88,63 @@ class Synchronize(object):
# if(self.topo.tabSort[i,1] == 'up'):
if(self.topo.tabSort[i,1] == 0):
if( self.topo.rank < self.topo.tabSort[i,0]) :
+ print 'UP',self.topo.rank,'Send to',self.topo.tabSort[i,0]
self.UPsend(self.topo.tabSort[i,0], f)
+ print 'ok'
else :
+ print 'UP', self.topo.rank,'Recv from',self.topo.tabSort[i,0]
self.UPrecv(self.topo.tabSort[i,0], f)
+ print 'ok'
# if(self.topo.tabSort[i,1] == 'down'):
if(self.topo.tabSort[i,1] == 1):
if( self.topo.rank < self.topo.tabSort[i,0]) :
+ print 'DOWN',self.topo.rank,'Send to',self.topo.tabSort[i,0]
self.DOWNsend(self.topo.tabSort[i,0], f)
+ print 'ok'
else :
+ print 'DOWN', self.topo.rank,'Recv from',self.topo.tabSort[i,0]
self.DOWNrecv(self.topo.tabSort[i,0], f)
+ print 'ok'
# if(self.topo.tabSort[i,1] == 'east'):
if(self.topo.tabSort[i,1] == 2):
if( self.topo.rank < self.topo.tabSort[i,0]) :
+ print 'EST',self.topo.rank,'Send to',self.topo.tabSort[i,0]
self.EASTsend(self.topo.tabSort[i,0], f)
+ print 'ok'
else :
+ print 'EST', self.topo.rank,'Recv from',self.topo.tabSort[i,0]
self.EASTrecv(self.topo.tabSort[i,0], f)
+ print 'ok'
# if(self.topo.tabSort[i,1] == 'west'):
if(self.topo.tabSort[i,1] == 3):
if( self.topo.rank < self.topo.tabSort[i,0]) :
+ print 'WEST',self.topo.rank,'Send to',self.topo.tabSort[i,0]
self.WESTsend(self.topo.tabSort[i,0], f)
+ print 'ok'
else :
+ print 'WEST', self.topo.rank,'Recv from',self.topo.tabSort[i,0]
self.WESTrecv(self.topo.tabSort[i,0], f)
+ print 'ok'
# if(self.topo.tabSort[i,1] == 'north'):
if(self.topo.tabSort[i,1] == 4):
if( self.topo.rank < self.topo.tabSort[i,0]) :
+ print 'NORTH',self.topo.rank,'Send to',self.topo.tabSort[i,0]
self.NORTHsend(self.topo.tabSort[i,0], f)
+ print 'ok'
else :
+ print 'NORTH', self.topo.rank,'Recv from',self.topo.tabSort[i,0]
self.NORTHrecv(self.topo.tabSort[i,0], f)
+ print 'ok'
# if(self.topo.tabSort[i,1] == 'south'):
if(self.topo.tabSort[i,1] == 5):
if( self.topo.rank < self.topo.tabSort[i,0]) :
+ print 'SOUTH',self.topo.rank,'Send to',self.topo.tabSort[i,0]
self.SOUTHsend(self.topo.tabSort[i,0], f)
+ print 'ok'
else :
+ print 'SOUTH', self.topo.rank,'Recv from',self.topo.tabSort[i,0]
self.SOUTHrecv(self.topo.tabSort[i,0], f)
+ print 'ok'
self.compute_time = time.time() - self.compute_time
self.total_time += self.compute_time
@@ -128,23 +153,36 @@ class Synchronize(object):
def UPsend(self, proc, listFields):
for i in xrange(self.topo.dim) :
- self.comm.Send(listFields[i][ \
- self.topo.mesh.resolution[0]-2*self.topo.ghosts[0]:self.topo.mesh.resolution[0]-self.topo.ghosts[0],:,:]
- , dest=proc, tag=77)
- data = listFields[i][ \
- self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:]
- self.comm.Recv(data, source=proc, tag=77)
- listFields[i][self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] = data
+ subsizes = np.asarray(listFields[i][ \
+ self.topo.mesh.resolution[0]-2*self.topo.ghosts[0]:self.topo.mesh.resolution[0]-self.topo.ghosts[0],:,:].shape)
+ startsSend = np.asarray([self.topo.mesh.resolution[0]-2*self.topo.ghosts[0],0,0])
+ UPtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F)
+ UPtypeSend.Commit()
+ self.comm.Send([listFields[i], UPtypeSend ], dest=proc, tag=77)
+ startsRecv = [self.topo.mesh.resolution[0]-self.topo.ghosts[0],0,0]
+ UPtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F)
+ UPtypeRecv.Commit()
+ data = listFields[i]
+ self.comm.Recv([data, UPtypeRecv] , source=proc, tag=77)
+ listFields[i][
+ self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:]\
+ = data[self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:]
+
def DOWNsend(self, proc, listFields):
for i in xrange(self.topo.dim) :
- self.comm.Send(listFields[i][ \
- self.topo.ghosts[0]:2*self.topo.ghosts[0],:,:]
- , dest=proc, tag=77)
- data = listFields[i][ \
- 0:self.topo.ghosts[0],:,:]
- self.comm.Recv(data, source=proc, tag=77)
- listFields[i][0:self.topo.ghosts[0],:,:] = data
+ subsizes = np.asarray(listFields[i][ \
+ self.topo.ghosts[0]:2*self.topo.ghosts[0],:,:].shape)
+ startsSend = np.asarray([self.topo.ghosts[0],0,0])
+ DOWNtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F)
+ DOWNtypeSend.Commit()
+ self.comm.Send([ listFields[i], DOWNtypeSend ], dest=proc, tag=77)
+ startsRecv = [0,0,0]
+ DOWNtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F)
+ DOWNtypeRecv.Commit()
+ data = listFields[i]
+ self.comm.Recv([data, DOWNtypeRecv], source=proc, tag=77)
+ listFields[i][0:self.topo.ghosts[0],:,:] = data[0:self.topo.ghosts[0],:,:]
def EASTsend(self, proc, listFields):
for i in xrange(self.topo.dim) :
@@ -213,25 +251,36 @@ class Synchronize(object):
def UPrecv(self, proc, listFields):
for i in xrange(self.topo.dim) :
- data = listFields[i][ \
+ subsizes = np.asarray(listFields[i][ \
+ self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:].shape)
+ startsRecv = np.asarray([self.topo.mesh.resolution[0]-self.topo.ghosts[0],0,0])
+ UPtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F)
+ UPtypeRecv.Commit()
+ data = listFields[i]
+ self.comm.Recv([data, UPtypeRecv], source=proc, tag=77)
+ listFields[i][self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] = data[ \
self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:]
- self.comm.Recv(data, source=proc, tag=77)
- listFields[i][ \
- self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] =data
- self.comm.Send(listFields[i][
- self.topo.mesh.resolution[0]-2*self.topo.ghosts[0]:self.topo.mesh.resolution[0]-self.topo.ghosts[0],:,:]
- , dest=proc, tag=77)
+ startsSend = [self.topo.mesh.resolution[0]-2*self.topo.ghosts[0],0,0]
+ UPtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F)
+ UPtypeSend.Commit()
+ self.comm.Send([listFields[i], UPtypeSend], dest=proc, tag=77)
+
def DOWNrecv(self, proc, listFields):
for i in xrange(self.topo.dim) :
- data = listFields[i][ \
- 0:self.topo.ghosts[0],:,:]
- self.comm.Recv(data, source=proc, tag=77)
- listFields[i][ \
- 0:self.topo.ghosts[0],:,:] =data
- self.comm.Send(listFields[i][ \
- self.topo.ghosts[0]:2*self.topo.ghosts[0],:,:]
- , dest=proc, tag=77)
+ subsizes = np.asarray(listFields[i][ \
+ 0:self.topo.ghosts[0],:,:].shape)
+ startsRecv = np.asarray([0,0,0])
+ DOWNtypeRecv= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F)
+ DOWNtypeRecv.Commit()
+ data=listFields[i]
+ self.comm.Recv([data, DOWNtypeRecv], source = proc, tag =77)
+ listFields[i][0:self.topo.ghosts[0],:,:] = data[0:self.topo.ghosts[0],:,:]
+ startsSend = [self.topo.ghosts[0],0,0]
+ DOWNtypeSend= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F)
+ DOWNtypeSend.Commit()
+ self.comm.Send([listFields[i], DOWNtypeSend], dest = proc, tag =77)
+
def EASTrecv(self, proc, listFields):
for i in xrange(self.topo.dim) :
--
GitLab