From 93d6d788674f6c195156c5e4e1821f3985d9da49 Mon Sep 17 00:00:00 2001 From: Chloe Mimeau <Chloe.Mimeau@imag.fr> Date: Tue, 29 Jan 2013 10:18:54 +0000 Subject: [PATCH] --- HySoP/hysop/tools/cpu_data_transfer_S.py | 304 +++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 HySoP/hysop/tools/cpu_data_transfer_S.py diff --git a/HySoP/hysop/tools/cpu_data_transfer_S.py b/HySoP/hysop/tools/cpu_data_transfer_S.py new file mode 100644 index 000000000..14eee7ce5 --- /dev/null +++ b/HySoP/hysop/tools/cpu_data_transfer_S.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +""" +@package tools +Communicator MPI to synchronize ghosts of scalar fields +""" +from ..constants import * +import numpy as np +import mpi4py.MPI as MPI +import time + + +class SynchronizeS(object): + """ + Synchronization of ghosts values for scalar fields + """ + + def __init__(self, topology): + """ + Constructor. + @param topology : Local topology + """ + self.topo = topology + self.total_time = 0. + self.size= topology.mesh.resolution + + + def apply(self, *listFields): + """ + Application of data synchronization to a list of scalar Fields + @ list of scalar Fields + Make the synchronization with this law : + to synchronize P1 with P2 : + if P1.rank < P2.rank + P1 send its own and after recv ghost + P2 recv ghost and send its own + else + do the reciproque + This algorithm is performed orientation by orientation, + using a sorted table containing neighbours ranks and their orientation as follows : + """ +# Rappel de la correspondance direction <-> indice +# self.topo.tabSort = np.array([[self.up,0] , [self.down,1] , [self.east,2], [self.west,3], +# [self.north,4],[self.south,5]]) + self.compute_time = time.time() + self.comm = MPI.COMM_WORLD + + for f in listFields: + for i in xrange(self.topo.tabSort[:,0].shape[0]) : + if ( self.topo.rank == self.topo.tabSort[i,0]): + # UP DOWN + if(self.topo.tabSort[i,1] == 0): + f[self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] =\ + f[self.topo.ghosts[0]:2*self.topo.ghosts[0],:,:] + + if(self.topo.tabSort[i,1] == 1): + f[0:self.topo.ghosts[0],:,:] =\ + f[self.topo.mesh.resolution[0]-2*self.topo.ghosts[0]:self.topo.mesh.resolution[0]-self.topo.ghosts[0],:,:] + + # WEST EAST + if(self.topo.tabSort[i,1] == 2): + f[:,self.topo.mesh.resolution[1]-self.topo.ghosts[1]:self.topo.mesh.resolution[1],:] =\ + f[:,self.topo.ghosts[1]:2*self.topo.ghosts[1],:] + + if(self.topo.tabSort[i,1] == 3): + f[:,0:self.topo.ghosts[1],:] =\ + f[:,self.topo.mesh.resolution[1]-2*self.topo.ghosts[1]:self.topo.mesh.resolution[1]-self.topo.ghosts[1],:] + + # NORTH SOUTH + if(self.topo.tabSort[i,1] == 4): + f[0][:,:,self.topo.mesh.resolution[2]-self.topo.ghosts[2]:self.topo.mesh.resolution[2]] =\ + f[0][:,:,self.topo.ghosts[2]:2*self.topo.ghosts[2]] + + if(self.topo.tabSort[i,1] == 5): + f[:,:,0:self.topo.ghosts[2]] =\ + f[:,:,self.topo.mesh.resolution[2]-2*self.topo.ghosts[2]:self.topo.mesh.resolution[2]-self.topo.ghosts[2]] + else : +# if(self.topo.tabSort[i,1] == 'up'): + if(self.topo.tabSort[i,1] == 0): + if( self.topo.rank < self.topo.tabSort[i,0]) : + self.UPsend(self.topo.tabSort[i,0], f) + else : + self.UPrecv(self.topo.tabSort[i,0], f) +# if(self.topo.tabSort[i,1] == 'down'): + if(self.topo.tabSort[i,1] == 1): + if( self.topo.rank < self.topo.tabSort[i,0]) : + self.DOWNsend(self.topo.tabSort[i,0], f) + else : + self.DOWNrecv(self.topo.tabSort[i,0], f) +# if(self.topo.tabSort[i,1] == 'east'): + if(self.topo.tabSort[i,1] == 2): + if( self.topo.rank < self.topo.tabSort[i,0]) : + self.EASTsend(self.topo.tabSort[i,0], f) + else : + self.EASTrecv(self.topo.tabSort[i,0], f) +# if(self.topo.tabSort[i,1] == 'west'): + if(self.topo.tabSort[i,1] == 3): + if( self.topo.rank < self.topo.tabSort[i,0]) : + self.WESTsend(self.topo.tabSort[i,0], f) + else : + self.WESTrecv(self.topo.tabSort[i,0], f) +# if(self.topo.tabSort[i,1] == 'north'): + if(self.topo.tabSort[i,1] == 4): + if( self.topo.rank < self.topo.tabSort[i,0]) : + self.NORTHsend(self.topo.tabSort[i,0], f) + else : + self.NORTHrecv(self.topo.tabSort[i,0], f) +# if(self.topo.tabSort[i,1] == 'south'): + if(self.topo.tabSort[i,1] == 5): + if( self.topo.rank < self.topo.tabSort[i,0]) : + self.SOUTHsend(self.topo.tabSort[i,0], f) + else : + self.SOUTHrecv(self.topo.tabSort[i,0], f) + self.compute_time = time.time() - self.compute_time + self.total_time += self.compute_time + + return self.compute_time + + + def UPsend(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + self.topo.mesh.resolution[0]-2*self.topo.ghosts[0]:self.topo.mesh.resolution[0]-self.topo.ghosts[0],:,:].shape) + startsSend = np.asarray([self.topo.mesh.resolution[0]-2*self.topo.ghosts[0],0,0]) + UPtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + UPtypeSend.Commit() + self.comm.Send([ listFields, UPtypeSend ], dest=proc, tag=77) + startsRecv = [self.topo.mesh.resolution[0]-self.topo.ghosts[0],0,0] + UPtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + UPtypeRecv.Commit() + data = listFields + self.comm.Recv([data, UPtypeRecv] , source=proc, tag=77) + listFields[self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:]\ + = data[self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] + + def DOWNsend(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + self.topo.ghosts[0]:2*self.topo.ghosts[0],:,:].shape) + startsSend = np.asarray([self.topo.ghosts[0],0,0]) + DOWNtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + DOWNtypeSend.Commit() + self.comm.Send([ listFields, DOWNtypeSend ], dest=proc, tag=77) + startsRecv = [0,0,0] + DOWNtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + DOWNtypeRecv.Commit() + data = listFields + self.comm.Recv([data, DOWNtypeRecv], source=proc, tag=77) + listFields[0:self.topo.ghosts[0],:,:] = data[0:self.topo.ghosts[0],:,:] + + def EASTsend(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,self.topo.mesh.resolution[1]-2*self.topo.ghosts[1]:self.topo.mesh.resolution[1]-self.topo.ghosts[1],:].shape) + startsSend = np.asarray([0,self.topo.mesh.resolution[1]-2*self.topo.ghosts[1],0]) + EASTtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + EASTtypeSend.Commit() + self.comm.Send([ listFields, EASTtypeSend ], dest=proc, tag=33) + startsRecv = [0,self.topo.mesh.resolution[1]-self.topo.ghosts[1],0] + EASTtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + EASTtypeRecv.Commit() + data = listFields + self.comm.Recv([data, EASTtypeRecv] , source=proc, tag=33) + listFields[:,self.topo.mesh.resolution[1]-self.topo.ghosts[1]:self.topo.mesh.resolution[1] ,:]\ + = data[:,self.topo.mesh.resolution[1]-self.topo.ghosts[1]:self.topo.mesh.resolution[1],:] + + def WESTsend(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,self.topo.ghosts[1]:2*self.topo.ghosts[1],:].shape) + startsSend = np.asarray([0,self.topo.ghosts[1],0]) + WESTtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + WESTtypeSend.Commit() + self.comm.Send([ listFields, WESTtypeSend ], dest=proc, tag=33) + startsRecv = [0,0,0] + WESTtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + WESTtypeRecv.Commit() + data = listFields + self.comm.Recv([data, WESTtypeRecv], source=proc, tag=33) + listFields[:,0:self.topo.ghosts[1],:] = data[:,0:self.topo.ghosts[1],:] + + def NORTHsend(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,:,self.topo.mesh.resolution[2]-2*self.topo.ghosts[2]:self.topo.mesh.resolution[2]-self.topo.ghosts[2]].shape) + startsSend = np.asarray([0,0,self.topo.mesh.resolution[2]-2*self.topo.ghosts[2]]) + NORTHtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + NORTHtypeSend.Commit() + self.comm.Send([ listFields, NORTHtypeSend ], dest=proc, tag=55) + startsRecv = [0,0,self.topo.mesh.resolution[2]-self.topo.ghosts[2]] + NORTHtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + NORTHtypeRecv.Commit() + data = listFields + self.comm.Recv([data, NORTHtypeRecv] , source=proc, tag=55) + listFields[:,:, self.topo.mesh.resolution[2]-self.topo.ghosts[2]:self.topo.mesh.resolution[2]]\ + = data[:,:,self.topo.mesh.resolution[2]-self.topo.ghosts[2]:self.topo.mesh.resolution[2]] + + def SOUTHsend(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,:,self.topo.ghosts[2]:2*self.topo.ghosts[2]].shape) + startsSend = np.asarray([0,0,self.topo.ghosts[2]]) + SOUTHtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + SOUTHtypeSend.Commit() + self.comm.Send([ listFields, SOUTHtypeSend ], dest=proc, tag=55) + startsRecv = [0,0,0] + SOUTHtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + SOUTHtypeRecv.Commit() + data = listFields + self.comm.Recv([data, SOUTHtypeRecv], source=proc, tag=55) + listFields[:,:,0:self.topo.ghosts[2]] = data[:,:,0:self.topo.ghosts[2]] + + def UPrecv(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:].shape) + startsRecv = np.asarray([self.topo.mesh.resolution[0]-self.topo.ghosts[0],0,0]) + UPtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + UPtypeRecv.Commit() + data = listFields + self.comm.Recv([data, UPtypeRecv], source=proc, tag=77) + listFields[self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] = \ + data[self.topo.mesh.resolution[0]-self.topo.ghosts[0]:self.topo.mesh.resolution[0],:,:] + startsSend = [self.topo.mesh.resolution[0]-2*self.topo.ghosts[0],0,0] + UPtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + UPtypeSend.Commit() + self.comm.Send([listFields, UPtypeSend], dest=proc, tag=77) + + def DOWNrecv(self, proc, listFields): + subsizes = np.asarray(listFields[0:self.topo.ghosts[0],:,:].shape) + startsRecv = np.asarray([0,0,0]) + DOWNtypeRecv= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + DOWNtypeRecv.Commit() + data=listFields + self.comm.Recv([data, DOWNtypeRecv], source = proc, tag =77) + listFields[0:self.topo.ghosts[0],:,:] = data[0:self.topo.ghosts[0],:,:] + startsSend = [self.topo.ghosts[0],0,0] + DOWNtypeSend= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + DOWNtypeSend.Commit() + self.comm.Send([listFields, DOWNtypeSend], dest = proc, tag =77) + + def EASTrecv(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,self.topo.mesh.resolution[1]-self.topo.ghosts[1]:self.topo.mesh.resolution[1],:].shape) + startsRecv = np.asarray([0,self.topo.mesh.resolution[1]-self.topo.ghosts[1],0]) + EASTtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + EASTtypeRecv.Commit() + data = listFields + self.comm.Recv([data, EASTtypeRecv], source=proc, tag=33) + listFields[:,self.topo.mesh.resolution[1]-self.topo.ghosts[1]:self.topo.mesh.resolution[1],:] = \ + data[:,self.topo.mesh.resolution[1]-self.topo.ghosts[1]:self.topo.mesh.resolution[1],:] + startsSend = [0,self.topo.mesh.resolution[1]-2*self.topo.ghosts[1],0] + EASTtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + EASTtypeSend.Commit() + self.comm.Send([listFields, EASTtypeSend], dest=proc, tag=33) + + def WESTrecv(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,0:self.topo.ghosts[1],:].shape) + startsRecv = np.asarray([0,0,0]) + WESTtypeRecv= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + WESTtypeRecv.Commit() + data=listFields + self.comm.Recv([data, WESTtypeRecv], source = proc, tag =33) + listFields[:,0:self.topo.ghosts[1],:] = data[:,0:self.topo.ghosts[1],:] + startsSend = [0,self.topo.ghosts[1],0] + WESTtypeSend= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + WESTtypeSend.Commit() + self.comm.Send([listFields, WESTtypeSend], dest = proc, tag =33) + + def NORTHrecv(self, proc, listFields): + subsizes = np.asarray(listFields[ \ + :,:,self.topo.mesh.resolution[2]-self.topo.ghosts[2]:self.topo.mesh.resolution[2]].shape) + startsRecv = np.asarray([0,0,self.topo.mesh.resolution[2]-self.topo.ghosts[2]]) + NORTHtypeRecv = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + NORTHtypeRecv.Commit() + data = listFields + self.comm.Recv([data, NORTHtypeRecv], source=proc, tag=55) + listFields[:,:,self.topo.mesh.resolution[2]-self.topo.ghosts[2]:self.topo.mesh.resolution[2]] = \ + data[:,:,self.topo.mesh.resolution[2]-self.topo.ghosts[2]:self.topo.mesh.resolution[2]] + startsSend = [0,0,self.topo.mesh.resolution[2]-2*self.topo.ghosts[2]] + NORTHtypeSend = MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + NORTHtypeSend.Commit() + self.comm.Send([listFields, NORTHtypeSend], dest=proc, tag=55) + + def SOUTHrecv(self, proc, listFields): + subsizes = np.asarray(listFields[:,:,0:self.topo.ghosts[2]].shape) + startsRecv = np.asarray([0,0,0]) + SOUTHtypeRecv= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsRecv, order=MPI.ORDER_F) + SOUTHtypeRecv.Commit() + data=listFields + self.comm.Recv([data, SOUTHtypeRecv], source = proc, tag =55) + listFields[:,:,0:self.topo.ghosts[2]] = data[:,:,0:self.topo.ghosts[2]] + startsSend = [0,0,self.topo.ghosts[2]] + SOUTHtypeSend= MPI.DOUBLE.Create_subarray(self.size, subsizes, startsSend, order=MPI.ORDER_F) + SOUTHtypeSend.Commit() + self.comm.Send([listFields, SOUTHtypeSend], dest = proc, tag =55) + + def printComputeTime(self): + self.timings_info[0] = "\"Synchronization total\"" + self.timings_info[1] = str(self.total_time) + print "Synchronization total time : ", self.total_time + print "Time of the last Synchronization iteration :", self.compute_time + + def __str__(self): + s = "Synchronize. " + return s + +if __name__ == "__main__": + print __doc__ + print "- Provided class : Synchronize scalar fields" + print SynchronizeS.__doc__ -- GitLab