Skip to content
Snippets Groups Projects
Commit aea25a67 authored by Eric Dalissier's avatar Eric Dalissier
Browse files

elimination of SHIFT and replace by neighbours, and test ok for greaterSend option

parent 8800239f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
HySoP/hysop/operator/synchronizeGhosts_d.py
+ 106
− 101
View file @ aea25a67
...@@ -31,6 +31,8 @@ class SynchronizeGhosts_d(DiscreteOperator): ...@@ -31,6 +31,8 @@ class SynchronizeGhosts_d(DiscreteOperator):
self.compute_time = 0. self.compute_time = 0.
self.fieldslist = fieldslist self.fieldslist = fieldslist
self.topology = topology self.topology = topology
if (main_rank == 0):
print 'CUT SPACE', self.topology.dims
self.transferMethod = transferMethod self.transferMethod = transferMethod
@debug @debug
...@@ -64,22 +66,25 @@ class SynchronizeGhosts_d(DiscreteOperator): ...@@ -64,22 +66,25 @@ class SynchronizeGhosts_d(DiscreteOperator):
order=ORDER, dtype=PARMES_INTEGER) order=ORDER, dtype=PARMES_INTEGER)
self.SendRecvList[0], self.SendRecvList[1] = \ self.SendRecvList[0], self.SendRecvList[1] = \
self.topology.Shift(0, 1) self.topology.Shift(0, 1)
self.SendRecvList[1], self.SendRecvList[1] = \ self.SendRecvList[2], self.SendRecvList[3] = \
self.topology.Shift(1, 1) self.topology.Shift(1, 1)
if (self.topology.dim > 2): if (self.topology.dim > 2):
self.SendRecvList[2], self.SendRecvList[1] = \ self.SendRecvList[4], self.SendRecvList[5] = \
self.topology.Shift(2, 1) self.topology.Shift(2, 1)
if (self.transferMethod == 'greaterSend'): if (self.transferMethod == 'greaterSend'):
self.tabSort = self.topology.neighbours tab = self.topology.neighbours
self.down, self.up = self.topoMPI.Shift(0, 1) self.tabSort = np.zeros(
if(self.topology.dim > 1): [self.topology.dim * 2,2],
self.west, self.east = self.topoMPI.Shift(1, 1) order=ORDER, dtype=PARMES_INTEGER)
self.tabSort[:, 1] = range(self.topology.dim * 2)
self.tabSort[0, 0] = tab[0, 0]
self.tabSort[1, 0] = tab[1, 0]
self.tabSort[2, 0] = tab[0, 1]
self.tabSort[3, 0] = tab[1, 1]
if(self.topology.dim > 2): if(self.topology.dim > 2):
self.south, self.north = self.topoMPI.Shift(2, 1) self.tabSort[4, 0] = tab[0, 2]
self.tabSort = np.array( self.tabSort[5, 0] = tab[1, 2]
[[self.up, 0], [self.down, 1], [self.east, 2], [self.west, 3],
[self.north, 4], [self.south, 5]])
tabSort1 = self.tabSort[self.tabSort[:, 0] <= main_rank] tabSort1 = self.tabSort[self.tabSort[:, 0] <= main_rank]
tabSort2 = self.tabSort[self.tabSort[:, 0] > main_rank] tabSort2 = self.tabSort[self.tabSort[:, 0] > main_rank]
if (np.asarray(tabSort1[:, 0].shape) > 0): if (np.asarray(tabSort1[:, 0].shape) > 0):
...@@ -226,40 +231,40 @@ class SynchronizeGhosts_d(DiscreteOperator): ...@@ -226,40 +231,40 @@ class SynchronizeGhosts_d(DiscreteOperator):
for i in xrange(self.tabSort[:, 0].shape[0]): for i in xrange(self.tabSort[:, 0].shape[0]):
if (main_rank == self.tabSort[i, 0]): if (main_rank == self.tabSort[i, 0]):
## Without communication ## Without communication
# UP DOWN # WEST
if (self.tabSort[i, 1] == 0): if (self.tabSort[i, 1] == 0):
for j in xrange(dim): for j in xrange(dim):
f[j][resolution[0] - ghosts[0]: resolution[0], f[j][resolution[0] - ghosts[0]: resolution[0],
:, :] =\ :, :] =\
f[j][ghosts[0]:2 * ghosts[0], :, :] f[j][ghosts[0]:2 * ghosts[0], :, :]
# EAST
if (self.tabSort[i, 1] == 1): if (self.tabSort[i, 1] == 1):
for j in xrange(dim): for j in xrange(dim):
f[j][0:ghosts[0], :, :] =\ f[j][0:ghosts[0], :, :] =\
f[j][resolution[0] - 2 * f[j][resolution[0] - 2 *
ghosts[0]:resolution[0] - ghosts[0], :, :] ghosts[0]:resolution[0] - ghosts[0], :, :]
# WEST EAST # SOUTH
if (self.tabSort[i, 1] == 2): if (self.tabSort[i, 1] == 2):
for j in xrange(dim): for j in xrange(dim):
f[j][:, resolution[1] - f[j][:, resolution[1] -
ghosts[1]:resolution[1], :] =\ ghosts[1]:resolution[1], :] =\
f[j][:, ghosts[1]:2 * f[j][:, ghosts[1]:2 *
ghosts[1], :] ghosts[1], :]
#NORTH
if(self.tabSort[i, 1] == 3): if(self.tabSort[i, 1] == 3):
for j in xrange(dim): for j in xrange(dim):
f[j][:, 0:ghosts[1], :] =\ f[j][:, 0:ghosts[1], :] =\
f[j][:, resolution[1] - 2 * f[j][:, resolution[1] - 2 *
ghosts[1]:resolution[1] - ghosts[1], :] ghosts[1]:resolution[1] - ghosts[1], :]
# NORTH SOUTH # DOWN
if(self.tabSort[i, 1] == 4): if(self.tabSort[i, 1] == 4):
for j in xrange(dim): for j in xrange(dim):
f[j][:, :, resolution[2] - f[j][:, :, resolution[2] -
ghosts[2]:resolution[2]] =\ ghosts[2]:resolution[2]] =\
f[j][:, :, ghosts[2]:2 * ghosts[2]] f[j][:, :, ghosts[2]:2 * ghosts[2]]
# UP
if(self.tabSort[i, 1] == 5): if(self.tabSort[i, 1] == 5):
for j in xrange(dim): for j in xrange(dim):
f[j][:, :, 0:ghosts[2]] =\ f[j][:, :, 0:ghosts[2]] =\
...@@ -267,68 +272,55 @@ class SynchronizeGhosts_d(DiscreteOperator): ...@@ -267,68 +272,55 @@ class SynchronizeGhosts_d(DiscreteOperator):
ghosts[2]:resolution[2] - ghosts[2]] ghosts[2]:resolution[2] - ghosts[2]]
else: else:
### PART COMMUNICATION ### PART COMMUNICATION
# UP # WEST
if(self.tabSort[i, 1] == 0): if(self.tabSort[i, 1] == 0):
if (main_rank < self.tabSort[i, 0]): if (main_rank < self.tabSort[i, 0]):
self.UPsend(self.tabSo.mesh.coords) self.WESTsend(self.tabSort[i, 0], f)
else: else:
self.UPrecv(self.tabSort[i, 0], f) self.WESTrecv(self.tabSort[i, 0], f)
# DOWN # EST
if (self.tabSort[i, 1] == 1): if (self.tabSort[i, 1] == 1):
if (main_rank < self.tabSort[i, 0]): if (main_rank < self.tabSort[i, 0]):
self.DOWNsend(self.tab.mesh.coords) self.EASTsend(self.tabSort[i, 0], f)
else:
self.DOWNrecv(self.tabSort[i, 0], f)
# NORTH
if (self.tabSort[i, 1] == 2):
if (main_rank < self.tabSort[i, 0]):
self.NORTHsend(self.tabSort[i, 0], f)
else: else:
self.NORTHrecv(self.tabSort[i, 0], f) self.EASTrecv(self.tabSort[i, 0], f)
# SOUTH # SOUTH
if (self.tabSort[i, 1] == 3): if(self.tabSort[i, 1] == 2):
if (main_rank < self.tabSort[i, 0]): if (main_rank < self.tabSort[i, 0]):
self.SOUTHsend(self.tabSort[i, 0], f) self.SOUTHsend(self.tabSort[i, 0], f)
else: else:
self.SOUTHrecv(self.tabSort[i, 0], f) self.SOUTHrecv(self.tabSort[i, 0], f)
# EAST # NORTH
if(self.tabSort[i, 1] == 4): if (self.tabSort[i, 1] == 3):
if (main_rank < self.tabSort[i, 0]): if (main_rank < self.tabSort[i, 0]):
self.EASTsend(self.tabSort[i, 0], f) self.NORTHsend(self.tabSort[i, 0], f)
else: else:
self.EASTrecv(self.tabSort[i, 0], f) self.NORTHrecv(self.tabSort[i, 0], f)
# WEST # DOWN
if (self.tabSort[i, 1] == 4):
if (main_rank < self.tabSort[i, 0]):
self.DOWNsend(self.tabSort[i, 0], f)
else:
self.DOWNrecv(self.tabSort[i, 0], f)
# UP
if (self.tabSort[i, 1] == 5): if (self.tabSort[i, 1] == 5):
if (main_rank < self.tabSort[i, 0]): if (main_rank < self.tabSort[i, 0]):
self.WESTsend(self.tabSort[i, 0], f) self.UPsend(self.tabSort[i, 0], f)
else: else:
self.WESTrecv(self.tabSort[i, 0], f) self.UPrecv(self.tabSort[i, 0], f)
# self.compute_time = time.time() - self.compute_time # self.compute_time = time.time() - self.compute_time
# self.total_time += self.compute_time # self.total_time += self.compute_time
# return self.discreteOperator.apply(*args) # return self.discreteOperator.apply(*args)
def UPsend(self, proc, listFields): def WESTsend(self, proc, listFields):
ghosts = self.topology.ghosts
resolution = self.resolution
for i in xrange(self.topology.dim):
self.dataSendZ = np.array(np.copy(listFields[i][:, :,
resolution[2] - 2 * ghosts[2]:resolution[2] - ghosts[2],
]), order=ORDER)
main_comm.Ssend(self.dataSendZ, dest=proc, tag=11)
main_comm.Recv(self.dataSendZ, source=proc, tag=11)
listFields[i][:, :,
resolution[2] - ghosts[2]:resolution[2]] =\
self.dataSendZ
def DOWNsend(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
self.dataSendZ = np.array(np.copy(listFields[i][ self.dataSendX = np.array(np.copy(listFields[i][
:, :, ghosts[2]:2 * ghosts[2]]), order=ORDER) ghosts[0]:ghosts[0] * 2, :, :]), order=ORDER)
main_comm.Ssend(self.dataSendZ, dest=proc, tag=22) main_comm.Ssend(self.dataSendX, dest=proc, tag=66)
main_comm.Recv(self.dataSendZ, source=proc, tag=22) main_comm.Recv(self.dataSendX, source=proc, tag=55)
listFields[i][:, :, 0:ghosts[2]] = self.dataSendZ listFields[i][0:ghosts[0], :, :] = self.dataSendX
def EASTsend(self, proc, listFields): def EASTsend(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
...@@ -338,20 +330,19 @@ class SynchronizeGhosts_d(DiscreteOperator): ...@@ -338,20 +330,19 @@ class SynchronizeGhosts_d(DiscreteOperator):
resolution[0] - 2 * ghosts[0]:resolution[0] - ghosts[0], resolution[0] - 2 * ghosts[0]:resolution[0] - ghosts[0],
:, :]), :, :]),
order=ORDER) order=ORDER)
main_comm.Ssend(self.dataSendY, dest=proc, tag=33) main_comm.Recv(self.dataSendX, source=proc, tag=55)
main_comm.Recv(self.dataSendY, source=proc, tag=33)
listFields[i][resolution[0] - ghosts[0]:resolution[0], :, :] =\ listFields[i][resolution[0] - ghosts[0]:resolution[0], :, :] =\
self.dataSendX self.dataSendX
def WESTsend(self, proc, listFields): def SOUTHsend(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
self.dataSendX = np.array(np.copy(listFields[i][ self.dataSendY = np.array(np.copy(listFields[i][
ghosts[0]:ghosts[0] * 2, :, :]), order=ORDER) :, ghosts[1]:2 * ghosts[1], :]), order=ORDER)
main_comm.Ssend(self.dataSendX, dest=proc, tag=44) main_comm.Ssend(self.dataSendY, dest=proc, tag=44)
main_comm.Recv(self.dataSendX, source=proc, tag=44) main_comm.Recv(self.dataSendY, source=proc, tag=33)
listFields[i][0:ghosts[0], :, :] = self.dataSendX listFields[i][:, 0:ghosts[1], :] = self.dataSendY
def NORTHsend(self, proc, listFields): def NORTHsend(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
...@@ -361,90 +352,104 @@ class SynchronizeGhosts_d(DiscreteOperator): ...@@ -361,90 +352,104 @@ class SynchronizeGhosts_d(DiscreteOperator):
:, :,
resolution[1] - 2 * ghosts[1]:resolution[1] - ghosts[1], :]), resolution[1] - 2 * ghosts[1]:resolution[1] - ghosts[1], :]),
order=ORDER) order=ORDER)
main_comm.Ssend(self.dataSendY, dest=proc, tag=55) main_comm.Ssend(self.dataSendY, dest=proc, tag=33)
main_comm.Recv(self.dataSendY, source=proc, tag=55) main_comm.Recv(self.dataSendY, source=proc, tag=44)
listFields[i][:, resolution[1] - ghosts[1]:resolution[1], :] =\ listFields[i][:, resolution[1] - ghosts[1]:resolution[1], :] =\
self.dataSendY self.dataSendY
def SOUTHsend(self, proc, listFields): def DOWNsend(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
self.dataSendY = np.array(np.copy(listFields[i][ self.dataSendZ = np.array(np.copy(listFields[i][
:, ghosts[1]:2 * ghosts[1], :]), order=ORDER) :, :, ghosts[2]:2 * ghosts[2]]), order=ORDER)
main_comm.Ssend(self.dataSendY, dest=proc, tag=66) main_comm.Ssend(self.dataSendZ, dest=proc, tag=11)
main_comm.Recv(self.dataSendY, source=proc, tag=66) main_comm.Recv(self.dataSendZ, source=proc, tag=22)
listFields[i][:, 0:ghosts[1], :] = self.dataSendY listFields[i][:, :, 0:ghosts[2]] = self.dataSendZ
def UPrecv(self, proc, listFields): def UPsend(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendZ, source=proc, tag=22) self.dataSendZ = np.array(np.copy(listFields[i][:, :,
listFields[i][:, :, resolution[2] - ghosts[2]:resolution[2]] =\
self.dataSendZ
self.dataSendZ = np.array(np.copy(listFields[i][
:, :,
resolution[2] - 2 * ghosts[2]:resolution[2] - ghosts[2] resolution[2] - 2 * ghosts[2]:resolution[2] - ghosts[2]
]), ]), order=ORDER)
order=ORDER)
main_comm.Ssend(self.dataSendZ, dest=proc, tag=22) main_comm.Ssend(self.dataSendZ, dest=proc, tag=22)
main_comm.Recv(self.dataSendZ, source=proc, tag=11)
listFields[i][:, :,
resolution[2] - ghosts[2]:resolution[2]] =\
self.dataSendZ
def DOWNrecv(self, proc, listFields): def WESTrecv(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendZ, source=proc, tag=11) main_comm.Recv(self.dataSendX, source=proc, tag=66)
listFields[i][:, :, 0:self.topo.ghosts[2]] = self.dataSendZ listFields[i][0:ghosts[0], :, :] = self.dataSendX
data = np.array(np.copy(listFields[i][ self.dataSendX = np.array(np.copy(listFields[i][
:, :, ghosts[2]:ghosts[2] * 2]), order=ORDER) ghosts[0]:2 * ghosts[0], :, :]), order=ORDER)
main_comm.Ssend(self.dataSendZ, dest=proc, tag=11) main_comm.Ssend(self.dataSendX, dest=proc, tag=55)
def EASTrecv(self, proc, listFields): def EASTrecv(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendX, source=proc, tag=44) print 'shapeEST', self.dataSendX.shape
main_comm.Recv(self.dataSendX, source=proc, tag=66)
listFields[i][resolution[0] - ghosts[0]:resolution[0], :, :] =\ listFields[i][resolution[0] - ghosts[0]:resolution[0], :, :] =\
self.dataSendX self.dataSendX
self.dataSendX = np.array(np.copy(listFields[i][ self.dataSendX = np.array(np.copy(listFields[i][
resolution[0] - 2 * ghosts[0]:resolution[0] - ghosts[0],\ resolution[0] - 2 * ghosts[0]:resolution[0] - ghosts[0],\
:, :]), :, :]),
order=ORDER) order=ORDER)
main_comm.Ssend(self.dataSendX, dest=proc, tag=44) main_comm.Ssend(self.dataSendX, dest=proc, tag=55)
def WESTrecv(self, proc, listFields): def SOUTHrecv(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendX, source=proc, tag=33) main_comm.Recv(self.dataSendY, source=proc, tag=33)
listFields[i][0:ghosts[0], :, :] = self.dataSendX listFields[i][:, 0:ghosts[1], :] = self.dataSendY
self.dataSendX = np.array(np.copy(listFields[i][ self.dataSendY = np.array(np.copy(listFields[i][
ghosts[0]:2 * ghosts[0], :, :]), order=ORDER) :, ghosts[1]:2 * ghosts[1], :]), order=ORDER)
main_comm.Ssend(self.dataSendX, dest=proc, tag=33) main_comm.Ssend(self.dataSendY, dest=proc, tag=44)
def NORTHrecv(self, proc, listFields): def NORTHrecv(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendY, source=proc, tag=66) main_comm.Recv(self.dataSendY, source=proc, tag=44)
listFields[i][:, resolution[1] - ghosts[1]:resolution[1], :] =\ listFields[i][:, resolution[1] - ghosts[1]:resolution[1], :] =\
self.dataSendY self.dataSendY
self.dataSendY = np.array(np.copy(listFields[i][ self.dataSendY = np.array(np.copy(listFields[i][
:, resolution[1] - 2 * ghosts[1]:resolution[1] - :, resolution[1] - 2 * ghosts[1]:resolution[1] -
ghosts[1], :]), ghosts[1], :]),
order=ORDER) order=ORDER)
main_comm.Ssend(self.dataSendY, dest=proc, tag=66) main_comm.Ssend(self.dataSendY, dest=proc, tag=33)
def SOUTHrecv(self, proc, listFields): def DOWNrecv(self, proc, listFields):
ghosts = self.topology.ghosts ghosts = self.topology.ghosts
resolution = self.resolution resolution = self.resolution
for i in xrange(self.topology.dim): for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendY, source=proc, tag=55) main_comm.Recv(self.dataSendZ, source=proc, tag=22)
listFields[i][:, 0:ghosts[1], :] = self.dataSendY listFields[i][:, :, 0:ghosts[2]] = self.dataSendZ
self.dataSendY = np.array(np.copy(listFields[i][ data = np.array(np.copy(listFields[i][
:, ghosts[1]:2 * ghosts[1], :]), order=ORDER) :, :, ghosts[2]:ghosts[2] * 2]), order=ORDER)
main_comm.Ssend(self.dataSendY, dest=proc, tag=55) main_comm.Ssend(self.dataSendZ, dest=proc, tag=11)
def UPrecv(self, proc, listFields):
ghosts = self.topology.ghosts
resolution = self.resolution
for i in xrange(self.topology.dim):
main_comm.Recv(self.dataSendZ, source=proc, tag=11)
listFields[i][:, :, resolution[2] - ghosts[2]:resolution[2]] =\
self.dataSendZ
self.dataSendZ = np.array(np.copy(listFields[i][
:, :,
resolution[2] - 2 * ghosts[2]:resolution[2] - ghosts[2]
]),
order=ORDER)
main_comm.Ssend(self.dataSendZ, dest=proc, tag=22)
def __str__(self): def __str__(self):
"""ToString method""" """ToString method"""
......
HySoP/hysop/operator/tests/test_synchronizeGhosts.py
+ 12
− 32
View file @ aea25a67
...@@ -24,43 +24,23 @@ def test_Synchro(): ...@@ -24,43 +24,23 @@ def test_Synchro():
boxLength = [1., 1., 1.] boxLength = [1., 1., 1.]
boxMin = [0., 0., 0.] boxMin = [0., 0., 0.]
nbElem = [nb, nb, nb] nbElem = [nb, nb, nb]
nbGhosts = 2. nbGhosts = [2, 2, 2]
dom = pp.Box(dim, length=boxLength, origin=boxMin) dom = pp.Box(dim, length=boxLength, origin=boxMin)
## Fields ## Fields
field = Field(domain=dom, name='velocity', isVector=True) field = Field(domain=dom, name='velocity', isVector=True)
# a = Analytic(field, formula=computeVel, velo = AnalyticalField(domain=dom, formula=computeVel,
# resolution={field: nbElem}, name='Velocity', isVector=True)
# )
# a.setUp() Synchrotest = SynchronizeGhosts(velo, resolution={velo: nbElem},
# velo = AnalyticalField(domain=dom, formula=computeVel, method='greaterSend')
# name='Velocity', isVector=True) if main_rank ==0 :
print "Setup Synchro"
Synchrotest = SynchronizeGhosts(field, resolution={field: nbElem, }, Synchrotest.setUp(nbGhosts)
method='greaterSend') if main_rank ==0 :
# Synchrotest = SynchronizeGhosts(velo, resolution={velo: nbElem}, print "Apply Synchro"
# method='greaterSend')
# DiscreteOperator.__init__(velo)
# Operator.setUp(self)
# topoId, topo = dom.addTopology(
# Cartesian(domain=dom, dim=3,
# globalMeshResolution=[nb, nb, nb], # {velo: nbElem},
# comm=main_comm, periods=np.asarray([True, True, True]),
# ghosts=[nbGhosts, nbGhosts, nbGhosts]))
# df, dfId = velo.discretize(topo)
# self.discreteFieldId[v] = dfId
# DiscreteOperator.__init__( velo)
# synchro = SynchronizeGhosts(velo, topo, 'greaterSend')
# synchro.discreteFieldId[velo] = dfId
# self.discreteOperator = Stretching_d(self, method=self.method,
# **self.config)
# self.discreteOperator.setUp()
print "Setup Synchro"
Synchrotest.setUp()
print "Apply Synchro"
Synchrotest.apply() Synchrotest.apply()
if __name__ == "__main__": if __name__ == "__main__":
print "Rank", main_rank # print "Rank", main_rank
test_Synchro() test_Synchro()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment