Skip to content
Snippets Groups Projects
Commit 8e4f1136 authored by EXT Jean-Matthieu Etancelin's avatar EXT Jean-Matthieu Etancelin
Browse files

fix numpy deprecation and allocation alignment

parent 32051efc
No related branches found
No related tags found
2 merge requests!24Resolve "Add python3.x support",!15WIP: Resolve "HySoP with tasks"
import numpy as np import numpy as np
import ctypes as C
from hysop.constants import MemoryOrdering, default_order from hysop.constants import MemoryOrdering, default_order
from hysop.tools.types import check_instance from hysop.tools.types import check_instance
from hysop.core.memory.buffer import Buffer, PooledBuffer from hysop.core.memory.buffer import Buffer, PooledBuffer
class HostBuffer(np.ndarray, Buffer): class HostBuffer(np.ndarray, Buffer):
""" """
Host buffer class. Host buffer class.
...@@ -11,22 +13,22 @@ class HostBuffer(np.ndarray, Buffer): ...@@ -11,22 +13,22 @@ class HostBuffer(np.ndarray, Buffer):
__array_priority__ = -1.0 __array_priority__ = -1.0
def __new__(cls, size, def __new__(cls, size,
shape=None, dtype=np.uint8, order=None, shape=None, dtype=np.uint8, order=None,
buffer=None, offset=0, strides=None): buffer=None, offset=0, strides=None):
from_buffer = False from_buffer = False
if isinstance(buffer, Buffer): if isinstance(buffer, Buffer):
__buffer = buffer __buffer = buffer
buffer = buffer.buf buffer = buffer.buf
from_buffer = True from_buffer = True
obj = super(HostBuffer,cls).__new__(cls, obj = super(HostBuffer, cls).__new__(cls,
shape=shape or (size,), dtype=dtype, order=order, shape=shape or (size,), dtype=dtype, order=order,
buffer=buffer, offset=offset, strides=strides) buffer=buffer, offset=offset, strides=strides)
#keep a reference to the buffer (usefull for pooled buffers) # keep a reference to the buffer (usefull for pooled buffers)
#such that buffer.__del__ will only be called when all views # such that buffer.__del__ will only be called when all views
#on this HostBuffer have been destroyed. # on this HostBuffer have been destroyed.
if from_buffer and isinstance(__buffer, HostPooledBuffer): if from_buffer and isinstance(__buffer, HostPooledBuffer):
obj._hysop_base_data = __buffer obj._hysop_base_data = __buffer
...@@ -38,11 +40,13 @@ class HostBuffer(np.ndarray, Buffer): ...@@ -38,11 +40,13 @@ class HostBuffer(np.ndarray, Buffer):
def __str__(self): def __str__(self):
return self.view(np.ndarray).__str__() return self.view(np.ndarray).__str__()
def __repr__(self): def __repr__(self):
return self.view(np.ndarray).__repr__() return self.view(np.ndarray).__repr__()
def get_int_ptr(self): def get_int_ptr(self):
return self.ctypes.data return self.ctypes.data
def release(self): def release(self):
pass pass
...@@ -51,7 +55,7 @@ class HostBuffer(np.ndarray, Buffer): ...@@ -51,7 +55,7 @@ class HostBuffer(np.ndarray, Buffer):
""" """
Given int ptr should never be freed, numpy take ownership. Given int ptr should never be freed, numpy take ownership.
""" """
buf = np.core.multiarray.int_asbuffer(int_ptr_value, size) buf = np.ctypeslib.as_array(C.cast(int_ptr_value, C.POINTER(C.c_uint8)), (size,))
return cls.from_buffer(buf) return cls.from_buffer(buf)
@classmethod @classmethod
...@@ -62,7 +66,7 @@ class HostBuffer(np.ndarray, Buffer): ...@@ -62,7 +66,7 @@ class HostBuffer(np.ndarray, Buffer):
def aligned_view(self, alignment, size=None): def aligned_view(self, alignment, size=None):
assert self.ndim == 1 assert self.ndim == 1
assert self.dtype == np.uint8 assert self.dtype == np.uint8
assert alignment>0 assert alignment > 0
assert not (alignment & (alignment-1)), 'alignment is not a power of 2.' assert not (alignment & (alignment-1)), 'alignment is not a power of 2.'
ptr = self.get_int_ptr() ptr = self.get_int_ptr()
offset = -ptr % alignment offset = -ptr % alignment
...@@ -70,7 +74,7 @@ class HostBuffer(np.ndarray, Buffer): ...@@ -70,7 +74,7 @@ class HostBuffer(np.ndarray, Buffer):
size = self.size-offset size = self.size-offset
else: else:
assert self.size >= (offset+size) assert self.size >= (offset+size)
buf = self.__getitem__(slice(offset,offset+size)) buf = self.__getitem__(slice(offset, offset+size))
return buf return buf
@classmethod @classmethod
...@@ -82,6 +86,7 @@ class HostBuffer(np.ndarray, Buffer): ...@@ -82,6 +86,7 @@ class HostBuffer(np.ndarray, Buffer):
int_ptr = property(get_int_ptr) int_ptr = property(get_int_ptr)
class HostPooledBuffer(PooledBuffer): class HostPooledBuffer(PooledBuffer):
def get_array(self): def get_array(self):
return self._bufview return self._bufview
......
import traceback import traceback
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
from hysop import __VERBOSE__, __TRACE_MEMALLOCS__, __BACKTRACE_BIG_MEMALLOCS__ from hysop import __VERBOSE__, __TRACE_MEMALLOCS__, __BACKTRACE_BIG_MEMALLOCS__
...@@ -7,12 +6,13 @@ from hysop.tools.units import bytes2str ...@@ -7,12 +6,13 @@ from hysop.tools.units import bytes2str
from hysop.tools.types import first_not_None from hysop.tools.types import first_not_None
from hysop.tools.handle import TaggedObject from hysop.tools.handle import TaggedObject
class AllocatorBase(TaggedObject, metaclass=ABCMeta): class AllocatorBase(TaggedObject, metaclass=ABCMeta):
""" """
Base class for allocators. Base class for allocators.
""" """
is_deferred=False is_deferred = False
def __new__(cls, verbose, **kwds): def __new__(cls, verbose, **kwds):
return super(AllocatorBase, cls).__new__(cls, tag_prefix='al', **kwds) return super(AllocatorBase, cls).__new__(cls, tag_prefix='al', **kwds)
...@@ -24,8 +24,10 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta): ...@@ -24,8 +24,10 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
def __eq__(self, other): def __eq__(self, other):
return (self is other) return (self is other)
def __ne__(self, other): def __ne__(self, other):
return (self is not other) return (self is not other)
def __hash__(self): def __hash__(self):
return id(self) return id(self)
...@@ -39,12 +41,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta): ...@@ -39,12 +41,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
try_count = 0 try_count = 0
while try_count < 2: while try_count < 2:
try: try:
if (alignment > 1): if alignment is None or alignment == 1:
buf = self.allocate_aligned(size=size, alignment=alignment)
else:
buf = self.allocate(nbytes=size) buf = self.allocate(nbytes=size)
else:
buf = self.allocate_aligned(size=size, alignment=alignment)
if (buf is None): if (buf is None):
msg='{}.allocate(): returned allocation is None.'.format(self.__class__) msg = '{}.allocate(): returned allocation is None.'.format(self.__class__)
raise ValueError(msg) raise ValueError(msg)
return buf return buf
except MemoryError: except MemoryError:
...@@ -80,8 +82,8 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta): ...@@ -80,8 +82,8 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
""" """
if (self._verbose or __BACKTRACE_BIG_MEMALLOCS__) and verbose: if (self._verbose or __BACKTRACE_BIG_MEMALLOCS__) and verbose:
print('{}allocating block of size {}.'.format( print('{}allocating block of size {}.'.format(
self.prefix(), bytes2str(nbytes))) self.prefix(), bytes2str(nbytes)))
if __BACKTRACE_BIG_MEMALLOCS__ and nbytes>64*1024*1024: if __BACKTRACE_BIG_MEMALLOCS__ and nbytes > 64*1024*1024:
print('[BIG ALLOCATION BACKTRACE]') print('[BIG ALLOCATION BACKTRACE]')
print(''.join(traceback.format_stack())) print(''.join(traceback.format_stack()))
print('[END OF TRACE]') print('[END OF TRACE]')
...@@ -91,12 +93,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta): ...@@ -91,12 +93,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
""" """
Allocate size bytes aligned on alignment. Allocate size bytes aligned on alignment.
""" """
assert alignment>0 assert alignment > 0
assert (alignment & (alignment-1))==0, 'alignment is not a power of 2.' assert (alignment & (alignment-1)) == 0, 'alignment is not a power of 2.'
nbytes = size + alignment - 1 nbytes = size + alignment - 1
if self._verbose: if self._verbose:
print('{}allocating block of size {}, to satisfy {} aligned on {} bytes.'.format( print('{}allocating block of size {}, to satisfy {} aligned on {} bytes.'.format(
self.prefix(), bytes2str(nbytes), bytes2str(size), alignment)) self.prefix(), bytes2str(nbytes), bytes2str(size), alignment))
return self.allocate(nbytes, verbose=False).aligned_view(alignment=alignment, size=size) return self.allocate(nbytes, verbose=False).aligned_view(alignment=alignment, size=size)
def try_release_blocks(self): def try_release_blocks(self):
...@@ -111,4 +113,3 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta): ...@@ -111,4 +113,3 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
Release the allocated buffer. Release the allocated buffer.
""" """
buf.release() buf.release()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment