Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
hysop
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
particle_methods
hysop
Commits
8e4f1136
Commit
8e4f1136
authored
4 years ago
by
EXT Jean-Matthieu Etancelin
Browse files
Options
Downloads
Patches
Plain Diff
fix numpy deprecation and allocation alignment
parent
32051efc
No related branches found
Branches containing commit
No related tags found
Tags containing commit
2 merge requests
!24
Resolve "Add python3.x support"
,
!15
WIP: Resolve "HySoP with tasks"
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
hysop/backend/host/host_buffer.py
+17
-12
17 additions, 12 deletions
hysop/backend/host/host_buffer.py
hysop/core/memory/allocator.py
+13
-12
13 additions, 12 deletions
hysop/core/memory/allocator.py
with
30 additions
and
24 deletions
hysop/backend/host/host_buffer.py
+
17
−
12
View file @
8e4f1136
import
numpy
as
np
import
numpy
as
np
import
ctypes
as
C
from
hysop.constants
import
MemoryOrdering
,
default_order
from
hysop.constants
import
MemoryOrdering
,
default_order
from
hysop.tools.types
import
check_instance
from
hysop.tools.types
import
check_instance
from
hysop.core.memory.buffer
import
Buffer
,
PooledBuffer
from
hysop.core.memory.buffer
import
Buffer
,
PooledBuffer
class
HostBuffer
(
np
.
ndarray
,
Buffer
):
class
HostBuffer
(
np
.
ndarray
,
Buffer
):
"""
"""
Host buffer class.
Host buffer class.
...
@@ -11,22 +13,22 @@ class HostBuffer(np.ndarray, Buffer):
...
@@ -11,22 +13,22 @@ class HostBuffer(np.ndarray, Buffer):
__array_priority__
=
-
1.0
__array_priority__
=
-
1.0
def
__new__
(
cls
,
size
,
def
__new__
(
cls
,
size
,
shape
=
None
,
dtype
=
np
.
uint8
,
order
=
None
,
shape
=
None
,
dtype
=
np
.
uint8
,
order
=
None
,
buffer
=
None
,
offset
=
0
,
strides
=
None
):
buffer
=
None
,
offset
=
0
,
strides
=
None
):
from_buffer
=
False
from_buffer
=
False
if
isinstance
(
buffer
,
Buffer
):
if
isinstance
(
buffer
,
Buffer
):
__buffer
=
buffer
__buffer
=
buffer
buffer
=
buffer
.
buf
buffer
=
buffer
.
buf
from_buffer
=
True
from_buffer
=
True
obj
=
super
(
HostBuffer
,
cls
).
__new__
(
cls
,
obj
=
super
(
HostBuffer
,
cls
).
__new__
(
cls
,
shape
=
shape
or
(
size
,),
dtype
=
dtype
,
order
=
order
,
shape
=
shape
or
(
size
,),
dtype
=
dtype
,
order
=
order
,
buffer
=
buffer
,
offset
=
offset
,
strides
=
strides
)
buffer
=
buffer
,
offset
=
offset
,
strides
=
strides
)
#keep a reference to the buffer (usefull for pooled buffers)
#
keep a reference to the buffer (usefull for pooled buffers)
#such that buffer.__del__ will only be called when all views
#
such that buffer.__del__ will only be called when all views
#on this HostBuffer have been destroyed.
#
on this HostBuffer have been destroyed.
if
from_buffer
and
isinstance
(
__buffer
,
HostPooledBuffer
):
if
from_buffer
and
isinstance
(
__buffer
,
HostPooledBuffer
):
obj
.
_hysop_base_data
=
__buffer
obj
.
_hysop_base_data
=
__buffer
...
@@ -38,11 +40,13 @@ class HostBuffer(np.ndarray, Buffer):
...
@@ -38,11 +40,13 @@ class HostBuffer(np.ndarray, Buffer):
def
__str__
(
self
):
def
__str__
(
self
):
return
self
.
view
(
np
.
ndarray
).
__str__
()
return
self
.
view
(
np
.
ndarray
).
__str__
()
def
__repr__
(
self
):
def
__repr__
(
self
):
return
self
.
view
(
np
.
ndarray
).
__repr__
()
return
self
.
view
(
np
.
ndarray
).
__repr__
()
def
get_int_ptr
(
self
):
def
get_int_ptr
(
self
):
return
self
.
ctypes
.
data
return
self
.
ctypes
.
data
def
release
(
self
):
def
release
(
self
):
pass
pass
...
@@ -51,7 +55,7 @@ class HostBuffer(np.ndarray, Buffer):
...
@@ -51,7 +55,7 @@ class HostBuffer(np.ndarray, Buffer):
"""
"""
Given int ptr should never be freed, numpy take ownership.
Given int ptr should never be freed, numpy take ownership.
"""
"""
buf
=
np
.
c
ore
.
multiarray
.
int_asbuffer
(
int_ptr_value
,
size
)
buf
=
np
.
c
typeslib
.
as_array
(
C
.
cast
(
int_ptr_value
,
C
.
POINTER
(
C
.
c_uint8
)),
(
size
,)
)
return
cls
.
from_buffer
(
buf
)
return
cls
.
from_buffer
(
buf
)
@classmethod
@classmethod
...
@@ -62,7 +66,7 @@ class HostBuffer(np.ndarray, Buffer):
...
@@ -62,7 +66,7 @@ class HostBuffer(np.ndarray, Buffer):
def
aligned_view
(
self
,
alignment
,
size
=
None
):
def
aligned_view
(
self
,
alignment
,
size
=
None
):
assert
self
.
ndim
==
1
assert
self
.
ndim
==
1
assert
self
.
dtype
==
np
.
uint8
assert
self
.
dtype
==
np
.
uint8
assert
alignment
>
0
assert
alignment
>
0
assert
not
(
alignment
&
(
alignment
-
1
)),
'
alignment is not a power of 2.
'
assert
not
(
alignment
&
(
alignment
-
1
)),
'
alignment is not a power of 2.
'
ptr
=
self
.
get_int_ptr
()
ptr
=
self
.
get_int_ptr
()
offset
=
-
ptr
%
alignment
offset
=
-
ptr
%
alignment
...
@@ -70,7 +74,7 @@ class HostBuffer(np.ndarray, Buffer):
...
@@ -70,7 +74,7 @@ class HostBuffer(np.ndarray, Buffer):
size
=
self
.
size
-
offset
size
=
self
.
size
-
offset
else
:
else
:
assert
self
.
size
>=
(
offset
+
size
)
assert
self
.
size
>=
(
offset
+
size
)
buf
=
self
.
__getitem__
(
slice
(
offset
,
offset
+
size
))
buf
=
self
.
__getitem__
(
slice
(
offset
,
offset
+
size
))
return
buf
return
buf
@classmethod
@classmethod
...
@@ -82,6 +86,7 @@ class HostBuffer(np.ndarray, Buffer):
...
@@ -82,6 +86,7 @@ class HostBuffer(np.ndarray, Buffer):
int_ptr
=
property
(
get_int_ptr
)
int_ptr
=
property
(
get_int_ptr
)
class
HostPooledBuffer
(
PooledBuffer
):
class
HostPooledBuffer
(
PooledBuffer
):
def
get_array
(
self
):
def
get_array
(
self
):
return
self
.
_bufview
return
self
.
_bufview
...
...
This diff is collapsed.
Click to expand it.
hysop/core/memory/allocator.py
+
13
−
12
View file @
8e4f1136
import
traceback
import
traceback
from
abc
import
ABCMeta
,
abstractmethod
from
abc
import
ABCMeta
,
abstractmethod
from
hysop
import
__VERBOSE__
,
__TRACE_MEMALLOCS__
,
__BACKTRACE_BIG_MEMALLOCS__
from
hysop
import
__VERBOSE__
,
__TRACE_MEMALLOCS__
,
__BACKTRACE_BIG_MEMALLOCS__
...
@@ -7,12 +6,13 @@ from hysop.tools.units import bytes2str
...
@@ -7,12 +6,13 @@ from hysop.tools.units import bytes2str
from
hysop.tools.types
import
first_not_None
from
hysop.tools.types
import
first_not_None
from
hysop.tools.handle
import
TaggedObject
from
hysop.tools.handle
import
TaggedObject
class
AllocatorBase
(
TaggedObject
,
metaclass
=
ABCMeta
):
class
AllocatorBase
(
TaggedObject
,
metaclass
=
ABCMeta
):
"""
"""
Base class for allocators.
Base class for allocators.
"""
"""
is_deferred
=
False
is_deferred
=
False
def
__new__
(
cls
,
verbose
,
**
kwds
):
def
__new__
(
cls
,
verbose
,
**
kwds
):
return
super
(
AllocatorBase
,
cls
).
__new__
(
cls
,
tag_prefix
=
'
al
'
,
**
kwds
)
return
super
(
AllocatorBase
,
cls
).
__new__
(
cls
,
tag_prefix
=
'
al
'
,
**
kwds
)
...
@@ -24,8 +24,10 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
...
@@ -24,8 +24,10 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
def
__eq__
(
self
,
other
):
def
__eq__
(
self
,
other
):
return
(
self
is
other
)
return
(
self
is
other
)
def
__ne__
(
self
,
other
):
def
__ne__
(
self
,
other
):
return
(
self
is
not
other
)
return
(
self
is
not
other
)
def
__hash__
(
self
):
def
__hash__
(
self
):
return
id
(
self
)
return
id
(
self
)
...
@@ -39,12 +41,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
...
@@ -39,12 +41,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
try_count
=
0
try_count
=
0
while
try_count
<
2
:
while
try_count
<
2
:
try
:
try
:
if
(
alignment
>
1
):
if
alignment
is
None
or
alignment
==
1
:
buf
=
self
.
allocate_aligned
(
size
=
size
,
alignment
=
alignment
)
else
:
buf
=
self
.
allocate
(
nbytes
=
size
)
buf
=
self
.
allocate
(
nbytes
=
size
)
else
:
buf
=
self
.
allocate_aligned
(
size
=
size
,
alignment
=
alignment
)
if
(
buf
is
None
):
if
(
buf
is
None
):
msg
=
'
{}.allocate(): returned allocation is None.
'
.
format
(
self
.
__class__
)
msg
=
'
{}.allocate(): returned allocation is None.
'
.
format
(
self
.
__class__
)
raise
ValueError
(
msg
)
raise
ValueError
(
msg
)
return
buf
return
buf
except
MemoryError
:
except
MemoryError
:
...
@@ -80,8 +82,8 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
...
@@ -80,8 +82,8 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
"""
"""
if
(
self
.
_verbose
or
__BACKTRACE_BIG_MEMALLOCS__
)
and
verbose
:
if
(
self
.
_verbose
or
__BACKTRACE_BIG_MEMALLOCS__
)
and
verbose
:
print
(
'
{}allocating block of size {}.
'
.
format
(
print
(
'
{}allocating block of size {}.
'
.
format
(
self
.
prefix
(),
bytes2str
(
nbytes
)))
self
.
prefix
(),
bytes2str
(
nbytes
)))
if
__BACKTRACE_BIG_MEMALLOCS__
and
nbytes
>
64
*
1024
*
1024
:
if
__BACKTRACE_BIG_MEMALLOCS__
and
nbytes
>
64
*
1024
*
1024
:
print
(
'
[BIG ALLOCATION BACKTRACE]
'
)
print
(
'
[BIG ALLOCATION BACKTRACE]
'
)
print
(
''
.
join
(
traceback
.
format_stack
()))
print
(
''
.
join
(
traceback
.
format_stack
()))
print
(
'
[END OF TRACE]
'
)
print
(
'
[END OF TRACE]
'
)
...
@@ -91,12 +93,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
...
@@ -91,12 +93,12 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
"""
"""
Allocate size bytes aligned on alignment.
Allocate size bytes aligned on alignment.
"""
"""
assert
alignment
>
0
assert
alignment
>
0
assert
(
alignment
&
(
alignment
-
1
))
==
0
,
'
alignment is not a power of 2.
'
assert
(
alignment
&
(
alignment
-
1
))
==
0
,
'
alignment is not a power of 2.
'
nbytes
=
size
+
alignment
-
1
nbytes
=
size
+
alignment
-
1
if
self
.
_verbose
:
if
self
.
_verbose
:
print
(
'
{}allocating block of size {}, to satisfy {} aligned on {} bytes.
'
.
format
(
print
(
'
{}allocating block of size {}, to satisfy {} aligned on {} bytes.
'
.
format
(
self
.
prefix
(),
bytes2str
(
nbytes
),
bytes2str
(
size
),
alignment
))
self
.
prefix
(),
bytes2str
(
nbytes
),
bytes2str
(
size
),
alignment
))
return
self
.
allocate
(
nbytes
,
verbose
=
False
).
aligned_view
(
alignment
=
alignment
,
size
=
size
)
return
self
.
allocate
(
nbytes
,
verbose
=
False
).
aligned_view
(
alignment
=
alignment
,
size
=
size
)
def
try_release_blocks
(
self
):
def
try_release_blocks
(
self
):
...
@@ -111,4 +113,3 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
...
@@ -111,4 +113,3 @@ class AllocatorBase(TaggedObject, metaclass=ABCMeta):
Release the allocated buffer.
Release the allocated buffer.
"""
"""
buf
.
release
()
buf
.
release
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment