Grok  10.0.3
aligned_allocator.h
Go to the documentation of this file.
1 // Copyright 2020 Google LLC
2 // SPDX-License-Identifier: Apache-2.0
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 
16 #ifndef HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
17 #define HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
18 
19 // Memory allocator with support for alignment and offsets.
20 
21 #include <stddef.h>
22 
23 #include <memory>
24 
25 #include "hwy/highway_export.h"
26 
27 namespace hwy {
28 
29 // Minimum alignment of allocated memory for use in HWY_ASSUME_ALIGNED, which
30 // requires a literal. This matches typical L1 cache line sizes, which prevents
31 // false sharing.
32 #define HWY_ALIGNMENT 64
33 
34 // Pointers to functions equivalent to malloc/free with an opaque void* passed
35 // to them.
36 using AllocPtr = void* (*)(void* opaque, size_t bytes);
37 using FreePtr = void (*)(void* opaque, void* memory);
38 
39 // Returns null or a pointer to at least `payload_size` (which can be zero)
40 // bytes of newly allocated memory, aligned to the larger of HWY_ALIGNMENT and
41 // the vector size. Calls `alloc` with the passed `opaque` pointer to obtain
42 // memory or malloc() if it is null.
43 HWY_DLLEXPORT void* AllocateAlignedBytes(size_t payload_size,
44  AllocPtr alloc_ptr, void* opaque_ptr);
45 
46 // Frees all memory. No effect if `aligned_pointer` == nullptr, otherwise it
47 // must have been returned from a previous call to `AllocateAlignedBytes`.
48 // Calls `free_ptr` with the passed `opaque_ptr` pointer to free the memory; if
49 // `free_ptr` function is null, uses the default free().
50 HWY_DLLEXPORT void FreeAlignedBytes(const void* aligned_pointer,
51  FreePtr free_ptr, void* opaque_ptr);
52 
53 // Class that deletes the aligned pointer passed to operator() calling the
54 // destructor before freeing the pointer. This is equivalent to the
55 // std::default_delete but for aligned objects. For a similar deleter equivalent
56 // to free() for aligned memory see AlignedFreer().
58  public:
59  AlignedDeleter() : free_(nullptr), opaque_ptr_(nullptr) {}
60  AlignedDeleter(FreePtr free_ptr, void* opaque_ptr)
61  : free_(free_ptr), opaque_ptr_(opaque_ptr) {}
62 
63  template <typename T>
64  void operator()(T* aligned_pointer) const {
65  return DeleteAlignedArray(aligned_pointer, free_, opaque_ptr_,
66  TypedArrayDeleter<T>);
67  }
68 
69  private:
70  template <typename T>
71  static void TypedArrayDeleter(void* ptr, size_t size_in_bytes) {
72  size_t elems = size_in_bytes / sizeof(T);
73  for (size_t i = 0; i < elems; i++) {
74  // Explicitly call the destructor on each element.
75  (static_cast<T*>(ptr) + i)->~T();
76  }
77  }
78 
79  // Function prototype that calls the destructor for each element in a typed
80  // array. TypeArrayDeleter<T> would match this prototype.
81  using ArrayDeleter = void (*)(void* t_ptr, size_t t_size);
82 
83  HWY_DLLEXPORT static void DeleteAlignedArray(void* aligned_pointer,
84  FreePtr free_ptr,
85  void* opaque_ptr,
86  ArrayDeleter deleter);
87 
89  void* opaque_ptr_;
90 };
91 
92 // Unique pointer to T with custom aligned deleter. This can be a single
93 // element U or an array of element if T is a U[]. The custom aligned deleter
94 // will call the destructor on U or each element of a U[] in the array case.
95 template <typename T>
96 using AlignedUniquePtr = std::unique_ptr<T, AlignedDeleter>;
97 
98 // Aligned memory equivalent of make_unique<T> using the custom allocators
99 // alloc/free with the passed `opaque` pointer. This function calls the
100 // constructor with the passed Args... and calls the destructor of the object
101 // when the AlignedUniquePtr is destroyed.
102 template <typename T, typename... Args>
104  void* opaque, Args&&... args) {
105  T* ptr = static_cast<T*>(AllocateAlignedBytes(sizeof(T), alloc, opaque));
106  return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...),
107  AlignedDeleter(free, opaque));
108 }
109 
110 // Similar to MakeUniqueAlignedWithAlloc but using the default alloc/free
111 // functions.
112 template <typename T, typename... Args>
114  T* ptr = static_cast<T*>(AllocateAlignedBytes(
115  sizeof(T), /*alloc_ptr=*/nullptr, /*opaque_ptr=*/nullptr));
116  return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...),
117  AlignedDeleter());
118 }
119 
120 // Helpers for array allocators (avoids overflow)
121 namespace detail {
122 
123 // Returns x such that 1u << x == n (if n is a power of two).
124 static inline constexpr size_t ShiftCount(size_t n) {
125  return (n <= 1) ? 0 : 1 + ShiftCount(n / 2);
126 }
127 
128 template <typename T>
129 T* AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void* opaque_ptr) {
130  constexpr size_t size = sizeof(T);
131 
132  constexpr bool is_pow2 = (size & (size - 1)) == 0;
133  constexpr size_t bits = ShiftCount(size);
134  static_assert(!is_pow2 || (1ull << bits) == size, "ShiftCount is incorrect");
135 
136  const size_t bytes = is_pow2 ? items << bits : items * size;
137  const size_t check = is_pow2 ? bytes >> bits : bytes / size;
138  if (check != items) {
139  return nullptr; // overflowed
140  }
141  return static_cast<T*>(AllocateAlignedBytes(bytes, alloc_ptr, opaque_ptr));
142 }
143 
144 } // namespace detail
145 
146 // Aligned memory equivalent of make_unique<T[]> for array types using the
147 // custom allocators alloc/free. This function calls the constructor with the
148 // passed Args... on every created item. The destructor of each element will be
149 // called when the AlignedUniquePtr is destroyed.
150 template <typename T, typename... Args>
152  size_t items, AllocPtr alloc, FreePtr free, void* opaque, Args&&... args) {
153  T* ptr = detail::AllocateAlignedItems<T>(items, alloc, opaque);
154  if (ptr != nullptr) {
155  for (size_t i = 0; i < items; i++) {
156  new (ptr + i) T(std::forward<Args>(args)...);
157  }
158  }
159  return AlignedUniquePtr<T[]>(ptr, AlignedDeleter(free, opaque));
160 }
161 
162 template <typename T, typename... Args>
163 AlignedUniquePtr<T[]> MakeUniqueAlignedArray(size_t items, Args&&... args) {
164  return MakeUniqueAlignedArrayWithAlloc<T, Args...>(
165  items, nullptr, nullptr, nullptr, std::forward<Args>(args)...);
166 }
167 
168 // Custom deleter for std::unique_ptr equivalent to using free() as a deleter
169 // but for aligned memory.
171  public:
172  // Pass address of this to ctor to skip deleting externally-owned memory.
173  static void DoNothing(void* /*opaque*/, void* /*aligned_pointer*/) {}
174 
175  AlignedFreer() : free_(nullptr), opaque_ptr_(nullptr) {}
176  AlignedFreer(FreePtr free_ptr, void* opaque_ptr)
177  : free_(free_ptr), opaque_ptr_(opaque_ptr) {}
178 
179  template <typename T>
180  void operator()(T* aligned_pointer) const {
181  // TODO(deymo): assert that we are using a POD type T.
182  FreeAlignedBytes(aligned_pointer, free_, opaque_ptr_);
183  }
184 
185  private:
187  void* opaque_ptr_;
188 };
189 
190 // Unique pointer to single POD, or (if T is U[]) an array of POD. For non POD
191 // data use AlignedUniquePtr.
192 template <typename T>
193 using AlignedFreeUniquePtr = std::unique_ptr<T, AlignedFreer>;
194 
195 // Allocate an aligned and uninitialized array of POD values as a unique_ptr.
196 // Upon destruction of the unique_ptr the aligned array will be freed.
197 template <typename T>
199  FreePtr free, void* opaque) {
201  detail::AllocateAlignedItems<T>(items, alloc, opaque),
202  AlignedFreer(free, opaque));
203 }
204 
205 // Same as previous AllocateAligned(), using default allocate/free functions.
206 template <typename T>
208  return AllocateAligned<T>(items, nullptr, nullptr, nullptr);
209 }
210 
211 } // namespace hwy
212 #endif // HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_
Definition: aligned_allocator.h:57
void * opaque_ptr_
Definition: aligned_allocator.h:89
void operator()(T *aligned_pointer) const
Definition: aligned_allocator.h:64
AlignedDeleter(FreePtr free_ptr, void *opaque_ptr)
Definition: aligned_allocator.h:60
AlignedDeleter()
Definition: aligned_allocator.h:59
void(*)(void *t_ptr, size_t t_size) ArrayDeleter
Definition: aligned_allocator.h:81
FreePtr free_
Definition: aligned_allocator.h:88
static HWY_DLLEXPORT void DeleteAlignedArray(void *aligned_pointer, FreePtr free_ptr, void *opaque_ptr, ArrayDeleter deleter)
static void TypedArrayDeleter(void *ptr, size_t size_in_bytes)
Definition: aligned_allocator.h:71
Definition: aligned_allocator.h:170
AlignedFreer()
Definition: aligned_allocator.h:175
void operator()(T *aligned_pointer) const
Definition: aligned_allocator.h:180
AlignedFreer(FreePtr free_ptr, void *opaque_ptr)
Definition: aligned_allocator.h:176
void * opaque_ptr_
Definition: aligned_allocator.h:187
FreePtr free_
Definition: aligned_allocator.h:186
static void DoNothing(void *, void *)
Definition: aligned_allocator.h:173
#define HWY_DLLEXPORT
Definition: highway_export.h:13
T * AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void *opaque_ptr)
Definition: aligned_allocator.h:129
static constexpr size_t ShiftCount(size_t n)
Definition: aligned_allocator.h:124
Definition: aligned_allocator.h:27
AlignedUniquePtr< T > MakeUniqueAligned(Args &&... args)
Definition: aligned_allocator.h:113
std::unique_ptr< T, AlignedDeleter > AlignedUniquePtr
Definition: aligned_allocator.h:96
AlignedUniquePtr< T[]> MakeUniqueAlignedArrayWithAlloc(size_t items, AllocPtr alloc, FreePtr free, void *opaque, Args &&... args)
Definition: aligned_allocator.h:151
HWY_DLLEXPORT void * AllocateAlignedBytes(size_t payload_size, AllocPtr alloc_ptr, void *opaque_ptr)
AlignedUniquePtr< T[]> MakeUniqueAlignedArray(size_t items, Args &&... args)
Definition: aligned_allocator.h:163
void(*)(void *opaque, void *memory) FreePtr
Definition: aligned_allocator.h:37
AlignedFreeUniquePtr< T[]> AllocateAligned(const size_t items, AllocPtr alloc, FreePtr free, void *opaque)
Definition: aligned_allocator.h:198
AlignedUniquePtr< T > MakeUniqueAlignedWithAlloc(AllocPtr alloc, FreePtr free, void *opaque, Args &&... args)
Definition: aligned_allocator.h:103
void *(*)(void *opaque, size_t bytes) AllocPtr
Definition: aligned_allocator.h:36
HWY_DLLEXPORT void FreeAlignedBytes(const void *aligned_pointer, FreePtr free_ptr, void *opaque_ptr)
std::unique_ptr< T, AlignedFreer > AlignedFreeUniquePtr
Definition: aligned_allocator.h:193