LiVES  3.2.0
rpmalloc.h
Go to the documentation of this file.
1 /* rpmalloc.h - Memory allocator - Public Domain - 2016 Mattias Jansson
2 
3  This library provides a cross-platform lock free thread caching malloc implementation in C11.
4  The latest source code is always available at
5 
6  https://github.com/mjansson/rpmalloc
7 
8  This library is put in the public domain; you can redistribute it and/or modify it without any restrictions.
9 
10 */
11 
12 #pragma once
13 
14 #include <stddef.h>
15 
16 #ifdef __cplusplus
17 extern "C" {
18 #endif
19 
20 #if defined(__clang__) || defined(__GNUC__)
21 # define RPMALLOC_EXPORT __attribute__((visibility("default")))
22 # define RPMALLOC_ALLOCATOR
23 # if (defined(__clang_major__) && (__clang_major__ < 4)) || (defined(__GNUC__) && defined(ENABLE_PRELOAD) && ENABLE_PRELOAD)
24 # define RPMALLOC_ATTRIB_MALLOC
25 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
26 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)
27 # else
28 # define RPMALLOC_ATTRIB_MALLOC __attribute__((__malloc__))
29 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) __attribute__((alloc_size(size)))
30 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) __attribute__((alloc_size(count, size)))
31 # endif
32 # define RPMALLOC_CDECL
33 #elif defined(_MSC_VER)
34 # define RPMALLOC_EXPORT
35 # define RPMALLOC_ALLOCATOR __declspec(allocator) __declspec(restrict)
36 # define RPMALLOC_ATTRIB_MALLOC
37 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
38 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size)
39 # define RPMALLOC_CDECL __cdecl
40 #else
41 # define RPMALLOC_EXPORT
42 # define RPMALLOC_ALLOCATOR
43 # define RPMALLOC_ATTRIB_MALLOC
44 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
45 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size)
46 # define RPMALLOC_CDECL
47 #endif
48 
50 // a very small overhead due to some size calculations not being compile time constants
51 #ifndef RPMALLOC_CONFIGURABLE
52 #define RPMALLOC_CONFIGURABLE 0
53 #endif
54 
56 // Will introduce a very small overhead to track fully allocated spans in heaps
57 #ifndef RPMALLOC_FIRST_CLASS_HEAPS
58 #define RPMALLOC_FIRST_CLASS_HEAPS 0
59 #endif
60 
62 #define RPMALLOC_NO_PRESERVE 1
63 // in which case the original pointer is still valid (just like a call to realloc which failes to allocate
65 // a new block).
66 #define RPMALLOC_GROW_OR_FAIL 2
67 
70  size_t mapped;
72  size_t mapped_peak;
74  size_t cached;
76  size_t huge_alloc;
80  size_t mapped_total;
84 
87  size_t sizecache;
89  size_t spancache;
95  struct {
97  size_t current;
99  size_t peak;
101  size_t to_global;
103  size_t from_global;
105  size_t to_cache;
107  size_t from_cache;
109  size_t to_reserved;
113  size_t map_calls;
114  } span_use[32];
116  struct {
120  size_t alloc_peak;
122  size_t alloc_total;
124  size_t free_total;
132  size_t map_calls;
133  } size_use[128];
135 
136 typedef struct rpmalloc_config_t {
138  // aligned to the rpmalloc span size, which will always be a power of two.
139  // Optionally the function can store an alignment offset in the offset variable
140  // in case it performs alignment and the returned pointer is offset from the
141  // actual start of the memory region due to this alignment. The alignment offset
142  // will be passed to the memory unmap function. The alignment offset MUST NOT be
143  // larger than 65535 (storable in an uint16_t), if it is you must use natural
144  // alignment to shift it into 16 bits. If you set a memory_map function, you
145  // must also set a memory_unmap function or else the default implementation will
146  // be used for both.
147  void *(*memory_map)(size_t size, size_t *offset);
149  // If release is set to non-zero, the unmap is for an entire span range as returned by
150  // a previous call to memory_map and that the entire range should be released. The
151  // release argument holds the size of the entire span range. If release is set to 0,
152  // the unmap is a partial decommit of a subset of the mapped memory range.
153  // If you set a memory_unmap function, you must also set a memory_map function or
154  // else the default implementation will be used for both.
155  void (*memory_unmap)(void *address, size_t size, size_t offset, size_t release);
157  // requests to memory_map will be made with size set to a multiple of the page size.
158  // Used if RPMALLOC_CONFIGURABLE is defined to 1, otherwise system page size is used.
159  size_t page_size;
161  // range (unless 0 - set to 0 to use the default span size). Used if RPMALLOC_CONFIGURABLE
162  // is defined to 1.
163  size_t span_size;
165  // be used to minimize the system call overhead at the cost of virtual memory address
166  // space. The extra mapped pages will not be written until actually used, so physical
167  // committed memory should not be affected in the default implementation. Will be
168  // aligned to a multiple of spans that match memory page size in case of huge pages.
171  // zero, the allocator will try to enable huge pages and auto detect the configuration.
172  // If this is set to non-zero and page_size is also non-zero, the allocator will
173  // assume huge pages have been configured and enabled prior to initializing the
174  // allocator.
175  // For Windows, see https://docs.microsoft.com/en-us/windows/desktop/memory/large-page-support
176  // For Linux, see https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
179 
181 RPMALLOC_EXPORT int
182 rpmalloc_initialize(void);
183 
185 RPMALLOC_EXPORT int
187 
190 rpmalloc_config(void);
191 
193 RPMALLOC_EXPORT void
194 rpmalloc_finalize(void);
195 
197 RPMALLOC_EXPORT void
199 
201 RPMALLOC_EXPORT void
203 
205 RPMALLOC_EXPORT void
207 
209 RPMALLOC_EXPORT int
211 
213 RPMALLOC_EXPORT void
215 
217 RPMALLOC_EXPORT void
219 
221 RPMALLOC_EXPORT void
222 rpmalloc_dump_statistics(void *file);
223 
227 
229 RPMALLOC_EXPORT void
230 rpfree(void *ptr);
231 
235 
239 
241 // with optional control flags (see RPMALLOC_NO_PRESERVE).
242 // Alignment must be a power of two and a multiple of sizeof(void*),
243 // and should ideally be less than memory page size. A caveat of rpmalloc
244 // internals is that this must also be strictly less than the span size (default 64KiB)
246 rpaligned_realloc(void *ptr, size_t alignment, size_t size, size_t oldsize,
247  unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3);
248 
250 // Alignment must be a power of two and a multiple of sizeof(void*),
251 // and should ideally be less than memory page size. A caveat of rpmalloc
252 // internals is that this must also be strictly less than the span size (default 64KiB)
254 rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
255 
257 // Alignment must be a power of two and a multiple of sizeof(void*),
258 // and should ideally be less than memory page size. A caveat of rpmalloc
259 // internals is that this must also be strictly less than the span size (default 64KiB)
261 rpaligned_calloc(size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
262 
264 // Alignment must be a power of two and a multiple of sizeof(void*),
265 // and should ideally be less than memory page size. A caveat of rpmalloc
266 // internals is that this must also be strictly less than the span size (default 64KiB)
268 rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
269 
271 // Alignment must be a power of two and a multiple of sizeof(void*),
272 // and should ideally be less than memory page size. A caveat of rpmalloc
273 // internals is that this must also be strictly less than the span size (default 64KiB)
274 RPMALLOC_EXPORT int
275 rpposix_memalign(void **memptr, size_t alignment, size_t size);
276 
278 RPMALLOC_EXPORT size_t
279 rpmalloc_usable_size(void *ptr);
280 
281 #if RPMALLOC_FIRST_CLASS_HEAPS
282 
284 typedef struct heap_t rpmalloc_heap_t;
285 
287 // if none available. Heap API is imlemented with the strict assumption that only one single
288 // thread will call heap functions for a given heap at any given time, no functions are thread safe.
289 RPMALLOC_EXPORT rpmalloc_heap_t *
290 rpmalloc_heap_acquire(void);
291 
293 // Releasing a heap will enable it to be reused by other threads. Safe to pass a null pointer.
294 RPMALLOC_EXPORT void
295 rpmalloc_heap_release(rpmalloc_heap_t *heap);
296 
299 rpmalloc_heap_alloc(rpmalloc_heap_t *heap, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2);
300 
302 // block will have the requested alignment. Alignment must be a power of two and a multiple of sizeof(void*),
303 // and should ideally be less than memory page size. A caveat of rpmalloc
304 // internals is that this must also be strictly less than the span size (default 64KiB).
306 rpmalloc_heap_aligned_alloc(rpmalloc_heap_t *heap, size_t alignment,
308 
311 rpmalloc_heap_calloc(rpmalloc_heap_t *heap, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3);
312 
314 // block will have the requested alignment. Alignment must either be zero, or a power of two and a multiple of sizeof(void*),
315 // and should ideally be less than memory page size. A caveat of rpmalloc
316 // internals is that this must also be strictly less than the span size (default 64KiB).
318 rpmalloc_heap_aligned_calloc(rpmalloc_heap_t *heap, size_t alignment, size_t num,
320 
322 // by the same heap given to this function.
324 rpmalloc_heap_realloc(rpmalloc_heap_t *heap, void *ptr, size_t size,
325  unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3);
326 
328 // by the same heap given to this function. The returned block will have the requested alignment.
329 // Alignment must be either zero, or a power of two and a multiple of sizeof(void*), and should ideally be
330 // less than memory page size. A caveat of rpmalloc internals is that this must also be strictly less than
331 // the span size (default 64KiB).
333 rpmalloc_heap_aligned_realloc(rpmalloc_heap_t *heap, void *ptr, size_t alignment, size_t size,
334  unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3);
335 
337 // by the same heap given to this function.
338 RPMALLOC_EXPORT void
339 rpmalloc_heap_free(rpmalloc_heap_t *heap, void *ptr);
340 
342 RPMALLOC_EXPORT void
343 rpmalloc_heap_free_all(rpmalloc_heap_t *heap);
344 
346 // for a single thread, a heap can never be shared between multiple threads. The previous
347 // current heap for the calling thread is released to be reused by other threads.
348 RPMALLOC_EXPORT void
349 rpmalloc_heap_thread_set_current(rpmalloc_heap_t *heap);
350 
351 #endif
352 
353 #ifdef __cplusplus
354 }
355 #endif
rpmalloc_thread_statistics_t
Definition: rpmalloc.h:85
rpmalloc_thread_finalize
RPMALLOC_EXPORT void rpmalloc_thread_finalize(void)
Finalize allocator for calling thread.
Definition: rpmalloc.c:2577
rpposix_memalign
RPMALLOC_EXPORT int rpposix_memalign(void **memptr, size_t alignment, size_t size)
Allocate a memory block of at least the given size and alignment.
Definition: rpmalloc.c:2706
rpmalloc_thread_statistics_t::spans_to_cache
size_t spans_to_cache
Number of spans transitioned to cache.
Definition: rpmalloc.h:126
rpfree
RPMALLOC_EXPORT void rpfree(void *ptr)
Free the given memory block.
Definition: rpmalloc.c:2612
rpmalloc_initialize
RPMALLOC_EXPORT int rpmalloc_initialize(void)
Initialize allocator with default configuration.
Definition: rpmalloc.c:2318
rpmalloc_finalize
RPMALLOC_EXPORT void rpmalloc_finalize(void)
Finalize allocator.
Definition: rpmalloc.c:2522
rpmalloc_global_statistics
RPMALLOC_EXPORT void rpmalloc_global_statistics(rpmalloc_global_statistics_t *stats)
Get global statistics.
Definition: rpmalloc.c:2787
rpmalloc_thread_statistics_t::free_total
size_t free_total
Total number of frees.
Definition: rpmalloc.h:124
rpmalloc_global_statistics_t::mapped_peak
size_t mapped_peak
Peak amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_STAT...
Definition: rpmalloc.h:72
rpmalloc_thread_statistics_t::from_global
size_t from_global
Number of spans transitioned from global cache.
Definition: rpmalloc.h:103
rpmalloc
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rpmalloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1)
Allocate a memory block of at least the given size.
Definition: rpmalloc.c:2600
rpmalloc_global_statistics_t::unmapped_total
size_t unmapped_total
Total amount of memory unmapped since initialization (only if ENABLE_STATISTICS=1)
Definition: rpmalloc.h:82
rpmalloc_thread_statistics_t::to_reserved
size_t to_reserved
Number of spans transitioned to reserved state.
Definition: rpmalloc.h:109
rpmalloc_thread_statistics_t::global_to_thread
size_t global_to_thread
Total number of bytes transitioned from global cache to thread cache (only if ENABLE_STATISTICS=1)
Definition: rpmalloc.h:93
rpmalloc_thread_statistics_t::span_use
struct rpmalloc_thread_statistics_t::@0 span_use[32]
Per span count statistics (only if ENABLE_STATISTICS=1)
rpmalloc_config
RPMALLOC_EXPORT const rpmalloc_config_t * rpmalloc_config(void)
Get allocator configuration.
Definition: rpmalloc.c:2593
rpmalloc_thread_statistics_t::current
size_t current
Currently used number of spans.
Definition: rpmalloc.h:97
rpmalloc_thread_statistics_t::map_calls
size_t map_calls
Number of raw memory map calls (not hitting the reserve spans but resulting in actual OS mmap calls)
Definition: rpmalloc.h:113
rpmalloc_config_t
Definition: rpmalloc.h:136
rpmalloc_thread_statistics_t::spans_from_reserved
size_t spans_from_reserved
Number of spans transitioned from reserved state.
Definition: rpmalloc.h:130
rpmalloc_config_t::page_size
size_t page_size
Size of memory pages. The page size MUST be a power of two. All memory mapping.
Definition: rpmalloc.h:159
rpmalloc_thread_statistics_t::sizecache
size_t sizecache
Current number of bytes available in thread size class caches for small and medium sizes (<32KiB)
Definition: rpmalloc.h:87
rpaligned_calloc
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rpaligned_calloc(size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2
Allocate a memory block of at least the given size and alignment, and zero initialize it.
rpmemalign
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2)
Allocate a memory block of at least the given size and alignment.
Definition: rpmalloc.c:2701
rpmalloc_thread_statistics
RPMALLOC_EXPORT void rpmalloc_thread_statistics(rpmalloc_thread_statistics_t *stats)
Get per-thread statistics.
Definition: rpmalloc.c:2724
rpmalloc_global_statistics_t::huge_alloc
size_t huge_alloc
Current amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2Mi...
Definition: rpmalloc.h:76
RPMALLOC_ATTRIB_ALLOC_SIZE
#define RPMALLOC_ATTRIB_ALLOC_SIZE(size)
Definition: rpmalloc.h:44
RPMALLOC_EXPORT
#define RPMALLOC_EXPORT
Definition: rpmalloc.h:41
rpmalloc_thread_statistics_t::spancache
size_t spancache
Current number of bytes available in thread span caches for small and medium sizes (<32KiB)
Definition: rpmalloc.h:89
rpmalloc_config_t::enable_huge_pages
int enable_huge_pages
Enable use of large/huge pages. If this flag is set to non-zero and page size is.
Definition: rpmalloc.h:177
rpmalloc_thread_statistics_t::alloc_peak
size_t alloc_peak
Peak number of allocations.
Definition: rpmalloc.h:120
rpmalloc_global_statistics_t::huge_alloc_peak
size_t huge_alloc_peak
Peak amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB b...
Definition: rpmalloc.h:78
rpmalloc_is_thread_initialized
RPMALLOC_EXPORT int rpmalloc_is_thread_initialized(void)
Query if allocator is initialized for calling thread.
Definition: rpmalloc.c:2588
rpmalloc_thread_statistics_t::spans_from_cache
size_t spans_from_cache
Number of spans transitioned from cache.
Definition: rpmalloc.h:128
rpaligned_realloc
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rpaligned_realloc(void *ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3)
Reallocate the given block to at least the given size and alignment,.
Definition: rpmalloc.c:2656
rpmalloc_global_statistics_t
Definition: rpmalloc.h:68
rpaligned_alloc
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2)
Allocate a memory block of at least the given size and alignment.
Definition: rpmalloc.c:2669
rprealloc
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rprealloc(void *ptr, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2)
Reallocate the given block to at least the given size.
Definition: rpmalloc.c:2644
rpmalloc_global_statistics_t::mapped_total
size_t mapped_total
Total amount of memory mapped since initialization (only if ENABLE_STATISTICS=1)
Definition: rpmalloc.h:80
rpmalloc_usable_size
RPMALLOC_EXPORT size_t rpmalloc_usable_size(void *ptr)
Query the usable size of the given memory block (from given pointer to the end of block)
Definition: rpmalloc.c:2715
rpmalloc_thread_statistics_t::from_cache
size_t from_cache
Number of spans transitioned from thread cache.
Definition: rpmalloc.h:107
rpmalloc_config_t::span_map_count
size_t span_map_count
Number of spans to map at each request to map new virtual memory blocks. This can.
Definition: rpmalloc.h:169
RPMALLOC_ALLOCATOR
#define RPMALLOC_ALLOCATOR
Definition: rpmalloc.h:42
rpmalloc_thread_statistics_t::thread_to_global
size_t thread_to_global
Total number of bytes transitioned from thread cache to global cache (only if ENABLE_STATISTICS=1)
Definition: rpmalloc.h:91
rpmalloc_config_t::memory_unmap
void(* memory_unmap)(void *address, size_t size, size_t offset, size_t release)
Unmap the memory pages starting at address and spanning the given number of bytes.
Definition: rpmalloc.h:155
rpcalloc
RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void * rpcalloc(size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1
Allocate a memory block of at least the given size and zero initialize it.
rpmalloc_thread_statistics_t::from_reserved
size_t from_reserved
Number of spans transitioned from reserved state.
Definition: rpmalloc.h:111
rpmalloc_thread_statistics_t::size_use
struct rpmalloc_thread_statistics_t::@1 size_use[128]
Per size class statistics (only if ENABLE_STATISTICS=1)
rpmalloc_thread_statistics_t::to_cache
size_t to_cache
Number of spans transitioned to thread cache.
Definition: rpmalloc.h:105
rpmalloc_dump_statistics
RPMALLOC_EXPORT void rpmalloc_dump_statistics(void *file)
Dump all statistics in human readable format to file (should be a FILE*)
Definition: rpmalloc.c:2861
rpmalloc_global_statistics_t::mapped
size_t mapped
Current amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_S...
Definition: rpmalloc.h:70
rpmalloc_thread_statistics_t::alloc_total
size_t alloc_total
Total number of allocations.
Definition: rpmalloc.h:122
rpmalloc_thread_statistics_t::to_global
size_t to_global
Number of spans transitioned to global cache.
Definition: rpmalloc.h:101
rpmalloc_config_t::span_size
size_t span_size
Size of a span of memory blocks. MUST be a power of two, and in [4096,262144].
Definition: rpmalloc.h:163
RPMALLOC_ATTRIB_MALLOC
#define RPMALLOC_ATTRIB_MALLOC
Definition: rpmalloc.h:43
rpmalloc_thread_statistics_t::peak
size_t peak
High water mark of spans used.
Definition: rpmalloc.h:99
rpmalloc_global_statistics_t::cached
size_t cached
Current amount of memory in global caches for small and medium sizes (<32KiB)
Definition: rpmalloc.h:74
rpmalloc_thread_collect
RPMALLOC_EXPORT void rpmalloc_thread_collect(void)
Perform deferred deallocations pending for the calling thread heap.
Definition: rpmalloc.c:2720
rpmalloc_initialize_config
RPMALLOC_EXPORT int rpmalloc_initialize_config(const rpmalloc_config_t *config)
Initialize allocator with given configuration.
Definition: rpmalloc.c:2327
rpmalloc_thread_initialize
RPMALLOC_EXPORT void rpmalloc_thread_initialize(void)
Initialize allocator for calling thread.
Definition: rpmalloc.c:2562
rpmalloc_thread_statistics_t::alloc_current
size_t alloc_current
Current number of allocations.
Definition: rpmalloc.h:118
RPMALLOC_ATTRIB_ALLOC_SIZE2
#define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size)
Definition: rpmalloc.h:45