Index/include / boost / capy / ex / recycling_memory_resource.hpp

include/boost/capy/ex/recycling_memory_resource.hpp

93.8% Lines (45/48) 100.0% List of functions (9/9)
[ ] prev/next file n p prev/next uncovered line
recycling_memory_resource.hpp
f(x) Functions (9)
Function Calls Lines Blocks
boost::capy::recycling_memory_resource::round_up_pow2(unsigned long) :59 24828x 100.0% 100.0% boost::capy::recycling_memory_resource::get_class_index(unsigned long) :65 24828x 100.0% 86.0% boost::capy::recycling_memory_resource::bucket::pop() :76 16693x 100.0% 100.0% boost::capy::recycling_memory_resource::bucket::pop(boost::capy::recycling_memory_resource::bucket&) :84 6769x 100.0% 100.0% boost::capy::recycling_memory_resource::bucket::push(void*) :95 18741x 100.0% 100.0% boost::capy::recycling_memory_resource::local() :116 31885x 100.0% 100.0% boost::capy::recycling_memory_resource::allocate_fast(unsigned long, unsigned long) :144 12414x 88.9% 83.0% boost::capy::recycling_memory_resource::deallocate_fast(void*, unsigned long, unsigned long) :163 12414x 84.6% 85.0% boost::capy::recycling_memory_resource::do_is_equal(std::pmr::memory_resource const&) const :195 2x 100.0% 100.0%
Line TLA Hits Source Code
1 //
2 // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 // Official repository: https://github.com/cppalliance/capy
8 //
9
10 #ifndef BOOST_CAPY_RECYCLING_MEMORY_RESOURCE_HPP
11 #define BOOST_CAPY_RECYCLING_MEMORY_RESOURCE_HPP
12
13 #include <boost/capy/detail/config.hpp>
14
15 #include <bit>
16 #include <cstddef>
17 #include <memory_resource>
18 #include <mutex>
19
20 namespace boost {
21 namespace capy {
22
23 /** Recycling memory resource with size-class buckets.
24
25 This memory resource recycles memory blocks using power-of-two
26 size classes for O(1) allocation lookup. It maintains a thread-local
27 pool for fast lock-free access and a global pool for cross-thread
28 block sharing.
29
30 Size classes: 64, 128, 256, 512, 1024, 2048 bytes.
31 Allocations larger than 2048 bytes bypass the pools entirely.
32
33 This is the default allocator used by run_async when no allocator
34 is specified.
35
36 @par Thread Safety
37 Thread-safe. The thread-local pool requires no synchronization.
38 The global pool uses a mutex for cross-thread access.
39
40 @par Example
41 @code
42 auto* mr = get_recycling_memory_resource();
43 run_async(ex, mr)(my_task());
44 @endcode
45
46 @see get_recycling_memory_resource
47 @see run_async
48 */
49 BOOST_CAPY_MSVC_WARNING_PUSH
50 BOOST_CAPY_MSVC_WARNING_DISABLE(4275) // non dll-interface base class
51 class BOOST_CAPY_DECL recycling_memory_resource : public std::pmr::memory_resource
52 {
53 static constexpr std::size_t num_classes = 6;
54 static constexpr std::size_t min_class_size = 64; // 2^6
55 static constexpr std::size_t max_class_size = 2048; // 2^11
56 static constexpr std::size_t bucket_capacity = 16;
57
58 static std::size_t
59 24828x round_up_pow2(std::size_t n) noexcept
60 {
61 24828x return n <= min_class_size ? min_class_size : std::bit_ceil(n);
62 }
63
64 static std::size_t
65 24828x get_class_index(std::size_t rounded) noexcept
66 {
67 24828x std::size_t idx = std::countr_zero(rounded) - 6; // 64 = 2^6
68 24828x return idx < num_classes ? idx : num_classes;
69 }
70
71 struct bucket
72 {
73 std::size_t count = 0;
74 void* ptrs[bucket_capacity] = {};
75
76 16693x void* pop() noexcept
77 {
78 16693x if(count == 0)
79 6769x return nullptr;
80 9924x return ptrs[--count];
81 }
82
83 // Peter Dimov's idea
84 6769x void* pop(bucket& b) noexcept
85 {
86 6769x if(count == 0)
87 6168x return nullptr;
88 4998x for(std::size_t i = 0; i < count; ++i)
89 4397x b.ptrs[i] = ptrs[i];
90 601x b.count = count - 1;
91 601x count = 0;
92 601x return b.ptrs[b.count];
93 }
94
95 18741x bool push(void* p) noexcept
96 {
97 18741x if(count >= bucket_capacity)
98 8216x return false;
99 10525x ptrs[count++] = p;
100 10525x return true;
101 }
102 };
103
104 struct pool
105 {
106 bucket buckets[num_classes];
107
108 // No destructor: a non-trivial dtor forces a guard variable on the
109 // thread_local in local(), checked on every alloc/free. Constant
110 // initialization plus a trivial dtor makes that access a bare TLS
111 // load. Cached blocks are instead reclaimed explicitly: per-thread
112 // by arm_thread_cleanup() at thread exit, and the global pool by
113 // global()'s holder destructor at process exit.
114 };
115
116 31885x static pool& local() noexcept
117 {
118 static thread_local pool p;
119 31885x return p;
120 }
121
122 static pool& global() noexcept;
123 static std::mutex& global_mutex() noexcept;
124
125 void* allocate_slow(std::size_t rounded, std::size_t idx);
126 void deallocate_slow(void* p, std::size_t idx);
127
128 // Register a thread-exit callback that drains this thread's local
129 // pool back to the OS. Called only off the hot path: unconditionally
130 // from the slow paths, and once per thread from deallocate_fast
131 // behind a guard-free flag.
132 static void arm_thread_cleanup() noexcept;
133
134 public:
135 ~recycling_memory_resource();
136
137 /** Allocate without virtual dispatch.
138
139 Handles the fast path inline (thread-local bucket pop)
140 and falls through to the slow path for global pool or
141 heap allocation.
142 */
143 void*
144 12414x allocate_fast(std::size_t bytes, std::size_t)
145 {
146 12414x std::size_t rounded = round_up_pow2(bytes);
147 12414x std::size_t idx = get_class_index(rounded);
148 12414x if(idx >= num_classes)
149 ✗ return ::operator new(bytes);
150 12414x auto& lp = local();
151 12414x if(auto* p = lp.buckets[idx].pop())
152 5645x return p;
153 6769x return allocate_slow(rounded, idx);
154 }
155
156 /** Deallocate without virtual dispatch.
157
158 Handles the fast path inline (thread-local bucket push)
159 and falls through to the slow path for global pool or
160 heap deallocation.
161 */
162 void
163 12414x deallocate_fast(void* p, std::size_t bytes, std::size_t)
164 {
165 12414x std::size_t rounded = round_up_pow2(bytes);
166 12414x std::size_t idx = get_class_index(rounded);
167 12414x if(idx >= num_classes)
168 {
169 ✗ ::operator delete(p);
170 ✗ return;
171 }
172 // Guard-free flag (constinit bool, trivial dtor): arms thread-exit
173 // cleanup exactly once for any thread that caches via deallocate,
174 // including consumer threads that never hit a slow path.
175 static thread_local bool armed = false;
176 12414x if(!armed)
177 {
178 282x armed = true;
179 282x arm_thread_cleanup();
180 }
181 12414x auto& lp = local();
182 12414x if(lp.buckets[idx].push(p))
183 6087x return;
184 6327x deallocate_slow(p, idx);
185 }
186
187 protected:
188 void*
189 do_allocate(std::size_t bytes, std::size_t) override;
190
191 void
192 do_deallocate(void* p, std::size_t bytes, std::size_t) override;
193
194 bool
195 2x do_is_equal(const memory_resource& other) const noexcept override
196 {
197 2x return this == &other;
198 }
199 };
200 BOOST_CAPY_MSVC_WARNING_POP
201
202 /** Returns pointer to the default recycling memory resource.
203
204 The returned pointer is valid for the lifetime of the program.
205 This is the default allocator used by run_async.
206
207 @return Pointer to the recycling memory resource.
208
209 @see recycling_memory_resource
210 @see run_async
211 */
212 BOOST_CAPY_DECL
213 std::pmr::memory_resource*
214 get_recycling_memory_resource() noexcept;
215
216 } // namespace capy
217 } // namespace boost
218
219 #endif
220