. /src /ex /thread_pool.cpp
100.00% Lines (140/140) 100.00% Functions (29/29)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // Copyright (c) 2026 Michael Vandeberg 3   // Copyright (c) 2026 Michael Vandeberg
4   // 4   //
5   // Distributed under the Boost Software License, Version 1.0. (See accompanying 5   // Distributed under the Boost Software License, Version 1.0. (See accompanying
6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7   // 7   //
8   // Official repository: https://github.com/boostorg/capy 8   // Official repository: https://github.com/boostorg/capy
9   // 9   //
10   10  
11   #include <boost/capy/ex/thread_pool.hpp> 11   #include <boost/capy/ex/thread_pool.hpp>
12   #include <boost/capy/continuation.hpp> 12   #include <boost/capy/continuation.hpp>
13   #include <boost/capy/detail/thread_local_ptr.hpp> 13   #include <boost/capy/detail/thread_local_ptr.hpp>
14   #include <boost/capy/ex/frame_allocator.hpp> 14   #include <boost/capy/ex/frame_allocator.hpp>
15   #include <boost/capy/test/thread_name.hpp> 15   #include <boost/capy/test/thread_name.hpp>
16   #include <algorithm> 16   #include <algorithm>
17   #include <atomic> 17   #include <atomic>
18   #include <condition_variable> 18   #include <condition_variable>
19   #include <cstdio> 19   #include <cstdio>
20   #include <mutex> 20   #include <mutex>
21   #include <thread> 21   #include <thread>
22   #include <vector> 22   #include <vector>
23   23  
24   /* 24   /*
25   Thread pool implementation using a shared work queue. 25   Thread pool implementation using a shared work queue.
26   26  
27   Work items are continuations linked via their intrusive next pointer, 27   Work items are continuations linked via their intrusive next pointer,
28   stored in a single queue protected by a mutex. No per-post heap 28   stored in a single queue protected by a mutex. No per-post heap
29   allocation: the continuation is owned by the caller and linked 29   allocation: the continuation is owned by the caller and linked
30   directly. Worker threads wait on a condition_variable until work 30   directly. Worker threads wait on a condition_variable until work
31   is available or stop is requested. 31   is available or stop is requested.
32   32  
33   Threads are started lazily on first post() via std::call_once to avoid 33   Threads are started lazily on first post() via std::call_once to avoid
34   spawning threads for pools that are constructed but never used. Each 34   spawning threads for pools that are constructed but never used. Each
35   thread is named with a configurable prefix plus index for debugger 35   thread is named with a configurable prefix plus index for debugger
36   visibility. 36   visibility.
37   37  
38   Work tracking: on_work_started/on_work_finished maintain the atomic 38   Work tracking: on_work_started/on_work_finished maintain the atomic
39   outstanding_work_ counter. on_work_started is lock-free; the worker 39   outstanding_work_ counter. on_work_started is lock-free; the worker
40   that drives the count to zero takes mutex_ and re-reads the count 40   that drives the count to zero takes mutex_ and re-reads the count
41   before deciding to stop, so the count and the stop decision stay 41   before deciding to stop, so the count and the stop decision stay
42   consistent even if work is started in between. join() blocks until 42   consistent even if work is started in between. join() blocks until
43   this counter reaches zero, then signals workers to stop and joins 43   this counter reaches zero, then signals workers to stop and joins
44   threads. 44   threads.
45   45  
46   Two shutdown paths: 46   Two shutdown paths:
47   - join(): waits for outstanding work to drain, then stops workers. 47   - join(): waits for outstanding work to drain, then stops workers.
48   - stop(): immediately signals workers to exit; queued work is abandoned. 48   - stop(): immediately signals workers to exit; queued work is abandoned.
49   - Destructor: stop() then join() (abandon + wait for threads). 49   - Destructor: stop() then join() (abandon + wait for threads).
50   */ 50   */
51   51  
52   namespace boost { 52   namespace boost {
53   namespace capy { 53   namespace capy {
54   54  
55   //------------------------------------------------------------------------------ 55   //------------------------------------------------------------------------------
56   56  
57   class thread_pool::impl 57   class thread_pool::impl
58   { 58   {
59   // Identifies the pool owning the current worker thread, or 59   // Identifies the pool owning the current worker thread, or
60   // nullptr if the calling thread is not a pool worker. Checked 60   // nullptr if the calling thread is not a pool worker. Checked
61   // by dispatch() to decide between symmetric transfer (inline 61   // by dispatch() to decide between symmetric transfer (inline
62   // resume) and post. 62   // resume) and post.
63   static inline detail::thread_local_ptr<impl const> current_; 63   static inline detail::thread_local_ptr<impl const> current_;
64   64  
65   // Intrusive queue of continuations: the next link is stored in 65   // Intrusive queue of continuations: the next link is stored in
66   // continuation::reserved (typed continuation* round-tripped through 66   // continuation::reserved (typed continuation* round-tripped through
67   // void*). No per-post allocation: the continuation is owned by the caller. 67   // void*). No per-post allocation: the continuation is owned by the caller.
68   continuation* head_ = nullptr; 68   continuation* head_ = nullptr;
69   continuation* tail_ = nullptr; 69   continuation* tail_ = nullptr;
70   70  
HITCBC 71   20394 void push(continuation* c) noexcept 71   20629 void push(continuation* c) noexcept
72   { 72   {
HITCBC 73   20394 c->reserved = nullptr; 73   20629 c->reserved = nullptr;
HITCBC 74   20394 if(tail_) 74   20629 if(tail_)
HITCBC 75   2035 tail_->reserved = c; 75   2242 tail_->reserved = c;
76   else 76   else
HITCBC 77   18359 head_ = c; 77   18387 head_ = c;
HITCBC 78   20394 tail_ = c; 78   20629 tail_ = c;
HITCBC 79   20394 } 79   20629 }
80   80  
HITCBC 81   20683 continuation* pop() noexcept 81   20900 continuation* pop() noexcept
82   { 82   {
HITCBC 83   20683 if(!head_) 83   20900 if(!head_)
HITCBC 84   289 return nullptr; 84   271 return nullptr;
HITCBC 85   20394 continuation* c = head_; 85   20629 continuation* c = head_;
HITCBC 86   20394 head_ = static_cast<continuation*>(head_->reserved); 86   20629 head_ = static_cast<continuation*>(head_->reserved);
HITCBC 87   20394 if(!head_) 87   20629 if(!head_)
HITCBC 88   18359 tail_ = nullptr; 88   18387 tail_ = nullptr;
HITCBC 89   20394 return c; 89   20629 return c;
90   } 90   }
91   91  
HITCBC 92   39874 bool empty() const noexcept 92   40423 bool empty() const noexcept
93   { 93   {
HITCBC 94   39874 return head_ == nullptr; 94   40423 return head_ == nullptr;
95   } 95   }
96   96  
97   std::mutex mutex_; 97   std::mutex mutex_;
98   std::condition_variable work_cv_; 98   std::condition_variable work_cv_;
99   std::condition_variable done_cv_; 99   std::condition_variable done_cv_;
100   std::vector<std::thread> threads_; 100   std::vector<std::thread> threads_;
101   std::atomic<std::size_t> outstanding_work_{0}; 101   std::atomic<std::size_t> outstanding_work_{0};
102   bool stop_{false}; 102   bool stop_{false};
103   bool joined_{false}; 103   bool joined_{false};
104   std::size_t num_threads_; 104   std::size_t num_threads_;
105   char thread_name_prefix_[13]{}; // 12 chars max + null terminator 105   char thread_name_prefix_[13]{}; // 12 chars max + null terminator
106   std::once_flag start_flag_; 106   std::once_flag start_flag_;
107   107  
108   public: 108   public:
HITCBC 109   289 ~impl() = default; 109   271 ~impl() = default;
110   110  
111   bool 111   bool
HITCBC 112   472 running_in_this_thread() const noexcept 112   454 running_in_this_thread() const noexcept
113   { 113   {
HITCBC 114   472 return current_.get() == this; 114   454 return current_.get() == this;
115   } 115   }
116   116  
117   // Destroy abandoned coroutine frames. Must be called 117   // Destroy abandoned coroutine frames. Must be called
118   // before execution_context::shutdown()/destroy() so 118   // before execution_context::shutdown()/destroy() so
119   // that suspended-frame destructors touching services 119   // that suspended-frame destructors touching services
120   // (e.g. cancelling registrations) run while those 120   // (e.g. cancelling registrations) run while those
121   // services are still valid. 121   // services are still valid.
122   void 122   void
HITCBC 123   289 drain_abandoned() noexcept 123   271 drain_abandoned() noexcept
124   { 124   {
HITCBC 125   492 while(auto* c = pop()) 125   477 while(auto* c = pop())
126   { 126   {
HITCBC 127   203 auto h = c->h; 127   206 auto h = c->h;
HITCBC 128   203 if(h && h != std::noop_coroutine()) 128   206 if(h && h != std::noop_coroutine())
HITCBC 129   151 h.destroy(); 129   155 h.destroy();
HITCBC 130   203 } 130   206 }
HITCBC 131   289 } 131   271 }
132   132  
HITCBC 133   289 impl(std::size_t num_threads, std::string_view thread_name_prefix) 133   271 impl(std::size_t num_threads, std::string_view thread_name_prefix)
HITCBC 134   289 : num_threads_(num_threads) 134   271 : num_threads_(num_threads)
135   { 135   {
HITCBC 136   289 if(num_threads_ == 0) 136   271 if(num_threads_ == 0)
HITCBC 137   4 num_threads_ = std::max( 137   4 num_threads_ = std::max(
HITCBC 138   2 std::thread::hardware_concurrency(), 1u); 138   2 std::thread::hardware_concurrency(), 1u);
139   139  
140   // Truncate prefix to 12 chars, leaving room for up to 3-digit index. 140   // Truncate prefix to 12 chars, leaving room for up to 3-digit index.
HITCBC 141   289 auto n = thread_name_prefix.copy(thread_name_prefix_, 12); 141   271 auto n = thread_name_prefix.copy(thread_name_prefix_, 12);
HITCBC 142   289 thread_name_prefix_[n] = '\0'; 142   271 thread_name_prefix_[n] = '\0';
HITCBC 143   289 } 143   271 }
144   144  
145   void 145   void
HITCBC 146   20394 post(continuation& c) 146   20629 post(continuation& c)
147   { 147   {
HITCBC 148   20394 ensure_started(); 148   20629 ensure_started();
149   { 149   {
HITCBC 150   20394 std::lock_guard<std::mutex> lock(mutex_); 150   20629 std::lock_guard<std::mutex> lock(mutex_);
HITCBC 151   20394 push(&c); 151   20629 push(&c);
HITCBC 152   20394 } 152   20629 }
HITCBC 153   20394 work_cv_.notify_one(); 153   20629 work_cv_.notify_one();
HITCBC 154   20394 } 154   20629 }
155   155  
156   void 156   void
HITCBC 157   472 on_work_started() noexcept 157   454 on_work_started() noexcept
158   { 158   {
HITCBC 159   472 outstanding_work_.fetch_add(1, std::memory_order_acq_rel); 159   454 outstanding_work_.fetch_add(1, std::memory_order_acq_rel);
HITCBC 160   472 } 160   454 }
161   161  
162   void 162   void
HITCBC 163   472 on_work_finished() noexcept 163   454 on_work_finished() noexcept
164   { 164   {
HITCBC 165   472 if(outstanding_work_.fetch_sub( 165   454 if(outstanding_work_.fetch_sub(
HITCBC 166   472 1, std::memory_order_acq_rel) == 1) 166   454 1, std::memory_order_acq_rel) == 1)
167   { 167   {
168   // fetch_sub's result can be stale: a concurrent 168   // fetch_sub's result can be stale: a concurrent
169   // on_work_started() may raise the count before we take the 169   // on_work_started() may raise the count before we take the
170   // lock, so re-read it here rather than trust the decrement. 170   // lock, so re-read it here rather than trust the decrement.
HITCBC 171   220 std::lock_guard<std::mutex> lock(mutex_); 171   202 std::lock_guard<std::mutex> lock(mutex_);
HITCBC 172   220 if(outstanding_work_.load( 172   202 if(outstanding_work_.load(
HITCBC 173   220 std::memory_order_acquire) == 0 && joined_ && !stop_) 173   202 std::memory_order_acquire) == 0 && joined_ && !stop_)
174   { 174   {
HITCBC 175   83 stop_ = true; 175   74 stop_ = true;
HITCBC 176   83 done_cv_.notify_all(); 176   74 done_cv_.notify_all();
HITCBC 177   83 work_cv_.notify_all(); 177   74 work_cv_.notify_all();
178   } 178   }
HITCBC 179   220 } 179   202 }
HITCBC 180   472 } 180   454 }
181   181  
182   void 182   void
HITCBC 183   448 join() noexcept 183   412 join() noexcept
184   { 184   {
185   { 185   {
HITCBC 186   448 std::unique_lock<std::mutex> lock(mutex_); 186   412 std::unique_lock<std::mutex> lock(mutex_);
HITCBC 187   448 if(joined_) 187   412 if(joined_)
HITCBC 188   159 return; 188   141 return;
HITCBC 189   289 joined_ = true; 189   271 joined_ = true;
190   190  
HITCBC 191   289 if(outstanding_work_.load( 191   271 if(outstanding_work_.load(
HITCBC 192   289 std::memory_order_acquire) == 0) 192   271 std::memory_order_acquire) == 0)
193   { 193   {
HITCBC 194   154 stop_ = true; 194   143 stop_ = true;
HITCBC 195   154 work_cv_.notify_all(); 195   143 work_cv_.notify_all();
196   } 196   }
197   else 197   else
198   { 198   {
HITCBC 199   135 done_cv_.wait(lock, [this]{ 199   128 done_cv_.wait(lock, [this]{
HITCBC 200   219 return stop_; 200   203 return stop_;
201   }); 201   });
202   } 202   }
HITCBC 203   448 } 203   412 }
204   204  
HITCBC 205   612 for(auto& t : threads_) 205   576 for(auto& t : threads_)
HITCBC 206   323 if(t.joinable()) 206   305 if(t.joinable())
HITCBC 207   323 t.join(); 207   305 t.join();
208   } 208   }
209   209  
210   void 210   void
HITCBC 211   291 stop() noexcept 211   273 stop() noexcept
212   { 212   {
213   { 213   {
HITCBC 214   291 std::lock_guard<std::mutex> lock(mutex_); 214   273 std::lock_guard<std::mutex> lock(mutex_);
HITCBC 215   291 stop_ = true; 215   273 stop_ = true;
HITCBC 216   291 } 216   273 }
HITCBC 217   291 work_cv_.notify_all(); 217   273 work_cv_.notify_all();
HITCBC 218   291 done_cv_.notify_all(); 218   273 done_cv_.notify_all();
HITCBC 219   291 } 219   273 }
220   220  
221   private: 221   private:
222   void 222   void
HITCBC 223   20394 ensure_started() 223   20629 ensure_started()
224   { 224   {
HITCBC 225   20394 std::call_once(start_flag_, [this]{ 225   20629 std::call_once(start_flag_, [this]{
HITCBC 226   242 threads_.reserve(num_threads_); 226   224 threads_.reserve(num_threads_);
HITCBC 227   565 for(std::size_t i = 0; i < num_threads_; ++i) 227   529 for(std::size_t i = 0; i < num_threads_; ++i)
HITCBC 228   646 threads_.emplace_back([this, i]{ run(i); }); 228   610 threads_.emplace_back([this, i]{ run(i); });
HITCBC 229   242 }); 229   224 });
HITCBC 230   20394 } 230   20629 }
231   231  
232   void 232   void
HITCBC 233   323 run(std::size_t index) 233   305 run(std::size_t index)
234   { 234   {
235   // Build name; set_current_thread_name truncates to platform limits. 235   // Build name; set_current_thread_name truncates to platform limits.
236   char name[16]; 236   char name[16];
HITCBC 237   323 std::snprintf(name, sizeof(name), "%s%zu", thread_name_prefix_, index); 237   305 std::snprintf(name, sizeof(name), "%s%zu", thread_name_prefix_, index);
HITCBC 238   323 set_current_thread_name(name); 238   305 set_current_thread_name(name);
239   239  
240   // Mark this thread as a worker of this pool so dispatch() 240   // Mark this thread as a worker of this pool so dispatch()
241   // can symmetric-transfer when called from within pool work. 241   // can symmetric-transfer when called from within pool work.
242   struct scoped_pool 242   struct scoped_pool
243   { 243   {
HITCBC 244   323 scoped_pool(impl const* p) noexcept { current_.set(p); } 244   305 scoped_pool(impl const* p) noexcept { current_.set(p); }
HITCBC 245   323 ~scoped_pool() noexcept { current_.set(nullptr); } 245   305 ~scoped_pool() noexcept { current_.set(nullptr); }
HITCBC 246   323 } guard(this); 246   305 } guard(this);
247   247  
248   for(;;) 248   for(;;)
249   { 249   {
HITCBC 250   20514 continuation* c = nullptr; 250   20728 continuation* c = nullptr;
251   { 251   {
HITCBC 252   20514 std::unique_lock<std::mutex> lock(mutex_); 252   20728 std::unique_lock<std::mutex> lock(mutex_);
HITCBC 253   20514 work_cv_.wait(lock, [this]{ 253   20728 work_cv_.wait(lock, [this]{
HITCBC 254   59458 return !empty() || 254   60328 return !empty() ||
HITCBC 255   59458 stop_; 255   60328 stop_;
256   }); 256   });
HITCBC 257   20514 if(stop_) 257   20728 if(stop_)
HITCBC 258   646 return; 258   610 return;
HITCBC 259   20191 c = pop(); 259   20423 c = pop();
HITCBC 260   20514 } 260   20728 }
HITCBC 261   20191 if(c) 261   20423 if(c)
HITCBC 262   20191 safe_resume(c->h); 262   20423 safe_resume(c->h);
HITCBC 263   20191 } 263   20423 }
HITCBC 264   323 } 264   305 }
265   }; 265   };
266   266  
267   //------------------------------------------------------------------------------ 267   //------------------------------------------------------------------------------
268   268  
HITCBC 269   289 thread_pool:: 269   271 thread_pool::
270   ~thread_pool() 270   ~thread_pool()
271   { 271   {
HITCBC 272   289 impl_->stop(); 272   271 impl_->stop();
HITCBC 273   289 impl_->join(); 273   271 impl_->join();
HITCBC 274   289 impl_->drain_abandoned(); 274   271 impl_->drain_abandoned();
HITCBC 275   289 shutdown(); 275   271 shutdown();
HITCBC 276   289 destroy(); 276   271 destroy();
HITCBC 277   289 delete impl_; 277   271 delete impl_;
HITCBC 278   289 } 278   271 }
279   279  
HITCBC 280   289 thread_pool:: 280   271 thread_pool::
HITCBC 281   289 thread_pool(std::size_t num_threads, std::string_view thread_name_prefix) 281   271 thread_pool(std::size_t num_threads, std::string_view thread_name_prefix)
HITCBC 282   289 : impl_(new impl(num_threads, thread_name_prefix)) 282   271 : impl_(new impl(num_threads, thread_name_prefix))
283   { 283   {
HITCBC 284   289 this->set_frame_allocator(std::allocator<void>{}); 284   271 this->set_frame_allocator(std::allocator<void>{});
HITCBC 285   289 } 285   271 }
286   286  
287   void 287   void
HITCBC 288   159 thread_pool:: 288   141 thread_pool::
289   join() noexcept 289   join() noexcept
290   { 290   {
HITCBC 291   159 impl_->join(); 291   141 impl_->join();
HITCBC 292   159 } 292   141 }
293   293  
294   void 294   void
HITCBC 295   2 thread_pool:: 295   2 thread_pool::
296   stop() noexcept 296   stop() noexcept
297   { 297   {
HITCBC 298   2 impl_->stop(); 298   2 impl_->stop();
HITCBC 299   2 } 299   2 }
300   300  
301   //------------------------------------------------------------------------------ 301   //------------------------------------------------------------------------------
302   302  
303   thread_pool::executor_type 303   thread_pool::executor_type
HITCBC 304   11695 thread_pool:: 304   11677 thread_pool::
305   get_executor() const noexcept 305   get_executor() const noexcept
306   { 306   {
HITCBC 307   11695 return executor_type( 307   11677 return executor_type(
HITCBC 308   11695 const_cast<thread_pool&>(*this)); 308   11677 const_cast<thread_pool&>(*this));
309   } 309   }
310   310  
311   void 311   void
HITCBC 312   472 thread_pool::executor_type:: 312   454 thread_pool::executor_type::
313   on_work_started() const noexcept 313   on_work_started() const noexcept
314   { 314   {
HITCBC 315   472 pool_->impl_->on_work_started(); 315   454 pool_->impl_->on_work_started();
HITCBC 316   472 } 316   454 }
317   317  
318   void 318   void
HITCBC 319   472 thread_pool::executor_type:: 319   454 thread_pool::executor_type::
320   on_work_finished() const noexcept 320   on_work_finished() const noexcept
321   { 321   {
HITCBC 322   472 pool_->impl_->on_work_finished(); 322   454 pool_->impl_->on_work_finished();
HITCBC 323   472 } 323   454 }
324   324  
325   void 325   void
HITCBC 326   19927 thread_pool::executor_type:: 326   20180 thread_pool::executor_type::
327   post(continuation& c) const 327   post(continuation& c) const
328   { 328   {
HITCBC 329   19927 pool_->impl_->post(c); 329   20180 pool_->impl_->post(c);
HITCBC 330   19927 } 330   20180 }
331   331  
332   std::coroutine_handle<> 332   std::coroutine_handle<>
HITCBC 333   472 thread_pool::executor_type:: 333   454 thread_pool::executor_type::
334   dispatch(continuation& c) const 334   dispatch(continuation& c) const
335   { 335   {
HITCBC 336   472 if(pool_->impl_->running_in_this_thread()) 336   454 if(pool_->impl_->running_in_this_thread())
HITCBC 337   5 return c.h; 337   5 return c.h;
HITCBC 338   467 pool_->impl_->post(c); 338   449 pool_->impl_->post(c);
HITCBC 339   467 return std::noop_coroutine(); 339   449 return std::noop_coroutine();
340   } 340   }
341   341  
342   } // capy 342   } // capy
343   } // boost 343   } // boost