. /include/boost/corosio /detail /object_pool.hpp
94.20% Lines (65/69) 100.00% Functions (10/10)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co) 2   // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/corosio 7   // Official repository: https://github.com/cppalliance/corosio
8   // 8   //
9   9  
10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 10   #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP 11   #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12   12  
13   #include <boost/corosio/detail/config.hpp> 13   #include <boost/corosio/detail/config.hpp>
14   #include <boost/corosio/detail/intrusive.hpp> 14   #include <boost/corosio/detail/intrusive.hpp>
15   #include <boost/corosio/detail/object_ref.hpp> 15   #include <boost/corosio/detail/object_ref.hpp>
16   #include <atomic> 16   #include <atomic>
17   #include <cstddef> 17   #include <cstddef>
18   #include <initializer_list> 18   #include <initializer_list>
19   #include <limits> 19   #include <limits>
20   #include <mutex> 20   #include <mutex>
21   #include <utility> 21   #include <utility>
22   #include <vector> 22   #include <vector>
23   23  
24   namespace boost::corosio::detail { 24   namespace boost::corosio::detail {
25   25  
26   /** Per-service recycling pool for I/O implementations. 26   /** Per-service recycling pool for I/O implementations.
27   27  
28   Replaces per-impl `shared_ptr` ownership: a service news an impl 28   Replaces per-impl `shared_ptr` ownership: a service news an impl
29   once, then `recycle()` (the impl's `retire()` action) parks 29   once, then `recycle()` (the impl's `retire()` action) parks
30   it on the free list and `acquire()` hands it back out, so steady-state 30   it on the free list and `acquire()` hands it back out, so steady-state
31   construct/destroy allocates nothing. Unbounded; memory is released 31   construct/destroy allocates nothing. Unbounded; memory is released
32   only on pool destruction. Debug poisoning, where an impl does it, 32   only on pool destruction. Debug poisoning, where an impl does it,
33   happens in that impl's `retire()`, not here (the pool cannot know 33   happens in that impl's `retire()`, not here (the pool cannot know
34   which bytes are resettable). 34   which bytes are resettable).
35   35  
36   A service tears down its impls by calling `shutdown()`, which 36   A service tears down its impls by calling `shutdown()`, which
37   enters shutting-down mode and visits every live impl in one 37   enters shutting-down mode and visits every live impl in one
38   critical section; subsequent zero-ref crossings delete instead of 38   critical section; subsequent zero-ref crossings delete instead of
39   recycling. 39   recycling.
40   40  
41   `adopt()` and `remove()` are protected ownership-transfer 41   `adopt()` and `remove()` are protected ownership-transfer
42   primitives for sanctioned extensions — a derived pool exposing 42   primitives for sanctioned extensions — a derived pool exposing
43   them to a caller that tracks some of its live impls through its 43   them to a caller that tracks some of its live impls through its
44   own structure instead of this pool's `live_` list (the timer 44   own structure instead of this pool's `live_` list (the timer
45   service, via its expiry heap) — rather than part of the public 45   service, via its expiry heap) — rather than part of the public
46   surface every ordinary service uses. 46   surface every ordinary service uses.
47   47  
48   @par Thread Safety 48   @par Thread Safety
49   Distinct objects: Safe. Shared objects: Safe; all operations lock 49   Distinct objects: Safe. Shared objects: Safe; all operations lock
50   an internal mutex. 50   an internal mutex.
51   */ 51   */
52   template<class Impl> 52   template<class Impl>
53   class object_pool 53   class object_pool
54   { 54   {
55   std::mutex mutex_; 55   std::mutex mutex_;
56   intrusive_list<Impl> live_; 56   intrusive_list<Impl> live_;
57   intrusive_list<Impl> free_; 57   intrusive_list<Impl> free_;
58   bool shutting_down_ = false; 58   bool shutting_down_ = false;
59   59  
60   /** Marks `refs_` on an impl currently parked in `free_`. 60   /** Marks `refs_` on an impl currently parked in `free_`.
61   61  
62   `intrusive_list::remove()` cannot distinguish "linked in 62   `intrusive_list::remove()` cannot distinguish "linked in
63   `free_`" from "linked in `live_`" by inspecting the node 63   `free_`" from "linked in `live_`" by inspecting the node
64   alone — both lists share one `next_`/`prev_` pair per impl — 64   alone — both lists share one `next_`/`prev_` pair per impl —
65   so `recycle()` cannot safely call `live_.remove()` on an impl 65   so `recycle()` cannot safely call `live_.remove()` on an impl
66   it does not already know is live. Stamping this sentinel into 66   it does not already know is live. Stamping this sentinel into
67   `refs_` the moment an impl joins `free_`, and checking it 67   `refs_` the moment an impl joins `free_`, and checking it
68   before touching either list, is what lets `recycle()` and 68   before touching either list, is what lets `recycle()` and
69   `remove()` tell the two cases apart in every build mode, not 69   `remove()` tell the two cases apart in every build mode, not
70   just under asserts. `acquire()` always overwrites it with 1 70   just under asserts. `acquire()` always overwrites it with 1
71   before the impl is usable again. 71   before the impl is usable again.
72   */ 72   */
73   static constexpr std::size_t pooled_sentinel = pooled_refs; 73   static constexpr std::size_t pooled_sentinel = pooled_refs;
74   74  
75   /// Check whether @p impl is parked in `free_` (`refs_` reads the 75   /// Check whether @p impl is parked in `free_` (`refs_` reads the
76   /// sentinel). Callers must check this before any list operation — 76   /// sentinel). Callers must check this before any list operation —
77   /// see the class-level comment on `pooled_sentinel`. 77   /// see the class-level comment on `pooled_sentinel`.
HITCBC 78   87881 static bool is_pooled(Impl const* impl) noexcept 78   86984 static bool is_pooled(Impl const* impl) noexcept
79   { 79   {
HITCBC 80   87881 return impl->refs_.load(std::memory_order_relaxed) == 80   86984 return impl->refs_.load(std::memory_order_relaxed) ==
HITCBC 81   87881 pooled_sentinel; 81   86984 pooled_sentinel;
82   } 82   }
83   83  
84   protected: 84   protected:
85   /** Track a fresh or reset impl as live. 85   /** Track a fresh or reset impl as live.
86   86  
87   @pre @p impl is not parked in `free_` (`refs_` must not read 87   @pre @p impl is not parked in `free_` (`refs_` must not read
88   `pooled_sentinel`). The same list-corruption hazard 88   `pooled_sentinel`). The same list-corruption hazard
89   `recycle()`'s class-level comment documents applies here: a 89   `recycle()`'s class-level comment documents applies here: a
90   free-listed impl pushed onto `live_` directly, bypassing 90   free-listed impl pushed onto `live_` directly, bypassing
91   `acquire()`, would end up linked in both lists through one shared 91   `acquire()`, would end up linked in both lists through one shared
92   `next_`/`prev_` pair. 92   `next_`/`prev_` pair.
93   93  
94   @pre The caller is not racing `shutdown()`. Every caller adopts 94   @pre The caller is not racing `shutdown()`. Every caller adopts
95   while constructing an object (`acquire()`, or a derived pool's 95   while constructing an object (`acquire()`, or a derived pool's
96   cache handing a parked impl back out), and construction is 96   cache handing a parked impl back out), and construction is
97   expected to stop before the service's own `shutdown()` runs; 97   expected to stop before the service's own `shutdown()` runs;
98   this is checked only in debug builds because the failure mode 98   this is checked only in debug builds because the failure mode
99   is a silently orphaned impl (no close callback), not memory 99   is a silently orphaned impl (no close callback), not memory
100   corruption. 100   corruption.
101   */ 101   */
HITCBC 102   32765 void adopt(Impl* impl) 102   32404 void adopt(Impl* impl)
103   { 103   {
HITCBC 104   32765 std::lock_guard lock(mutex_); 104   32404 std::lock_guard lock(mutex_);
HITCBC 105   32765 if (is_pooled(impl)) 105   32404 if (is_pooled(impl))
106   { 106   {
MISUBC 107   ✗ BOOST_COROSIO_ASSERT(false); 107   ✗ BOOST_COROSIO_ASSERT(false);
108   return; 108   return;
109   } 109   }
HITCBC 110   32765 BOOST_COROSIO_ASSERT(!shutting_down_); 110   32404 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 111   32765 live_.push_back(impl); 111   32404 live_.push_back(impl);
HITCBC 112   32765 } 112   32404 }
113   113  
114   /** Unlink a live impl without recycling (ownership transfer). 114   /** Unlink a live impl without recycling (ownership transfer).
115   115  
116   Used by a caller that is taking an impl out of this pool's 116   Used by a caller that is taking an impl out of this pool's
117   bookkeeping entirely (a thread-local cache slot, a direct 117   bookkeeping entirely (a thread-local cache slot, a direct
118   delete during shutdown) rather than parking it on `free_`. 118   delete during shutdown) rather than parking it on `free_`.
119   119  
120   @pre @p impl is not parked in `free_` (`refs_` must not read 120   @pre @p impl is not parked in `free_` (`refs_` must not read
121   `pooled_sentinel`). The same list-corruption hazard 121   `pooled_sentinel`). The same list-corruption hazard
122   `recycle()`'s class-level comment documents applies here: a 122   `recycle()`'s class-level comment documents applies here: a
123   free-listed impl would be spliced out using `live_`'s 123   free-listed impl would be spliced out using `live_`'s
124   boundary pointers instead of `free_`'s. 124   boundary pointers instead of `free_`'s.
125   125  
126   @return True if @p impl was linked in `live_` and has now 126   @return True if @p impl was linked in `live_` and has now
127   been removed; false if it was already detached. 127   been removed; false if it was already detached.
128   */ 128   */
HITCBC 129   18616 bool remove(Impl* impl) noexcept 129   18493 bool remove(Impl* impl) noexcept
130   { 130   {
HITCBC 131   18616 std::lock_guard lock(mutex_); 131   18493 std::lock_guard lock(mutex_);
HITCBC 132   18616 if (is_pooled(impl)) 132   18493 if (is_pooled(impl))
133   { 133   {
MISUBC 134   ✗ BOOST_COROSIO_ASSERT(false); 134   ✗ BOOST_COROSIO_ASSERT(false);
135   return false; 135   return false;
136   } 136   }
HITCBC 137   18616 return live_.remove(impl); 137   18493 return live_.remove(impl);
HITCBC 138   18616 } 138   18493 }
139   139  
140   public: 140   public:
141   /** Destroy the pool, freeing every impl still parked or live. 141   /** Destroy the pool, freeing every impl still parked or live.
142   142  
143   A live impl is deleted whatever its count. Keepalives it holds 143   A live impl is deleted whatever its count. Keepalives it holds
144   on itself (an embedded op's `object_ref`) are released by its 144   on itself (an embedded op's `object_ref`) are released by its
145   own destructor; the count is pinned first so none of those 145   own destructor; the count is pinned first so none of those
146   releases reaches zero and calls `retire()` on an object that 146   releases reaches zero and calls `retire()` on an object that
147   is already being destroyed. 147   is already being destroyed.
148   148  
149   @pre `shutdown()` has already run, or no impls are live. 149   @pre `shutdown()` has already run, or no impls are live.
150   */ 150   */
HITCBC 151   8909 ~object_pool() 151   8917 ~object_pool()
152   { 152   {
HITCBC 153   8909 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty()); 153   8917 BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
HITCBC 154   26727 for (auto* list : {&free_, &live_}) 154   26751 for (auto* list : {&free_, &live_})
155   { 155   {
HITCBC 156   26026 while (auto* impl = list->pop_front()) 156   25800 while (auto* impl = list->pop_front())
157   { 157   {
HITCBC 158   8208 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed); 158   7966 impl->refs_.store(pooled_sentinel, std::memory_order_relaxed);
HITCBC 159   8208 delete impl; 159   7966 delete impl;
160   } 160   }
161   } 161   }
HITCBC 162   8909 } 162   8917 }
163   163  
HITCBC 164   8899 object_pool() = default; 164   8907 object_pool() = default;
165   object_pool(object_pool const&) = delete; 165   object_pool(object_pool const&) = delete;
166   object_pool& operator=(object_pool const&) = delete; 166   object_pool& operator=(object_pool const&) = delete;
167   167  
168   /** Pop-or-create an impl with the acquisition protocol applied. 168   /** Pop-or-create an impl with the acquisition protocol applied.
169   169  
170   Recycled impls are `reuse()`-reset and their reference count 170   Recycled impls are `reuse()`-reset and their reference count
171   restored from the pooled sentinel; fresh impls are constructed 171   restored from the pooled sentinel; fresh impls are constructed
172   in place from `args` on the cold path. Either way the impl is 172   in place from `args` on the cold path. Either way the impl is
173   adopted as live. This is the sequence every service's 173   adopted as live. This is the sequence every service's
174   `construct()` used to hand-roll. Making the pool itself the 174   `construct()` used to hand-roll. Making the pool itself the
175   only place that calls `new Impl` gives the whole library one 175   only place that calls `new Impl` gives the whole library one
176   audit point for allocation policy. 176   audit point for allocation policy.
177   177  
178   @pre Not called concurrently with, or after, `shutdown()` on 178   @pre Not called concurrently with, or after, `shutdown()` on
179   this pool. Every caller reaches this through a service's 179   this pool. Every caller reaches this through a service's
180   `construct()`, and service shutdown order guarantees no 180   `construct()`, and service shutdown order guarantees no
181   `construct()` outlives that service's own `shutdown()`; the 181   `construct()` outlives that service's own `shutdown()`; the
182   window is only asserted in debug builds (see `adopt()`) 182   window is only asserted in debug builds (see `adopt()`)
183   because the failure mode is an orphaned impl, not corruption. 183   because the failure mode is an orphaned impl, not corruption.
184   184  
185   @param args Constructor arguments for `Impl` (cold path only). 185   @param args Constructor arguments for `Impl` (cold path only).
186   186  
187   @return The acquired impl, `refs_ == 1`, tracked live. 187   @return The acquired impl, `refs_ == 1`, tracked live.
188   */ 188   */
189   template<class... Args> 189   template<class... Args>
HITCBC 190   40262 Impl* acquire(Args&&... args) 190   39842 Impl* acquire(Args&&... args)
191   { 191   {
192   Impl* impl; 192   Impl* impl;
193   { 193   {
HITCBC 194   40262 std::lock_guard lock(mutex_); 194   39842 std::lock_guard lock(mutex_);
HITCBC 195   40262 BOOST_COROSIO_ASSERT(!shutting_down_); 195   39842 BOOST_COROSIO_ASSERT(!shutting_down_);
HITCBC 196   40262 impl = free_.pop_front(); 196   39842 impl = free_.pop_front();
HITCBC 197   40262 if (impl) 197   39842 if (impl)
198   { 198   {
HITCBC 199   22365 impl->refs_.store(1, std::memory_order_relaxed); 199   22182 impl->refs_.store(1, std::memory_order_relaxed);
HITCBC 200   22365 live_.push_back(impl); 200   22182 live_.push_back(impl);
201   } 201   }
HITCBC 202   40262 } 202   39842 }
203   // reuse() runs unlocked: acquire()'s precondition excludes a 203   // reuse() runs unlocked: acquire()'s precondition excludes a
204   // concurrent shutdown() visiting the impl mid-reset. 204   // concurrent shutdown() visiting the impl mid-reset.
HITCBC 205   40262 if (impl) 205   39842 if (impl)
206   { 206   {
HITCBC 207   22365 impl->reuse(); 207   22182 impl->reuse();
HITCBC 208   22365 return impl; 208   22182 return impl;
209   } 209   }
HITCBC 210   17897 impl = new Impl(std::forward<Args>(args)...); 210   17660 impl = new Impl(std::forward<Args>(args)...);
HITCBC 211   17897 adopt(impl); 211   17660 adopt(impl);
HITCBC 212   17897 return impl; 212   17660 return impl;
213   } 213   }
214   214  
215   /** Move a zero-ref impl from live to the free list. 215   /** Move a zero-ref impl from live to the free list.
216   216  
217   Called from `retire()`, possibly on a scheduler thread. 217   Called from `retire()`, possibly on a scheduler thread.
218   Deletes instead when shutting down. The impl must not be 218   Deletes instead when shutting down. The impl must not be
219   touched after this returns. 219   touched after this returns.
220   220  
221   Two accounting bugs are possible here, neither from this 221   Two accounting bugs are possible here, neither from this
222   pool's own bookkeeping but from a caller's refcounting (e.g. 222   pool's own bookkeeping but from a caller's refcounting (e.g.
223   a duplicate `retire()` for one zero-crossing). Both are 223   a duplicate `retire()` for one zero-crossing). Both are
224   made safe in every build mode, not just under asserts, because 224   made safe in every build mode, not just under asserts, because
225   neither can be told apart from the correct case by a 225   neither can be told apart from the correct case by a
226   `BOOST_COROSIO_ASSERT` alone that still runs the list 226   `BOOST_COROSIO_ASSERT` alone that still runs the list
227   operations around it: 227   operations around it:
228   228  
229   - `impl` is already parked in `free_` — the list-corruption 229   - `impl` is already parked in `free_` — the list-corruption
230   hazard the class-level `pooled_sentinel` comment documents. 230   hazard the class-level `pooled_sentinel` comment documents.
231   The `is_pooled()` check runs first and returns before either 231   The `is_pooled()` check runs first and returns before either
232   list is touched. 232   list is touched.
233   - `impl` is fully detached from `live_` and is not the 233   - `impl` is fully detached from `live_` and is not the
234   sentinel (not shutting down): `live_.remove()` correctly 234   sentinel (not shutting down): `live_.remove()` correctly
235   returns false, but then the impl is reachable through 235   returns false, but then the impl is reachable through
236   neither `acquire()` nor `shutdown()`'s visit while still 236   neither `acquire()` nor `shutdown()`'s visit while still
237   allocated — stranded. Deleted instead of left to leak. 237   allocated — stranded. Deleted instead of left to leak.
238   238  
239   While shutting down, an impl that is no longer in `live_` is 239   While shutting down, an impl that is no longer in `live_` is
240   left alone rather than deleted: something else already took 240   left alone rather than deleted: something else already took
241   it out of the pool's bookkeeping (a derived pool's owner, or a 241   it out of the pool's bookkeeping (a derived pool's owner, or a
242   duplicate retirement of an impl already deleted), and deleting 242   duplicate retirement of an impl already deleted), and deleting
243   it again here could be a double free. 243   it again here could be a double free.
244   */ 244   */
HITCBC 245   36500 void recycle(Impl* impl) 245   36087 void recycle(Impl* impl)
246   { 246   {
HITCBC 247   36500 bool do_delete = false; 247   36087 bool do_delete = false;
248   { 248   {
HITCBC 249   36500 std::lock_guard lock(mutex_); 249   36087 std::lock_guard lock(mutex_);
HITCBC 250   36500 if (is_pooled(impl)) 250   36087 if (is_pooled(impl))
251   { 251   {
MISUBC 252   ✗ BOOST_COROSIO_ASSERT(false); 252   ✗ BOOST_COROSIO_ASSERT(false);
253   return; 253   return;
254   } 254   }
255   255  
HITCBC 256   36500 bool const was_live = live_.remove(impl); 256   36087 bool const was_live = live_.remove(impl);
HITCBC 257   36500 if (shutting_down_) 257   36087 if (shutting_down_)
258   { 258   {
HITCBC 259   5942 do_delete = was_live; 259   5946 do_delete = was_live;
260   } 260   }
HITCBC 261   30558 else if (was_live) 261   30141 else if (was_live)
262   { 262   {
HITCBC 263   30558 impl->refs_.store( 263   30141 impl->refs_.store(
264   pooled_sentinel, std::memory_order_relaxed); 264   pooled_sentinel, std::memory_order_relaxed);
265   // LIFO: the most recently retired impl is the 265   // LIFO: the most recently retired impl is the
266   // cache-warmest candidate for the next acquire(). 266   // cache-warmest candidate for the next acquire().
HITCBC 267   30558 free_.push_front(impl); 267   30141 free_.push_front(impl);
268   } 268   }
269   else 269   else
270   { 270   {
MISUBC 271   ✗ BOOST_COROSIO_ASSERT(false); 271   ✗ BOOST_COROSIO_ASSERT(false);
272   do_delete = true; 272   do_delete = true;
273   } 273   }
HITCBC 274   36500 } 274   36087 }
HITCBC 275   36500 if (do_delete) 275   36087 if (do_delete)
HITCBC 276   5942 delete impl; 276   5946 delete impl;
277   } 277   }
278   278  
279   /** Enter shutdown mode and visit every live impl. 279   /** Enter shutdown mode and visit every live impl.
280   280  
281   Sets shutting-down (subsequent zero-crossings delete instead 281   Sets shutting-down (subsequent zero-crossings delete instead
282   of recycling) and runs `f` on each live impl, all under one 282   of recycling) and runs `f` on each live impl, all under one
283   critical section. Each visited impl is pinned with a reference 283   critical section. Each visited impl is pinned with a reference
284   for the call and released after the critical section ends, so 284   for the call and released after the critical section ends, so
285   `f` may hold on to it; an impl whose count already reached zero 285   `f` may hold on to it; an impl whose count already reached zero
286   is retiring and is skipped rather than revived. 286   is retiring and is skipped rather than revived.
287   287  
288   @note The callback runs under the pool's non-recursive mutex; 288   @note The callback runs under the pool's non-recursive mutex;
289   it must not call recycle(), adopt(), or remove() on this pool. 289   it must not call recycle(), adopt(), or remove() on this pool.
290   */ 290   */
291   template<class F> 291   template<class F>
HITCBC 292   13317 void shutdown(F&& f) 292   13325 void shutdown(F&& f)
293   { 293   {
HITCBC 294   13317 std::vector<Impl*> pinned; 294   13325 std::vector<Impl*> pinned;
295   { 295   {
HITCBC 296   13317 std::lock_guard lock(mutex_); 296   13325 std::lock_guard lock(mutex_);
HITCBC 297   13317 shutting_down_ = true; 297   13325 shutting_down_ = true;
298   // Sized before any pin, so a failure here leaves none held. 298   // Sized before any pin, so a failure here leaves none held.
HITCBC 299   13317 std::size_t n = 0; 299   13325 std::size_t n = 0;
HITCBC 300   22518 live_.for_each([&](Impl*) { ++n; }); 300   22522 live_.for_each([&](Impl*) { ++n; });
HITCBC 301   13317 pinned.reserve(n); 301   13325 pinned.reserve(n);
HITCBC 302   13317 live_.for_each( 302   13325 live_.for_each(
HITCBC 303   18408 [&](Impl* impl) 303   18400 [&](Impl* impl)
304   { 304   {
HITCBC 305   9204 if (try_acquire(impl)) 305   9200 if (try_acquire(impl))
306   { 306   {
HITCBC 307   9203 pinned.push_back(impl); 307   9199 pinned.push_back(impl);
HITCBC 308   9203 f(impl); 308   9199 f(impl);
309   } 309   }
310   }); 310   });
HITCBC 311   13317 } 311   13325 }
HITCBC 312   22520 for (auto* impl : pinned) 312   22524 for (auto* impl : pinned)
HITCBC 313   9203 release(impl); 313   9199 release(impl);
HITCBC 314   13317 } 314   13325 }
315   }; 315   };
316   316  
317   } // namespace boost::corosio::detail 317   } // namespace boost::corosio::detail
318   318  
319   #endif 319   #endif