TLA Line data Source code
1 : //
2 : // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/corosio
8 : //
9 :
10 : #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11 : #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12 :
13 : #include <boost/corosio/detail/config.hpp>
14 : #include <boost/corosio/detail/intrusive.hpp>
15 : #include <boost/corosio/detail/object_ref.hpp>
16 : #include <atomic>
17 : #include <cstddef>
18 : #include <initializer_list>
19 : #include <limits>
20 : #include <mutex>
21 : #include <utility>
22 : #include <vector>
23 :
24 : namespace boost::corosio::detail {
25 :
26 : /** Per-service recycling pool for I/O implementations.
27 :
28 : Replaces per-impl `shared_ptr` ownership: a service news an impl
29 : once, then `recycle()` (the impl's `retire()` action) parks
30 : it on the free list and `acquire()` hands it back out, so steady-state
31 : construct/destroy allocates nothing. Unbounded; memory is released
32 : only on pool destruction. Debug poisoning, where an impl does it,
33 : happens in that impl's `retire()`, not here (the pool cannot know
34 : which bytes are resettable).
35 :
36 : A service tears down its impls by calling `shutdown()`, which
37 : enters shutting-down mode and visits every live impl in one
38 : critical section; subsequent zero-ref crossings delete instead of
39 : recycling.
40 :
41 : `adopt()` and `remove()` are protected ownership-transfer
42 : primitives for sanctioned extensions — a derived pool exposing
43 : them to a caller that tracks some of its live impls through its
44 : own structure instead of this pool's `live_` list (the timer
45 : service, via its expiry heap) — rather than part of the public
46 : surface every ordinary service uses.
47 :
48 : @par Thread Safety
49 : Distinct objects: Safe. Shared objects: Safe; all operations lock
50 : an internal mutex.
51 : */
52 : template<class Impl>
53 : class object_pool
54 : {
55 : std::mutex mutex_;
56 : intrusive_list<Impl> live_;
57 : intrusive_list<Impl> free_;
58 : bool shutting_down_ = false;
59 :
60 : /** Marks `refs_` on an impl currently parked in `free_`.
61 :
62 : `intrusive_list::remove()` cannot distinguish "linked in
63 : `free_`" from "linked in `live_`" by inspecting the node
64 : alone — both lists share one `next_`/`prev_` pair per impl —
65 : so `recycle()` cannot safely call `live_.remove()` on an impl
66 : it does not already know is live. Stamping this sentinel into
67 : `refs_` the moment an impl joins `free_`, and checking it
68 : before touching either list, is what lets `recycle()` and
69 : `remove()` tell the two cases apart in every build mode, not
70 : just under asserts. `acquire()` always overwrites it with 1
71 : before the impl is usable again.
72 : */
73 : static constexpr std::size_t pooled_sentinel = pooled_refs;
74 :
75 : /// Check whether @p impl is parked in `free_` (`refs_` reads the
76 : /// sentinel). Callers must check this before any list operation —
77 : /// see the class-level comment on `pooled_sentinel`.
78 HIT 86984 : static bool is_pooled(Impl const* impl) noexcept
79 : {
80 86984 : return impl->refs_.load(std::memory_order_relaxed) ==
81 86984 : pooled_sentinel;
82 : }
83 :
84 : protected:
85 : /** Track a fresh or reset impl as live.
86 :
87 : @pre @p impl is not parked in `free_` (`refs_` must not read
88 : `pooled_sentinel`). The same list-corruption hazard
89 : `recycle()`'s class-level comment documents applies here: a
90 : free-listed impl pushed onto `live_` directly, bypassing
91 : `acquire()`, would end up linked in both lists through one shared
92 : `next_`/`prev_` pair.
93 :
94 : @pre The caller is not racing `shutdown()`. Every caller adopts
95 : while constructing an object (`acquire()`, or a derived pool's
96 : cache handing a parked impl back out), and construction is
97 : expected to stop before the service's own `shutdown()` runs;
98 : this is checked only in debug builds because the failure mode
99 : is a silently orphaned impl (no close callback), not memory
100 : corruption.
101 : */
102 32404 : void adopt(Impl* impl)
103 : {
104 32404 : std::lock_guard lock(mutex_);
105 32404 : if (is_pooled(impl))
106 : {
107 MIS 0 : BOOST_COROSIO_ASSERT(false);
108 : return;
109 : }
110 HIT 32404 : BOOST_COROSIO_ASSERT(!shutting_down_);
111 32404 : live_.push_back(impl);
112 32404 : }
113 :
114 : /** Unlink a live impl without recycling (ownership transfer).
115 :
116 : Used by a caller that is taking an impl out of this pool's
117 : bookkeeping entirely (a thread-local cache slot, a direct
118 : delete during shutdown) rather than parking it on `free_`.
119 :
120 : @pre @p impl is not parked in `free_` (`refs_` must not read
121 : `pooled_sentinel`). The same list-corruption hazard
122 : `recycle()`'s class-level comment documents applies here: a
123 : free-listed impl would be spliced out using `live_`'s
124 : boundary pointers instead of `free_`'s.
125 :
126 : @return True if @p impl was linked in `live_` and has now
127 : been removed; false if it was already detached.
128 : */
129 18493 : bool remove(Impl* impl) noexcept
130 : {
131 18493 : std::lock_guard lock(mutex_);
132 18493 : if (is_pooled(impl))
133 : {
134 MIS 0 : BOOST_COROSIO_ASSERT(false);
135 : return false;
136 : }
137 HIT 18493 : return live_.remove(impl);
138 18493 : }
139 :
140 : public:
141 : /** Destroy the pool, freeing every impl still parked or live.
142 :
143 : A live impl is deleted whatever its count. Keepalives it holds
144 : on itself (an embedded op's `object_ref`) are released by its
145 : own destructor; the count is pinned first so none of those
146 : releases reaches zero and calls `retire()` on an object that
147 : is already being destroyed.
148 :
149 : @pre `shutdown()` has already run, or no impls are live.
150 : */
151 8917 : ~object_pool()
152 : {
153 8917 : BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
154 26751 : for (auto* list : {&free_, &live_})
155 : {
156 25800 : while (auto* impl = list->pop_front())
157 : {
158 7966 : impl->refs_.store(pooled_sentinel, std::memory_order_relaxed);
159 7966 : delete impl;
160 : }
161 : }
162 8917 : }
163 :
164 8907 : object_pool() = default;
165 : object_pool(object_pool const&) = delete;
166 : object_pool& operator=(object_pool const&) = delete;
167 :
168 : /** Pop-or-create an impl with the acquisition protocol applied.
169 :
170 : Recycled impls are `reuse()`-reset and their reference count
171 : restored from the pooled sentinel; fresh impls are constructed
172 : in place from `args` on the cold path. Either way the impl is
173 : adopted as live. This is the sequence every service's
174 : `construct()` used to hand-roll. Making the pool itself the
175 : only place that calls `new Impl` gives the whole library one
176 : audit point for allocation policy.
177 :
178 : @pre Not called concurrently with, or after, `shutdown()` on
179 : this pool. Every caller reaches this through a service's
180 : `construct()`, and service shutdown order guarantees no
181 : `construct()` outlives that service's own `shutdown()`; the
182 : window is only asserted in debug builds (see `adopt()`)
183 : because the failure mode is an orphaned impl, not corruption.
184 :
185 : @param args Constructor arguments for `Impl` (cold path only).
186 :
187 : @return The acquired impl, `refs_ == 1`, tracked live.
188 : */
189 : template<class... Args>
190 39842 : Impl* acquire(Args&&... args)
191 : {
192 : Impl* impl;
193 : {
194 39842 : std::lock_guard lock(mutex_);
195 39842 : BOOST_COROSIO_ASSERT(!shutting_down_);
196 39842 : impl = free_.pop_front();
197 39842 : if (impl)
198 : {
199 22182 : impl->refs_.store(1, std::memory_order_relaxed);
200 22182 : live_.push_back(impl);
201 : }
202 39842 : }
203 : // reuse() runs unlocked: acquire()'s precondition excludes a
204 : // concurrent shutdown() visiting the impl mid-reset.
205 39842 : if (impl)
206 : {
207 22182 : impl->reuse();
208 22182 : return impl;
209 : }
210 17660 : impl = new Impl(std::forward<Args>(args)...);
211 17660 : adopt(impl);
212 17660 : return impl;
213 : }
214 :
215 : /** Move a zero-ref impl from live to the free list.
216 :
217 : Called from `retire()`, possibly on a scheduler thread.
218 : Deletes instead when shutting down. The impl must not be
219 : touched after this returns.
220 :
221 : Two accounting bugs are possible here, neither from this
222 : pool's own bookkeeping but from a caller's refcounting (e.g.
223 : a duplicate `retire()` for one zero-crossing). Both are
224 : made safe in every build mode, not just under asserts, because
225 : neither can be told apart from the correct case by a
226 : `BOOST_COROSIO_ASSERT` alone that still runs the list
227 : operations around it:
228 :
229 : - `impl` is already parked in `free_` — the list-corruption
230 : hazard the class-level `pooled_sentinel` comment documents.
231 : The `is_pooled()` check runs first and returns before either
232 : list is touched.
233 : - `impl` is fully detached from `live_` and is not the
234 : sentinel (not shutting down): `live_.remove()` correctly
235 : returns false, but then the impl is reachable through
236 : neither `acquire()` nor `shutdown()`'s visit while still
237 : allocated — stranded. Deleted instead of left to leak.
238 :
239 : While shutting down, an impl that is no longer in `live_` is
240 : left alone rather than deleted: something else already took
241 : it out of the pool's bookkeeping (a derived pool's owner, or a
242 : duplicate retirement of an impl already deleted), and deleting
243 : it again here could be a double free.
244 : */
245 36087 : void recycle(Impl* impl)
246 : {
247 36087 : bool do_delete = false;
248 : {
249 36087 : std::lock_guard lock(mutex_);
250 36087 : if (is_pooled(impl))
251 : {
252 MIS 0 : BOOST_COROSIO_ASSERT(false);
253 : return;
254 : }
255 :
256 HIT 36087 : bool const was_live = live_.remove(impl);
257 36087 : if (shutting_down_)
258 : {
259 5946 : do_delete = was_live;
260 : }
261 30141 : else if (was_live)
262 : {
263 30141 : impl->refs_.store(
264 : pooled_sentinel, std::memory_order_relaxed);
265 : // LIFO: the most recently retired impl is the
266 : // cache-warmest candidate for the next acquire().
267 30141 : free_.push_front(impl);
268 : }
269 : else
270 : {
271 MIS 0 : BOOST_COROSIO_ASSERT(false);
272 : do_delete = true;
273 : }
274 HIT 36087 : }
275 36087 : if (do_delete)
276 5946 : delete impl;
277 : }
278 :
279 : /** Enter shutdown mode and visit every live impl.
280 :
281 : Sets shutting-down (subsequent zero-crossings delete instead
282 : of recycling) and runs `f` on each live impl, all under one
283 : critical section. Each visited impl is pinned with a reference
284 : for the call and released after the critical section ends, so
285 : `f` may hold on to it; an impl whose count already reached zero
286 : is retiring and is skipped rather than revived.
287 :
288 : @note The callback runs under the pool's non-recursive mutex;
289 : it must not call recycle(), adopt(), or remove() on this pool.
290 : */
291 : template<class F>
292 13325 : void shutdown(F&& f)
293 : {
294 13325 : std::vector<Impl*> pinned;
295 : {
296 13325 : std::lock_guard lock(mutex_);
297 13325 : shutting_down_ = true;
298 : // Sized before any pin, so a failure here leaves none held.
299 13325 : std::size_t n = 0;
300 22522 : live_.for_each([&](Impl*) { ++n; });
301 13325 : pinned.reserve(n);
302 13325 : live_.for_each(
303 18400 : [&](Impl* impl)
304 : {
305 9200 : if (try_acquire(impl))
306 : {
307 9199 : pinned.push_back(impl);
308 9199 : f(impl);
309 : }
310 : });
311 13325 : }
312 22524 : for (auto* impl : pinned)
313 9199 : release(impl);
314 13325 : }
315 : };
316 :
317 : } // namespace boost::corosio::detail
318 :
319 : #endif
|
