. /include/boost/corosio /detail /timer_service.hpp
96.23% Lines (230/239) 100.00% Functions (28/28)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // Copyright (c) 2026 Steve Gerbino 3   // Copyright (c) 2026 Steve Gerbino
4   // 4   //
5   // Distributed under the Boost Software License, Version 1.0. (See accompanying 5   // Distributed under the Boost Software License, Version 1.0. (See accompanying
6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7   // 7   //
8   // Official repository: https://github.com/cppalliance/corosio 8   // Official repository: https://github.com/cppalliance/corosio
9   // 9   //
10   10  
11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
13   13  
14   #include <boost/corosio/detail/timer.hpp> 14   #include <boost/corosio/detail/timer.hpp>
15   #include <boost/corosio/detail/scheduler.hpp> 15   #include <boost/corosio/detail/scheduler.hpp>
16   #include <boost/corosio/detail/scheduler_op.hpp> 16   #include <boost/corosio/detail/scheduler_op.hpp>
17   #include <boost/corosio/detail/object_pool.hpp> 17   #include <boost/corosio/detail/object_pool.hpp>
18   #include <boost/corosio/detail/object_ref.hpp> 18   #include <boost/corosio/detail/object_ref.hpp>
19   #include <boost/corosio/detail/intrusive.hpp> 19   #include <boost/corosio/detail/intrusive.hpp>
20   #include <boost/corosio/detail/thread_local_ptr.hpp> 20   #include <boost/corosio/detail/thread_local_ptr.hpp>
21   #include <boost/capy/error.hpp> 21   #include <boost/capy/error.hpp>
22   #include <boost/capy/ex/execution_context.hpp> 22   #include <boost/capy/ex/execution_context.hpp>
23   #include <boost/capy/ex/abandon.hpp> 23   #include <boost/capy/ex/abandon.hpp>
24   #include <boost/capy/ex/executor_ref.hpp> 24   #include <boost/capy/ex/executor_ref.hpp>
25   #include <system_error> 25   #include <system_error>
26   26  
27   #include <atomic> 27   #include <atomic>
28   #include <chrono> 28   #include <chrono>
29   #include <coroutine> 29   #include <coroutine>
30   #include <cstddef> 30   #include <cstddef>
31   #include <limits> 31   #include <limits>
32   #include <mutex> 32   #include <mutex>
33   #include <thread> 33   #include <thread>
34   #include <stop_token> 34   #include <stop_token>
35   #include <utility> 35   #include <utility>
36   #include <vector> 36   #include <vector>
37   37  
38   namespace boost::corosio::detail { 38   namespace boost::corosio::detail {
39   39  
40   class scheduler; 40   class scheduler;
41   41  
42   /* 42   /*
43   Timer Service 43   Timer Service
44   ============= 44   =============
45   45  
46   Data Structures 46   Data Structures
47   --------------- 47   ---------------
48   waiter_node (defined in timer.hpp) holds per-waiter state: 48   waiter_node (defined in timer.hpp) holds per-waiter state:
49   coroutine handle, executor, error output, embedded 49   coroutine handle, executor, error output, embedded
50   completion_op. Each concurrent co_await t.wait() embeds one 50   completion_op. Each concurrent co_await t.wait() embeds one
51   waiter_node in the awaitable on the suspended coroutine's 51   waiter_node in the awaitable on the suspended coroutine's
52   frame — waits perform no allocation. 52   frame — waits perform no allocation.
53   53  
54   timer::implementation holds per-timer state: expiry, heap 54   timer::implementation holds per-timer state: expiry, heap
55   index, and the single published waiter. Each timer holds 55   index, and the single published waiter. Each timer holds
56   at most one waiter; process_expired's local cross-timer drain 56   at most one waiter; process_expired's local cross-timer drain
57   list still threads waiters through their intrusive hooks when 57   list still threads waiters through their intrusive hooks when
58   collecting several timers' waiters past the lock. 58   collecting several timers' waiters past the lock.
59   59  
60   timer_service owns a min-heap of active timers and recycles 60   timer_service owns a min-heap of active timers and recycles
61   retired impls through a per-service object_pool, fronted by a 61   retired impls through a per-service object_pool, fronted by a
62   thread-local single-slot cache. The heap is ordered by expiry 62   thread-local single-slot cache. The heap is ordered by expiry
63   time; the scheduler queries nearest_expiry() to set the 63   time; the scheduler queries nearest_expiry() to set the
64   epoll/timerfd timeout. 64   epoll/timerfd timeout.
65   65  
66   Optimization Strategy 66   Optimization Strategy
67   --------------------- 67   ---------------------
68   1. Deferred heap insertion — expires_after() stores the expiry 68   1. Deferred heap insertion — expires_after() stores the expiry
69   but does not insert into the heap. Insertion happens in wait(). 69   but does not insert into the heap. Insertion happens in wait().
70   2. Thread-local impl cache — single-slot per-thread cache, in 70   2. Thread-local impl cache — single-slot per-thread cache, in
71   front of the object_pool fallback; keeps a thread's hot impl 71   front of the object_pool fallback; keeps a thread's hot impl
72   out of the shared free list. 72   out of the shared free list.
73   3. Frame-resident waiter_node with embedded completion_op — 73   3. Frame-resident waiter_node with embedded completion_op —
74   eliminates heap allocation per wait/fire/cancel. 74   eliminates heap allocation per wait/fire/cancel.
75   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). 75   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry().
76   5. might_have_pending_waits_ flag — skips lock when no wait issued. 76   5. might_have_pending_waits_ flag — skips lock when no wait issued.
77   77  
78   Concurrency 78   Concurrency
79   ----------- 79   -----------
80   stop_token callbacks can fire from any thread. The impl_ 80   stop_token callbacks can fire from any thread. The impl_
81   pointer on waiter_node is used as a "still in list" marker. 81   pointer on waiter_node is used as a "still in list" marker.
82   A waiter_node's storage is the suspended coroutine's frame: 82   A waiter_node's storage is the suspended coroutine's frame:
83   every completion path must finish touching the node before 83   every completion path must finish touching the node before
84   posting the continuation or destroying the handle. 84   posting the continuation or destroying the handle.
85   */ 85   */
86   86  
87   inline void timer_service_invalidate_cache() noexcept; 87   inline void timer_service_invalidate_cache() noexcept;
88   88  
89   /** The timer service's recycling pool. 89   /** The timer service's recycling pool.
90   90  
91   Adds no behavior of its own — it exists only to re-expose 91   Adds no behavior of its own — it exists only to re-expose
92   `adopt()` and `remove()` as public, so the timer service's 92   `adopt()` and `remove()` as public, so the timer service's
93   thread-local cache can transfer a cached impl's `live_` tracking 93   thread-local cache can transfer a cached impl's `live_` tracking
94   on both sides of the handoff: `construct()` re-adopts a 94   on both sides of the handoff: `construct()` re-adopts a
95   TL-cached impl as live, `shutdown()` detaches one before its 95   TL-cached impl as live, `shutdown()` detaches one before its
96   direct delete. 96   direct delete.
97   */ 97   */
98   class timer_object_pool : public object_pool<timer::implementation> 98   class timer_object_pool : public object_pool<timer::implementation>
99   { 99   {
100   public: 100   public:
101   using object_pool::adopt; 101   using object_pool::adopt;
102   using object_pool::remove; 102   using object_pool::remove;
103   }; 103   };
104   104  
105   // timer_service class body — member function definitions are 105   // timer_service class body — member function definitions are
106   // out-of-class (after implementation and waiter_node are complete) 106   // out-of-class (after implementation and waiter_node are complete)
107   class BOOST_COROSIO_DECL timer_service final 107   class BOOST_COROSIO_DECL timer_service final
108   : public capy::execution_context::service 108   : public capy::execution_context::service
109   , public io_object::io_service 109   , public io_object::io_service
110   { 110   {
111   friend struct timer::implementation; 111   friend struct timer::implementation;
112   112  
113   public: 113   public:
114   using clock_type = std::chrono::steady_clock; 114   using clock_type = std::chrono::steady_clock;
115   using time_point = clock_type::time_point; 115   using time_point = clock_type::time_point;
116   116  
117   /// Type-erased callback for earliest-expiry-changed notifications. 117   /// Type-erased callback for earliest-expiry-changed notifications.
118   class callback 118   class callback
119   { 119   {
120   void* ctx_ = nullptr; 120   void* ctx_ = nullptr;
121   void (*fn_)(void*) = nullptr; 121   void (*fn_)(void*) = nullptr;
122   122  
123   public: 123   public:
124   /// Construct an empty callback. 124   /// Construct an empty callback.
HITCBC 125   4413 callback() = default; 125   4413 callback() = default;
126   126  
127   /// Construct a callback with the given context and function. 127   /// Construct a callback with the given context and function.
HITCBC 128   4413 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {} 128   4413 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {}
129   129  
130   /// Return true if the callback is non-empty. 130   /// Return true if the callback is non-empty.
131   explicit operator bool() const noexcept 131   explicit operator bool() const noexcept
132   { 132   {
133   return fn_ != nullptr; 133   return fn_ != nullptr;
134   } 134   }
135   135  
136   /// Invoke the callback. 136   /// Invoke the callback.
HITCBC 137   8301 void operator()() const 137   8070 void operator()() const
138   { 138   {
HITCBC 139   8301 if (fn_) 139   8070 if (fn_)
HITCBC 140   8301 fn_(ctx_); 140   8070 fn_(ctx_);
HITCBC 141   8301 } 141   8070 }
142   }; 142   };
143   143  
144   private: 144   private:
145   struct heap_entry 145   struct heap_entry
146   { 146   {
147   time_point time_; 147   time_point time_;
148   timer::implementation* timer_; 148   timer::implementation* timer_;
149   }; 149   };
150   150  
151   scheduler* sched_ = nullptr; 151   scheduler* sched_ = nullptr;
152   BOOST_COROSIO_MSVC_WARNING_PUSH 152   BOOST_COROSIO_MSVC_WARNING_PUSH
153   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface 153   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface
154   mutable std::mutex mutex_; 154   mutable std::mutex mutex_;
155   std::vector<heap_entry> heap_; 155   std::vector<heap_entry> heap_;
156   timer_object_pool pool_; 156   timer_object_pool pool_;
157   callback on_earliest_changed_; 157   callback on_earliest_changed_;
158   bool shutting_down_ = false; 158   bool shutting_down_ = false;
159   159  
160   // Cancellers that claimed a waiter and have yet to post it; 160   // Cancellers that claimed a waiter and have yet to post it;
161   // shutdown waits them out so the scheduler's drain finds the post. 161   // shutdown waits them out so the scheduler's drain finds the post.
162   std::size_t cancels_posting_ = 0; // guarded by mutex_ 162   std::size_t cancels_posting_ = 0; // guarded by mutex_
163   // Avoids mutex in nearest_expiry() and empty() 163   // Avoids mutex in nearest_expiry() and empty()
164   mutable std::atomic<std::int64_t> cached_nearest_ns_{ 164   mutable std::atomic<std::int64_t> cached_nearest_ns_{
165   (std::numeric_limits<std::int64_t>::max)()}; 165   (std::numeric_limits<std::int64_t>::max)()};
166   BOOST_COROSIO_MSVC_WARNING_POP 166   BOOST_COROSIO_MSVC_WARNING_POP
167   167  
168   public: 168   public:
169   /// Construct the timer service bound to a scheduler. 169   /// Construct the timer service bound to a scheduler.
HITCBC 170   4413 inline timer_service(capy::execution_context&, scheduler& sched) 170   4413 inline timer_service(capy::execution_context&, scheduler& sched)
HITCBC 171   4413 : sched_(&sched) 171   4413 : sched_(&sched)
172   { 172   {
HITCBC 173   4413 } 173   4413 }
174   174  
175   /// Return the associated scheduler. 175   /// Return the associated scheduler.
HITCBC 176   36410 inline scheduler& get_scheduler() noexcept 176   35708 inline scheduler& get_scheduler() noexcept
177   { 177   {
HITCBC 178   36410 return *sched_; 178   35708 return *sched_;
179   } 179   }
180   180  
181   /// Destroy the timer service. 181   /// Destroy the timer service.
HITCBC 182   8826 ~timer_service() override = default; 182   8826 ~timer_service() override = default;
183   183  
184   timer_service(timer_service const&) = delete; 184   timer_service(timer_service const&) = delete;
185   timer_service& operator=(timer_service const&) = delete; 185   timer_service& operator=(timer_service const&) = delete;
186   186  
187   /// Register a callback invoked when the earliest expiry changes. 187   /// Register a callback invoked when the earliest expiry changes.
HITCBC 188   4413 inline void set_on_earliest_changed(callback cb) 188   4413 inline void set_on_earliest_changed(callback cb)
189   { 189   {
HITCBC 190   4413 on_earliest_changed_ = cb; 190   4413 on_earliest_changed_ = cb;
HITCBC 191   4413 } 191   4413 }
192   192  
193   /// Return true if no timers are in the heap. 193   /// Return true if no timers are in the heap.
194   inline bool empty() const noexcept 194   inline bool empty() const noexcept
195   { 195   {
196   return cached_nearest_ns_.load(std::memory_order_acquire) == 196   return cached_nearest_ns_.load(std::memory_order_acquire) ==
197   (std::numeric_limits<std::int64_t>::max)(); 197   (std::numeric_limits<std::int64_t>::max)();
198   } 198   }
199   199  
200   /// Return the nearest timer expiry without acquiring the mutex. 200   /// Return the nearest timer expiry without acquiring the mutex.
HITCBC 201   26867 inline time_point nearest_expiry() const noexcept 201   93928 inline time_point nearest_expiry() const noexcept
202   { 202   {
HITCBC 203   26867 auto ns = cached_nearest_ns_.load(std::memory_order_acquire); 203   93928 auto ns = cached_nearest_ns_.load(std::memory_order_acquire);
HITCBC 204   26867 return time_point(time_point::duration(ns)); 204   93928 return time_point(time_point::duration(ns));
205   } 205   }
206   206  
207   /// Cancel all pending timers and free cached resources. 207   /// Cancel all pending timers and free cached resources.
208   inline void shutdown() override; 208   inline void shutdown() override;
209   209  
210   /// Construct a new timer implementation. 210   /// Construct a new timer implementation.
211   inline io_object::implementation* construct() override; 211   inline io_object::implementation* construct() override;
212   212  
213   /// Destroy a timer implementation, cancelling pending waiters. 213   /// Destroy a timer implementation, cancelling pending waiters.
214   inline void destroy(io_object::implementation* p) override; 214   inline void destroy(io_object::implementation* p) override;
215   215  
216   /// Cancel and recycle a timer implementation. 216   /// Cancel and recycle a timer implementation.
217   inline void destroy_impl(timer::implementation& impl); 217   inline void destroy_impl(timer::implementation& impl);
218   218  
219   /// Publish the timer's waiter and insert the timer into the heap. 219   /// Publish the timer's waiter and insert the timer into the heap.
220   inline void insert_waiter(timer::implementation& impl, waiter_node* w); 220   inline void insert_waiter(timer::implementation& impl, waiter_node* w);
221   221  
222   /// Cancel the timer's published waiter, if any. 222   /// Cancel the timer's published waiter, if any.
223   inline void cancel_timer(timer::implementation& impl); 223   inline void cancel_timer(timer::implementation& impl);
224   224  
225   /// Cancel one specific waiter ( stop_token callback path ). 225   /// Cancel one specific waiter ( stop_token callback path ).
226   inline void cancel_waiter(waiter_node* w); 226   inline void cancel_waiter(waiter_node* w);
227   227  
228   /// Complete all waiters whose timers have expired. 228   /// Complete all waiters whose timers have expired.
229   inline std::size_t process_expired(); 229   inline std::size_t process_expired();
230   230  
231   private: 231   private:
HITCBC 232   176934 inline void refresh_cached_nearest() noexcept 232   245265 inline void refresh_cached_nearest() noexcept
233   { 233   {
HITCBC 234   176934 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)() 234   245265 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)()
HITCBC 235   60482 : heap_[0].time_.time_since_epoch().count(); 235   61763 : heap_[0].time_.time_since_epoch().count();
HITCBC 236   176934 cached_nearest_ns_.store(ns, std::memory_order_release); 236   245265 cached_nearest_ns_.store(ns, std::memory_order_release);
HITCBC 237   176934 } 237   245265 }
238   238  
239   inline void remove_timer_impl(timer::implementation& impl); 239   inline void remove_timer_impl(timer::implementation& impl);
240   inline void up_heap(std::size_t index); 240   inline void up_heap(std::size_t index);
241   inline void down_heap(std::size_t index); 241   inline void down_heap(std::size_t index);
242   inline void swap_heap(std::size_t i1, std::size_t i2); 242   inline void swap_heap(std::size_t i1, std::size_t i2);
243   }; 243   };
244   244  
245   // Thread-local single-slot impl cache, validated by comparing svc_. 245   // Thread-local single-slot impl cache, validated by comparing svc_.
246   // Keeps a hot impl out of the shared free list; both sides still 246   // Keeps a hot impl out of the shared free list; both sides still
247   // take the pool lock to move it in and out of live tracking. 247   // take the pool lock to move it in and out of live tracking.
248   // Cleared by timer_service_invalidate_cache() during shutdown. 248   // Cleared by timer_service_invalidate_cache() during shutdown.
249   249  
250   inline thread_local_ptr<timer::implementation> tl_cached_impl; 250   inline thread_local_ptr<timer::implementation> tl_cached_impl;
251   251  
252   // The POD TLS slot above never runs destructors, so a short-lived 252   // The POD TLS slot above never runs destructors, so a short-lived
253   // run() thread would leak its cached impl. Each push arms this 253   // run() thread would leak its cached impl. Each push arms this
254   // owner, whose destructor frees the slot at thread exit. A cached 254   // owner, whose destructor frees the slot at thread exit. A cached
255   // entry is a quiescent heap object (nothing in the heap or free 255   // entry is a quiescent heap object (nothing in the heap or free
256   // list) and deletion touches no service state, so it is safe after 256   // list) and deletion touches no service state, so it is safe after
257   // the owning service is gone (the stale-entry path in 257   // the owning service is gone (the stale-entry path in
258   // try_pop_tl_cache deletes the same way). 258   // try_pop_tl_cache deletes the same way).
259   struct tl_cache_owner 259   struct tl_cache_owner
260   { 260   {
HITCBC 261   68 ~tl_cache_owner() 261   61 ~tl_cache_owner()
262   { 262   {
HITCBC 263   68 delete tl_cached_impl.get(); 263   61 delete tl_cached_impl.get();
HITCBC 264   68 tl_cached_impl.set(nullptr); 264   61 tl_cached_impl.set(nullptr);
HITCBC 265   68 } 265   61 }
266   }; 266   };
267   267  
268   inline void 268   inline void
HITCBC 269   15379 arm_tl_cache_cleanup() noexcept 269   15248 arm_tl_cache_cleanup() noexcept
270   { 270   {
HITCBC 271   15379 [[maybe_unused]] thread_local tl_cache_owner owner; 271   15248 [[maybe_unused]] thread_local tl_cache_owner owner;
HITCBC 272   15379 } 272   15248 }
273   273  
274   inline timer::implementation* 274   inline timer::implementation*
HITCBC 275   20748 try_pop_tl_cache(timer_service* svc) noexcept 275   20401 try_pop_tl_cache(timer_service* svc) noexcept
276   { 276   {
HITCBC 277   20748 auto* impl = tl_cached_impl.get(); 277   20401 auto* impl = tl_cached_impl.get();
HITCBC 278   20748 if (impl) 278   20401 if (impl)
279   { 279   {
HITCBC 280   14868 tl_cached_impl.set(nullptr); 280   14744 tl_cached_impl.set(nullptr);
HITCBC 281   14868 if (impl->svc_ == svc) 281   14744 if (impl->svc_ == svc)
HITCBC 282   14868 return impl; 282   14744 return impl;
283   // Stale impl from a destroyed service 283   // Stale impl from a destroyed service
MISUBC 284   ✗ delete impl; 284   ✗ delete impl;
285   } 285   }
HITCBC 286   5880 return nullptr; 286   5657 return nullptr;
287   } 287   }
288   288  
289   inline bool 289   inline bool
HITCBC 290   17501 try_push_tl_cache(timer::implementation* impl) noexcept 290   17154 try_push_tl_cache(timer::implementation* impl) noexcept
291   { 291   {
HITCBC 292   17501 if (!tl_cached_impl.get()) 292   17154 if (!tl_cached_impl.get())
293   { 293   {
HITCBC 294   15379 arm_tl_cache_cleanup(); 294   15248 arm_tl_cache_cleanup();
HITCBC 295   15379 tl_cached_impl.set(impl); 295   15248 tl_cached_impl.set(impl);
HITCBC 296   15379 return true; 296   15248 return true;
297   } 297   }
HITCBC 298   2122 return false; 298   1906 return false;
299   } 299   }
300   300  
301   inline void 301   inline void
HITCBC 302   8826 timer_service_invalidate_cache() noexcept 302   8826 timer_service_invalidate_cache() noexcept
303   { 303   {
HITCBC 304   8826 delete tl_cached_impl.get(); 304   8826 delete tl_cached_impl.get();
HITCBC 305   8826 tl_cached_impl.set(nullptr); 305   8826 tl_cached_impl.set(nullptr);
HITCBC 306   8826 } 306   8826 }
307   307  
308   // timer_service out-of-class member function definitions 308   // timer_service out-of-class member function definitions
309   309  
310   inline void 310   inline void
HITCBC 311   8826 timer_service::shutdown() 311   8826 timer_service::shutdown()
312   { 312   {
HITCBC 313   8826 timer_service_invalidate_cache(); 313   8826 timer_service_invalidate_cache();
HITCBC 314   8826 shutting_down_ = true; 314   8826 shutting_down_ = true;
315   // Flag only; the real drain walks the expiry heap below, not 315   // Flag only; the real drain walks the expiry heap below, not
316   // object_pool's live_ list, so the visit callback is a no-op. 316   // object_pool's live_ list, so the visit callback is a no-op.
HITCBC 317   8826 pool_.shutdown([](timer::implementation*) {}); 317   8826 pool_.shutdown([](timer::implementation*) {});
318   318  
319   // Snapshot impls and detach them from the heap so that 319   // Snapshot impls and detach them from the heap so that
320   // coroutine-owned timer destructors (run by the unwinds the 320   // coroutine-owned timer destructors (run by the unwinds the
321   // abandons below start) cannot re-enter remove_timer_impl() and 321   // abandons below start) cannot re-enter remove_timer_impl() and
322   // mutate the vector during iteration. A stop requested on another thread 322   // mutate the vector during iteration. A stop requested on another thread
323   // reaches cancel_waiter() under the same lock: whichever side 323   // reaches cancel_waiter() under the same lock: whichever side
324   // claims a waiter first owns it, and a waiter its canceller 324   // claims a waiter first owns it, and a waiter its canceller
325   // claimed is posted, so the scheduler's drain abandons it. 325   // claimed is posted, so the scheduler's drain abandons it.
HITCBC 326   8826 std::vector<timer::implementation*> impls; 326   8826 std::vector<timer::implementation*> impls;
HITCBC 327   8826 std::vector<waiter_node*> waiters; 327   8826 std::vector<waiter_node*> waiters;
328   { 328   {
HITCBC 329   8826 std::lock_guard lock(mutex_); 329   8826 std::lock_guard lock(mutex_);
HITCBC 330   8826 impls.reserve(heap_.size()); 330   8826 impls.reserve(heap_.size());
HITCBC 331   8826 waiters.reserve(heap_.size()); 331   8826 waiters.reserve(heap_.size());
HITCBC 332   12060 for (auto& entry : heap_) 332   12068 for (auto& entry : heap_)
333   { 333   {
HITCBC 334   3234 entry.timer_->heap_index_.store( 334   3242 entry.timer_->heap_index_.store(
335   (std::numeric_limits<std::size_t>::max)(), 335   (std::numeric_limits<std::size_t>::max)(),
336   std::memory_order_relaxed); 336   std::memory_order_relaxed);
HITCBC 337   3234 impls.push_back(entry.timer_); 337   3242 impls.push_back(entry.timer_);
HITCBC 338   3234 if (auto* w = std::exchange(entry.timer_->waiter_, nullptr)) 338   3242 if (auto* w = std::exchange(entry.timer_->waiter_, nullptr))
339   { 339   {
HITCBC 340   3234 w->impl_ = nullptr; 340   3242 w->impl_ = nullptr;
HITCBC 341   3234 waiters.push_back(w); 341   3242 waiters.push_back(w);
342   } 342   }
343   } 343   }
HITCBC 344   8826 heap_.clear(); 344   8826 heap_.clear();
HITCBC 345   8826 cached_nearest_ns_.store( 345   8826 cached_nearest_ns_.store(
346   (std::numeric_limits<std::int64_t>::max)(), 346   (std::numeric_limits<std::int64_t>::max)(),
347   std::memory_order_release); 347   std::memory_order_release);
HITCBC 348   8826 } 348   8826 }
349   349  
350   // A stop on another thread may have claimed its waiter before the 350   // A stop on another thread may have claimed its waiter before the
351   // pass above and still be posting it; the scheduler drains next, 351   // pass above and still be posting it; the scheduler drains next,
352   // and must find that post. 352   // and must find that post.
353   for (;;) 353   for (;;)
354   { 354   {
355   { 355   {
HITCBC 356   10026 std::lock_guard lock(mutex_); 356   8826 std::lock_guard lock(mutex_);
HITCBC 357   10026 if (cancels_posting_ == 0) 357   8826 if (cancels_posting_ == 0)
HITCBC 358   8826 break; 358   8826 break;
HITCBC 359   10026 } 359   8826 }
MISLBC 360   1200 std::this_thread::yield(); 360   ✗ std::this_thread::yield();
MISLBC 361   1200 } 361   ✗ }
362   362  
363   // Cancel waiting timers. Each waiter called work_started() 363   // Cancel waiting timers. Each waiter called work_started()
364   // in implementation::wait(); a waiter given up here never 364   // in implementation::wait(); a waiter given up here never
365   // completes, so its work is finished here to keep the count 365   // completes, so its work is finished here to keep the count
366   // balanced. Resetting the stop callback outside the lock 366   // balanced. Resetting the stop callback outside the lock
367   // waits out a canceller already running, which finds the waiter 367   // waits out a canceller already running, which finds the waiter
368   // claimed. 368   // claimed.
HITCBC 369   12060 for (auto* w : waiters) 369   12068 for (auto* w : waiters)
370   { 370   {
HITCBC 371   3234 w->reset_stop_cb(); 371   3242 w->reset_stop_cb();
HITCBC 372   3234 auto c = std::exchange(w->cont_, {}); 372   3242 auto c = std::exchange(w->cont_, {});
HITCBC 373   3234 sched_->work_finished(); 373   3242 sched_->work_finished();
374   // Unwinding the frame also ends the node's storage 374   // Unwinding the frame also ends the node's storage
HITCBC 375   3234 if (c.h) 375   3242 if (c.h)
HITCBC 376   3234 capy::abandon(c); 376   3242 capy::abandon(c);
377   } 377   }
378   378  
HITCBC 379   12060 for (auto* impl : impls) 379   12068 for (auto* impl : impls)
380   { 380   {
381   // Unlink from the pool's live_ list before the direct delete 381   // Unlink from the pool's live_ list before the direct delete
382   // below, or ~object_pool()'s unconditional sweep would delete 382   // below, or ~object_pool()'s unconditional sweep would delete
383   // this impl a second time. 383   // this impl a second time.
HITCBC 384   3234 [[maybe_unused]] bool const removed = pool_.remove(impl); 384   3242 [[maybe_unused]] bool const removed = pool_.remove(impl);
HITCBC 385   3234 BOOST_COROSIO_ASSERT(removed); 385   3242 BOOST_COROSIO_ASSERT(removed);
HITCBC 386   3234 delete impl; 386   3242 delete impl;
387   } 387   }
388   388  
389   // Anything still parked in pool_ (free-listed or live with no 389   // Anything still parked in pool_ (free-listed or live with no
390   // pending wait at shutdown) is freed by object_pool's own destructor. 390   // pending wait at shutdown) is freed by object_pool's own destructor.
HITCBC 391   8826 } 391   8826 }
392   392  
393   inline io_object::implementation* 393   inline io_object::implementation*
HITCBC 394   20748 timer_service::construct() 394   20401 timer_service::construct()
395   { 395   {
HITCBC 396   20748 timer::implementation* impl = try_pop_tl_cache(this); 396   20401 timer::implementation* impl = try_pop_tl_cache(this);
HITCBC 397   20748 if (impl) 397   20401 if (impl)
398   { 398   {
HITCBC 399   14868 impl->svc_ = this; 399   14744 impl->svc_ = this;
HITCBC 400   14868 impl->reuse(); 400   14744 impl->reuse();
HITCBC 401   14868 impl->refs_.store(1, std::memory_order_relaxed); 401   14744 impl->refs_.store(1, std::memory_order_relaxed);
402   // The TL push that parked impl here also removed it from 402   // The TL push that parked impl here also removed it from
403   // live_ (see destroy_impl) — re-adopt it now that it is live 403   // live_ (see destroy_impl) — re-adopt it now that it is live
404   // again. 404   // again.
HITCBC 405   14868 pool_.adopt(impl); 405   14744 pool_.adopt(impl);
HITCBC 406   14868 return impl; 406   14744 return impl;
407   } 407   }
408   408  
HITCBC 409   5880 return pool_.acquire(*this); 409   5657 return pool_.acquire(*this);
410   } 410   }
411   411  
412   inline void 412   inline void
HITCBC 413   20748 timer_service::destroy(io_object::implementation* p) 413   20401 timer_service::destroy(io_object::implementation* p)
414   { 414   {
415   // During shutdown the drain loop owns every impl and deletes 415   // During shutdown the drain loop owns every impl and deletes
416   // them directly. A frame destroyed by that loop can unwind a 416   // them directly. A frame destroyed by that loop can unwind a
417   // handle whose impl was freed in an earlier iteration (a 417   // handle whose impl was freed in an earlier iteration (a
418   // timeout's parent frame owns the timeout timer while 418   // timeout's parent frame owns the timeout timer while
419   // suspended on the inner delay's timer), so bail out before 419   // suspended on the inner delay's timer), so bail out before
420   // even downcasting the pointer. 420   // even downcasting the pointer.
HITCBC 421   20748 if (shutting_down_) 421   20401 if (shutting_down_)
HITCBC 422   3247 return; 422   3247 return;
HITCBC 423   17501 destroy_impl(static_cast<timer::implementation&>(*p)); 423   17154 destroy_impl(static_cast<timer::implementation&>(*p));
424   } 424   }
425   425  
426   inline void 426   inline void
HITCBC 427   17501 timer_service::destroy_impl(timer::implementation& impl) 427   17154 timer_service::destroy_impl(timer::implementation& impl)
428   { 428   {
429   // During shutdown the impl is owned by the shutdown loop. 429   // During shutdown the impl is owned by the shutdown loop.
430   // Re-entering here (from a coroutine-owned timer destructor 430   // Re-entering here (from a coroutine-owned timer destructor
431   // triggered by h.destroy()) must not modify the heap or 431   // triggered by h.destroy()) must not modify the heap or
432   // recycle the impl — shutdown deletes it directly. 432   // recycle the impl — shutdown deletes it directly.
HITCBC 433   17501 if (shutting_down_) 433   17154 if (shutting_down_)
MISUBC 434   ✗ return; 434   ✗ return;
435   435  
HITCBC 436   17501 cancel_timer(impl); 436   17154 cancel_timer(impl);
437   437  
HITCBC 438   35002 if (impl.heap_index_.load(std::memory_order_relaxed) != 438   34308 if (impl.heap_index_.load(std::memory_order_relaxed) !=
HITCBC 439   17501 (std::numeric_limits<std::size_t>::max)()) 439   17154 (std::numeric_limits<std::size_t>::max)())
440   { 440   {
MISUBC 441   ✗ std::lock_guard lock(mutex_); 441   ✗ std::lock_guard lock(mutex_);
MISUBC 442   ✗ remove_timer_impl(impl); 442   ✗ remove_timer_impl(impl);
MISUBC 443   ✗ refresh_cached_nearest(); 443   ✗ refresh_cached_nearest();
MISUBC 444   ✗ } 444   ✗ }
445   445  
HITCBC 446   17501 if (try_push_tl_cache(&impl)) 446   17154 if (try_push_tl_cache(&impl))
447   { 447   {
448   // The TL slot takes sole ownership: it is deleted directly by 448   // The TL slot takes sole ownership: it is deleted directly by
449   // tl_cache_owner / the stale-service path, never through this 449   // tl_cache_owner / the stale-service path, never through this
450   // pool, so it must leave live_ now or ~object_pool() would see 450   // pool, so it must leave live_ now or ~object_pool() would see
451   // it as abandoned (and, at the wrong moment, double-delete it 451   // it as abandoned (and, at the wrong moment, double-delete it
452   // after a later construct() re-adopts it). 452   // after a later construct() re-adopts it).
HITCBC 453   15379 [[maybe_unused]] bool const removed = pool_.remove(&impl); 453   15248 [[maybe_unused]] bool const removed = pool_.remove(&impl);
HITCBC 454   15379 BOOST_COROSIO_ASSERT(removed); 454   15248 BOOST_COROSIO_ASSERT(removed);
HITCBC 455   15379 return; 455   15248 return;
456   } 456   }
457   457  
458   // No op keepalives on a timer impl: this is the only reference, 458   // No op keepalives on a timer impl: this is the only reference,
459   // so release() always recycles immediately via retire(). 459   // so release() always recycles immediately via retire().
HITCBC 460   2122 release(&impl); 460   1906 release(&impl);
461   } 461   }
462   462  
463   inline void 463   inline void
HITCBC 464   25362 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w) 464   24215 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w)
465   { 465   {
HITCBC 466   25362 bool notify = false; 466   24215 bool notify = false;
HITCBC 467   25362 bool lost_cancel = false; 467   24215 bool lost_cancel = false;
468   { 468   {
HITCBC 469   25362 std::lock_guard lock(mutex_); 469   24215 std::lock_guard lock(mutex_);
470   // Grow before publishing anything, so the push_back below 470   // Grow before publishing anything, so the push_back below
471   // cannot throw: a failure here leaves the waiter untouched, 471   // cannot throw: a failure here leaves the waiter untouched,
472   // the strong guarantee rearm_wait's recovery relies on. 472   // the strong guarantee rearm_wait's recovery relies on.
HITCBC 473   25362 if (impl.heap_index_.load(std::memory_order_relaxed) == 473   24215 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 474   50724 (std::numeric_limits<std::size_t>::max)() && 474   48430 (std::numeric_limits<std::size_t>::max)() &&
HITCBC 475   25362 heap_.size() == heap_.capacity()) 475   24215 heap_.size() == heap_.capacity())
HITCBC 476   955 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity()); 476   943 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity());
477   // Publish: from here the waiter is visible to the fire path and 477   // Publish: from here the waiter is visible to the fire path and
478   // to its own stop callback (impl_ non-null enables cancel_waiter). 478   // to its own stop callback (impl_ non-null enables cancel_waiter).
HITCBC 479   25362 w->impl_ = &impl; 479   24215 w->impl_ = &impl;
HITCBC 480   50724 if (impl.heap_index_.load(std::memory_order_relaxed) == 480   48430 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 481   25362 (std::numeric_limits<std::size_t>::max)()) 481   24215 (std::numeric_limits<std::size_t>::max)())
482   { 482   {
HITCBC 483   25362 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed); 483   24215 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed);
HITCBC 484   25362 heap_.push_back({impl.expiry_, &impl}); 484   24215 heap_.push_back({impl.expiry_, &impl});
HITCBC 485   25362 up_heap(heap_.size() - 1); 485   24215 up_heap(heap_.size() - 1);
HITCBC 486   25362 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0); 486   24215 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0);
HITCBC 487   25362 refresh_cached_nearest(); 487   24215 refresh_cached_nearest();
488   } 488   }
HITCBC 489   25362 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr); 489   24215 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr);
HITCBC 490   25362 impl.waiter_ = w; 490   24215 impl.waiter_ = w;
491   491  
492   // Lost-cancel re-check: a stop requested after the canceller was 492   // Lost-cancel re-check: a stop requested after the canceller was
493   // armed in wait() but before this publication found impl_ null 493   // armed in wait() but before this publication found impl_ null
494   // and returned a no-op. Observe it now and undo the insertion. 494   // and returned a no-op. Observe it now and undo the insertion.
HITCBC 495   25362 if (w->token_->stop_requested()) 495   24215 if (w->token_->stop_requested())
496   { 496   {
HITCBC 497   530 w->impl_ = nullptr; 497   239 w->impl_ = nullptr;
HITCBC 498   530 impl.waiter_ = nullptr; 498   239 impl.waiter_ = nullptr;
HITCBC 499   530 remove_timer_impl(impl); 499   239 remove_timer_impl(impl);
HITCBC 500   530 impl.might_have_pending_waits_.store( 500   239 impl.might_have_pending_waits_.store(
501   false, std::memory_order_release); 501   false, std::memory_order_release);
HITCBC 502   530 refresh_cached_nearest(); 502   239 refresh_cached_nearest();
HITCBC 503   530 lost_cancel = true; 503   239 lost_cancel = true;
HITCBC 504   530 notify = false; // insertion undone; nearest unchanged 504   239 notify = false; // insertion undone; nearest unchanged
505   } 505   }
HITCBC 506   25362 } 506   24215 }
HITCBC 507   25362 if (notify) 507   24215 if (notify)
HITCBC 508   8301 on_earliest_changed_(); 508   8070 on_earliest_changed_();
HITCBC 509   25362 if (lost_cancel) 509   24215 if (lost_cancel)
510   { 510   {
HITCBC 511   530 w->ec_ = make_error_code(capy::error::canceled); 511   239 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 512   530 sched_->post(&w->op_); 512   239 sched_->post(&w->op_);
513   } 513   }
HITCBC 514   25362 } 514   24215 }
515   515  
516   inline void 516   inline void
HITCBC 517   17501 timer_service::cancel_timer(timer::implementation& impl) 517   17154 timer_service::cancel_timer(timer::implementation& impl)
518   { 518   {
519   // Acquire pairs with the drains' release: a caller that skips the 519   // Acquire pairs with the drains' release: a caller that skips the
520   // lock may recycle the impl next, so the drain's writes to it must 520   // lock may recycle the impl next, so the drain's writes to it must
521   // happen before. 521   // happen before.
HITCBC 522   17501 if (!impl.might_have_pending_waits_.load(std::memory_order_acquire)) 522   17154 if (!impl.might_have_pending_waits_.load(std::memory_order_acquire))
HITCBC 523   17499 return; 523   17152 return;
524   524  
525   // No unlocked already-done fast-out here: it would need the 525   // No unlocked already-done fast-out here: it would need the
526   // non-atomic waiter_ (a race with concurrent drains), and an 526   // non-atomic waiter_ (a race with concurrent drains), and an
527   // index-only check is lifetime-unsafe because npos is stored 527   // index-only check is lifetime-unsafe because npos is stored
528   // before the drain finishes touching the impl. A stale-true 528   // before the drain finishes touching the impl. A stale-true
529   // flag is rare with the stateless API; the locked path below 529   // flag is rare with the stateless API; the locked path below
530   // re-validates. 530   // re-validates.
531   531  
HITCBC 532   2 waiter_node* canceled = nullptr; 532   2 waiter_node* canceled = nullptr;
533   533  
534   { 534   {
HITCBC 535   2 std::lock_guard lock(mutex_); 535   2 std::lock_guard lock(mutex_);
HITCBC 536   2 remove_timer_impl(impl); 536   2 remove_timer_impl(impl);
HITCBC 537   2 canceled = std::exchange(impl.waiter_, nullptr); 537   2 canceled = std::exchange(impl.waiter_, nullptr);
HITCBC 538   2 if (canceled) 538   2 if (canceled)
HITCBC 539   2 canceled->impl_ = nullptr; 539   2 canceled->impl_ = nullptr;
540   // Store false as the final touch of the impl under the lock so 540   // Store false as the final touch of the impl under the lock so
541   // a pre-lock false-flag check trusts it unqualified. 541   // a pre-lock false-flag check trusts it unqualified.
HITCBC 542   2 impl.might_have_pending_waits_.store(false, std::memory_order_release); 542   2 impl.might_have_pending_waits_.store(false, std::memory_order_release);
HITCBC 543   2 refresh_cached_nearest(); 543   2 refresh_cached_nearest();
HITCBC 544   2 } 544   2 }
545   545  
HITCBC 546   2 if (canceled) 546   2 if (canceled)
547   { 547   {
HITCBC 548   2 canceled->ec_ = make_error_code(capy::error::canceled); 548   2 canceled->ec_ = make_error_code(capy::error::canceled);
HITCBC 549   2 sched_->post(&canceled->op_); 549   2 sched_->post(&canceled->op_);
550   } 550   }
551   } 551   }
552   552  
553   inline void 553   inline void
HITCBC 554   3093 timer_service::cancel_waiter(waiter_node* w) 554   2474 timer_service::cancel_waiter(waiter_node* w)
555   { 555   {
556   { 556   {
HITCBC 557   3093 std::lock_guard lock(mutex_); 557   2474 std::lock_guard lock(mutex_);
558   // Already removed by another drain: cancel_timer, 558   // Already removed by another drain: cancel_timer,
559   // process_expired, or insert_waiter's lost-cancel recheck 559   // process_expired, or insert_waiter's lost-cancel recheck
HITCBC 560   3093 if (!w->impl_) 560   2474 if (!w->impl_)
HITCBC 561   921 return; 561   415 return;
HITCBC 562   2172 auto* impl = w->impl_; 562   2059 auto* impl = w->impl_;
HITCBC 563   2172 w->impl_ = nullptr; 563   2059 w->impl_ = nullptr;
HITCBC 564   2172 impl->waiter_ = nullptr; 564   2059 impl->waiter_ = nullptr;
HITCBC 565   2172 remove_timer_impl(*impl); 565   2059 remove_timer_impl(*impl);
HITCBC 566   2172 impl->might_have_pending_waits_.store(false, std::memory_order_release); 566   2059 impl->might_have_pending_waits_.store(false, std::memory_order_release);
HITCBC 567   2172 refresh_cached_nearest(); 567   2059 refresh_cached_nearest();
HITCBC 568   2172 ++cancels_posting_; 568   2059 ++cancels_posting_;
HITCBC 569   3093 } 569   2474 }
570   570  
HITCBC 571   2172 w->ec_ = make_error_code(capy::error::canceled); 571   2059 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 572   2172 sched_->post(&w->op_); 572   2059 sched_->post(&w->op_);
573   573  
HITCBC 574   2172 std::lock_guard lock(mutex_); 574   2059 std::lock_guard lock(mutex_);
HITCBC 575   2172 --cancels_posting_; 575   2059 --cancels_posting_;
HITCBC 576   2172 } 576   2059 }
577   577  
578   inline std::size_t 578   inline std::size_t
HITCBC 579   148868 timer_service::process_expired() 579   218750 timer_service::process_expired()
580   { 580   {
HITCBC 581   148868 intrusive_list<waiter_node> expired; 581   218750 intrusive_list<waiter_node> expired;
582   582  
583   { 583   {
HITCBC 584   148868 std::lock_guard lock(mutex_); 584   218750 std::lock_guard lock(mutex_);
HITCBC 585   148868 auto now = clock_type::now(); 585   218750 auto now = clock_type::now();
586   586  
HITCBC 587   168292 while (!heap_.empty() && heap_[0].time_ <= now) 587   237423 while (!heap_.empty() && heap_[0].time_ <= now)
588   { 588   {
HITCBC 589   19424 timer::implementation* t = heap_[0].timer_; 589   18673 timer::implementation* t = heap_[0].timer_;
HITCBC 590   19424 remove_timer_impl(*t); 590   18673 remove_timer_impl(*t);
HITCBC 591   19424 if (auto* w = std::exchange(t->waiter_, nullptr)) 591   18673 if (auto* w = std::exchange(t->waiter_, nullptr))
592   { 592   {
HITCBC 593   19424 w->impl_ = nullptr; 593   18673 w->impl_ = nullptr;
HITCBC 594   19424 w->ec_ = {}; 594   18673 w->ec_ = {};
HITCBC 595   19424 expired.push_back(w); 595   18673 expired.push_back(w);
596   } 596   }
HITCBC 597   19424 t->might_have_pending_waits_.store( 597   18673 t->might_have_pending_waits_.store(
598   false, std::memory_order_release); 598   false, std::memory_order_release);
599   } 599   }
600   600  
HITCBC 601   148868 refresh_cached_nearest(); 601   218750 refresh_cached_nearest();
HITCBC 602   148868 } 602   218750 }
603   603  
HITCBC 604   148868 std::size_t count = 0; 604   218750 std::size_t count = 0;
HITCBC 605   168292 while (auto* w = expired.pop_front()) 605   237423 while (auto* w = expired.pop_front())
606   { 606   {
HITCBC 607   19424 sched_->post(&w->op_); 607   18673 sched_->post(&w->op_);
HITCBC 608   19424 ++count; 608   18673 ++count;
HITCBC 609   19424 } 609   18673 }
610   610  
HITCBC 611   148868 return count; 611   218750 return count;
612   } 612   }
613   613  
614   inline void 614   inline void
HITCBC 615   22128 timer_service::remove_timer_impl(timer::implementation& impl) 615   20973 timer_service::remove_timer_impl(timer::implementation& impl)
616   { 616   {
HITCBC 617   22128 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed); 617   20973 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed);
HITCBC 618   22128 if (index >= heap_.size()) 618   20973 if (index >= heap_.size())
MISUBC 619   ✗ return; // Not in heap 619   ✗ return; // Not in heap
620   620  
HITCBC 621   22128 if (index == heap_.size() - 1) 621   20973 if (index == heap_.size() - 1)
622   { 622   {
623   // Last element, just pop 623   // Last element, just pop
HITCBC 624   3578 impl.heap_index_.store( 624   3023 impl.heap_index_.store(
625   (std::numeric_limits<std::size_t>::max)(), 625   (std::numeric_limits<std::size_t>::max)(),
626   std::memory_order_relaxed); 626   std::memory_order_relaxed);
HITCBC 627   3578 heap_.pop_back(); 627   3023 heap_.pop_back();
628   } 628   }
629   else 629   else
630   { 630   {
631   // Swap with last and reheapify 631   // Swap with last and reheapify
HITCBC 632   18550 swap_heap(index, heap_.size() - 1); 632   17950 swap_heap(index, heap_.size() - 1);
HITCBC 633   18550 impl.heap_index_.store( 633   17950 impl.heap_index_.store(
634   (std::numeric_limits<std::size_t>::max)(), 634   (std::numeric_limits<std::size_t>::max)(),
635   std::memory_order_relaxed); 635   std::memory_order_relaxed);
HITCBC 636   18550 heap_.pop_back(); 636   17950 heap_.pop_back();
637   637  
HITCBC 638   18550 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_) 638   17950 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_)
HITCBC 639   1 up_heap(index); 639   3 up_heap(index);
640   else 640   else
HITCBC 641   18549 down_heap(index); 641   17947 down_heap(index);
642   } 642   }
643   } 643   }
644   644  
645   inline void 645   inline void
HITCBC 646   25363 timer_service::up_heap(std::size_t index) 646   24218 timer_service::up_heap(std::size_t index)
647   { 647   {
HITCBC 648   33236 while (index > 0) 648   32155 while (index > 0)
649   { 649   {
HITCBC 650   24623 std::size_t parent = (index - 1) / 2; 650   23876 std::size_t parent = (index - 1) / 2;
HITCBC 651   24623 if (!(heap_[index].time_ < heap_[parent].time_)) 651   23876 if (!(heap_[index].time_ < heap_[parent].time_))
HITCBC 652   16750 break; 652   15939 break;
HITCBC 653   7873 swap_heap(index, parent); 653   7937 swap_heap(index, parent);
HITCBC 654   7873 index = parent; 654   7937 index = parent;
655   } 655   }
HITCBC 656   25363 } 656   24218 }
657   657  
658   inline void 658   inline void
HITCBC 659   18549 timer_service::down_heap(std::size_t index) 659   17947 timer_service::down_heap(std::size_t index)
660   { 660   {
HITCBC 661   18549 std::size_t child = index * 2 + 1; 661   17947 std::size_t child = index * 2 + 1;
HITCBC 662   41674 while (child < heap_.size()) 662   39466 while (child < heap_.size())
663   { 663   {
HITCBC 664   25391 std::size_t min_child = (child + 1 == heap_.size() || 664   23776 std::size_t min_child = (child + 1 == heap_.size() ||
HITCBC 665   21146 heap_[child].time_ < heap_[child + 1].time_) 665   19592 heap_[child].time_ < heap_[child + 1].time_)
HITCBC 666   46537 ? child 666   43368 ? child
HITCBC 667   25391 : child + 1; 667   23776 : child + 1;
668   668  
HITCBC 669   25391 if (heap_[index].time_ < heap_[min_child].time_) 669   23776 if (heap_[index].time_ < heap_[min_child].time_)
HITCBC 670   2266 break; 670   2257 break;
671   671  
HITCBC 672   23125 swap_heap(index, min_child); 672   21519 swap_heap(index, min_child);
HITCBC 673   23125 index = min_child; 673   21519 index = min_child;
HITCBC 674   23125 child = index * 2 + 1; 674   21519 child = index * 2 + 1;
675   } 675   }
HITCBC 676   18549 } 676   17947 }
677   677  
678   inline void 678   inline void
HITCBC 679   49548 timer_service::swap_heap(std::size_t i1, std::size_t i2) 679   47406 timer_service::swap_heap(std::size_t i1, std::size_t i2)
680   { 680   {
HITCBC 681   49548 heap_entry tmp = heap_[i1]; 681   47406 heap_entry tmp = heap_[i1];
HITCBC 682   49548 heap_[i1] = heap_[i2]; 682   47406 heap_[i1] = heap_[i2];
HITCBC 683   49548 heap_[i2] = tmp; 683   47406 heap_[i2] = tmp;
HITCBC 684   49548 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed); 684   47406 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed);
HITCBC 685   49548 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed); 685   47406 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed);
HITCBC 686   49548 } 686   47406 }
687   687  
688   // waiter_node's completion_op and canceller members are defined in 688   // waiter_node's completion_op and canceller members are defined in
689   // timer.cpp alongside implementation::wait(), for the same reason 689   // timer.cpp alongside implementation::wait(), for the same reason
690   // wait() lives there (see below). 690   // wait() lives there (see below).
691   691  
692   // timer::implementation::wait() is defined in timer.cpp, not here. 692   // timer::implementation::wait() is defined in timer.cpp, not here.
693   // It must be a non-inline definition in a translation unit that is 693   // It must be a non-inline definition in a translation unit that is
694   // always pulled into the link whenever detail::timer is used (every 694   // always pulled into the link whenever detail::timer is used (every
695   // consumer needs timer's constructors from that same object file). 695   // consumer needs timer's constructors from that same object file).
696   // An inline definition in this header would only be emitted in 696   // An inline definition in this header would only be emitted in
697   // translation units that happen to also include this header, which 697   // translation units that happen to also include this header, which
698   // is not guaranteed for every caller of wait_awaitable::await_suspend 698   // is not guaranteed for every caller of wait_awaitable::await_suspend
699   // in timer.hpp (e.g. code that only reaches timer.hpp through 699   // in timer.hpp (e.g. code that only reaches timer.hpp through
700   // delay.hpp, without transitively including a scheduler header). 700   // delay.hpp, without transitively including a scheduler header).
701   701  
702   // Free functions 702   // Free functions
703   703  
704   inline timer_service& 704   inline timer_service&
HITCBC 705   4413 get_timer_service(capy::execution_context& ctx, scheduler& sched) 705   4413 get_timer_service(capy::execution_context& ctx, scheduler& sched)
706   { 706   {
HITCBC 707   4413 return ctx.make_service<timer_service>(sched); 707   4413 return ctx.make_service<timer_service>(sched);
708   } 708   }
709   709  
710   } // namespace boost::corosio::detail 710   } // namespace boost::corosio::detail
711   711  
712   #endif 712   #endif