foundationdb/
budget.rs

1// Copyright 2018 foundationdb-rs developers, https://github.com/Clikengo/foundationdb-rs/graphs/contributors
2//
3// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5// http://opensource.org/licenses/MIT>, at your option. This file may not be
6// copied, modified, or distributed except according to those terms.
7
8//! Per-attempt usage accounting and the client-side budget.
9//!
10//! <div class="warning">
11//!
12//! The budget is a **client-side feature of this Rust binding**, not a native
13//! FoundationDB limit. Nothing here is enforced by the cluster: the binding
14//! counts the bytes and calls that go through [`Transaction`](crate::Transaction)
15//! and compares them with the configured limits when you ask it to, with
16//! [`Transaction::check_client_budget`](crate::Transaction::check_client_budget).
17//! The numbers are therefore an **estimate**, checked *between* operations:
18//! a single operation can overshoot a limit, and reads that are still in flight
19//! are not counted yet.
20//!
21//! It is fully decoupled from
22//! [`TransactionOption::Timeout`](crate::options::TransactionOption::Timeout) and
23//! [`TransactionOption::SizeLimit`](crate::options::TransactionOption::SizeLimit),
24//! which are enforced by the C client. Use those to bound what the database
25//! does; use the budget to bound what your own code does before it reaches
26//! them.
27//!
28//! </div>
29//!
30//! # Per-attempt semantics
31//!
32//! Accounting is always on and scoped to a single *transaction attempt*: usage
33//! is reset whenever the transaction restarts (`on_error`, `reset`), while the
34//! configured limits survive and apply to the new attempt. A retried
35//! transaction therefore gets a fresh time and byte allowance, exactly like it
36//! gets a fresh read version.
37//!
38//! # Determinism and simulation
39//!
40//! The byte and call counters are deterministic: they depend only on the
41//! operations your code issues. [`ClientBudget::time_limit`] is not, unless you
42//! say where time comes from: give the budget a [`Clock`] with
43//! [`ClientBudget::with_clock`] and every elapsed time of the attempt is
44//! measured with it. Under FoundationDB's deterministic simulator, pass a clock
45//! backed by simulated time so that a run replays identically.
46//!
47//! Without one the [`WallClock`] is used, which is the right default outside
48//! simulation but makes time budgets non-reproducible: do not rely on them in a
49//! simulated workload. See [`crate::env`] for the whole picture.
50
51use std::collections::HashMap;
52use std::fmt;
53use std::sync::Arc;
54use std::sync::Mutex;
55use std::sync::atomic::{AtomicU64, Ordering};
56use std::time::Duration;
57
58use crate::env::{Clock, WallClock};
59use crate::metrics::MetricKey;
60
61/// Client-side limits applied to a single transaction attempt.
62///
63/// Each field is optional, `None` meaning "no limit". A default `ClientBudget`
64/// limits nothing.
65///
66/// See the [module documentation](self): these limits are computed by the
67/// binding from what it observes, not enforced by FoundationDB.
68///
69/// # Example
70///
71/// ```
72/// # use foundationdb::*;
73/// # use std::time::Duration;
74/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
75/// # let db = Database::default()?;
76/// let trx = db.create_trx()?;
77/// trx.set_client_budget(ClientBudget {
78///     time_limit: Some(Duration::from_secs(2)),
79///     max_bytes_read: Some(10 * 1024 * 1024),
80///     ..ClientBudget::default()
81/// });
82/// # Ok(())
83/// # }
84/// ```
85#[derive(Debug, Clone, Default)]
86pub struct ClientBudget {
87    /// Maximum time an attempt may spend, measured from the start of the attempt
88    /// with the [`clock`](Self::clock) of this budget.
89    pub time_limit: Option<Duration>,
90    /// Maximum number of bytes read by an attempt, keys and values summed.
91    pub max_bytes_read: Option<u64>,
92    /// Maximum number of bytes written by an attempt, keys, values and mutation
93    /// parameters summed.
94    pub max_bytes_written: Option<u64>,
95    /// Where [`time_limit`](Self::time_limit) reads time from, `None` meaning
96    /// the [`WallClock`].
97    ///
98    /// Set it with [`with_clock`](Self::with_clock) to keep time budgets
99    /// deterministic, see the [module documentation](self).
100    pub clock: Option<Arc<dyn Clock>>,
101}
102
103impl ClientBudget {
104    /// Measures the attempts with `clock` instead of the [`WallClock`].
105    #[cfg_attr(
106        feature = "trace",
107        tracing::instrument(level = "debug", skip(self, clock))
108    )]
109    pub fn with_clock(self, clock: impl Clock + 'static) -> Self {
110        Self {
111            clock: Some(Arc::new(clock)),
112            ..self
113        }
114    }
115
116    /// Checks `usage` against these limits, returning the first exceeded one.
117    pub(crate) fn check(&self, usage: &AttemptUsage) -> Result<(), BudgetExceeded> {
118        if let Some(limit) = self.time_limit {
119            let used = usage.elapsed().as_millis() as u64;
120            let limit = limit.as_millis() as u64;
121            if used > limit {
122                return Err(BudgetExceeded {
123                    kind: BudgetKind::Time,
124                    used,
125                    limit,
126                });
127            }
128        }
129
130        if let Some(limit) = self.max_bytes_read {
131            let used = usage.bytes_read();
132            if used > limit {
133                return Err(BudgetExceeded {
134                    kind: BudgetKind::BytesRead,
135                    used,
136                    limit,
137                });
138            }
139        }
140
141        if let Some(limit) = self.max_bytes_written {
142            let used = usage.bytes_written();
143            if used > limit {
144                return Err(BudgetExceeded {
145                    kind: BudgetKind::BytesWritten,
146                    used,
147                    limit,
148                });
149            }
150        }
151
152        Ok(())
153    }
154}
155
156/// Which [`ClientBudget`] limit was exceeded.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub enum BudgetKind {
159    /// [`ClientBudget::time_limit`], in milliseconds.
160    Time,
161    /// [`ClientBudget::max_bytes_read`], in bytes.
162    BytesRead,
163    /// [`ClientBudget::max_bytes_written`], in bytes.
164    BytesWritten,
165}
166
167impl fmt::Display for BudgetKind {
168    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
169        match self {
170            BudgetKind::Time => write!(f, "time"),
171            BudgetKind::BytesRead => write!(f, "bytes read"),
172            BudgetKind::BytesWritten => write!(f, "bytes written"),
173        }
174    }
175}
176
177/// A [`ClientBudget`] limit was exceeded by the current transaction attempt.
178///
179/// `used` and `limit` are expressed in milliseconds for [`BudgetKind::Time`],
180/// in bytes otherwise.
181#[derive(Debug, Clone, Copy, PartialEq, Eq)]
182pub struct BudgetExceeded {
183    /// The limit that was exceeded.
184    pub kind: BudgetKind,
185    /// What the attempt used, milliseconds for [`BudgetKind::Time`], bytes
186    /// otherwise.
187    pub used: u64,
188    /// The configured limit, in the same unit as `used`.
189    pub limit: u64,
190}
191
192impl fmt::Display for BudgetExceeded {
193    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
194        let unit = match self.kind {
195            BudgetKind::Time => "ms",
196            BudgetKind::BytesRead | BudgetKind::BytesWritten => "bytes",
197        };
198        write!(
199            f,
200            "client budget exceeded ({}): used {} {}, limit {} {}. This is a client-side estimate of this binding, not a FoundationDB limit",
201            self.kind, self.used, unit, self.limit, unit
202        )
203    }
204}
205
206impl std::error::Error for BudgetExceeded {}
207
208/// Usage accounted for a single transaction attempt.
209///
210/// Counters are incremented by [`Transaction`](crate::Transaction) as
211/// operations are issued (writes) or resolved (reads). They are always on: no
212/// instrumentation is required to get them.
213///
214/// A new instance is created for every attempt, see the
215/// [module documentation](self).
216///
217/// The application metrics recorded with
218/// [`Transaction::set_custom_metric`](crate::Transaction::set_custom_metric) are
219/// stored here too, and are therefore per-attempt as well. Their map is
220/// allocated on first use, so a transaction that never records one pays
221/// nothing. Without a metrics consumer collecting them (typically
222/// [`Database::instrumented_run`](crate::Database::instrumented_run)), they are
223/// simply dropped with the generation.
224#[derive(Debug)]
225pub struct AttemptUsage {
226    /// Where the elapsed time of this attempt is measured.
227    clock: Arc<dyn Clock>,
228    /// The monotonic reading taken when the attempt started.
229    started_at: Duration,
230    bytes_read: AtomicU64,
231    bytes_written: AtomicU64,
232    keys_values_fetched: AtomicU64,
233    call_get: AtomicU64,
234    call_get_range: AtomicU64,
235    call_set: AtomicU64,
236    call_clear: AtomicU64,
237    call_clear_range: AtomicU64,
238    call_atomic_op: AtomicU64,
239    /// Application metrics of this attempt, allocated on first use.
240    custom: Mutex<Option<HashMap<MetricKey, u64>>>,
241}
242
243impl Default for AttemptUsage {
244    fn default() -> Self {
245        Self::new()
246    }
247}
248
249impl AttemptUsage {
250    /// Starts a new, empty accounting generation, measured with the
251    /// [`WallClock`].
252    pub fn new() -> Self {
253        Self::with_clock(None)
254    }
255
256    /// Starts a new, empty accounting generation measured with `clock`, or with
257    /// the [`WallClock`] when it is `None`.
258    pub(crate) fn with_clock(clock: Option<Arc<dyn Clock>>) -> Self {
259        let clock = clock.unwrap_or_else(|| Arc::new(WallClock::new()));
260        Self {
261            started_at: clock.monotonic(),
262            clock,
263            bytes_read: AtomicU64::new(0),
264            bytes_written: AtomicU64::new(0),
265            keys_values_fetched: AtomicU64::new(0),
266            call_get: AtomicU64::new(0),
267            call_get_range: AtomicU64::new(0),
268            call_set: AtomicU64::new(0),
269            call_clear: AtomicU64::new(0),
270            call_clear_range: AtomicU64::new(0),
271            call_atomic_op: AtomicU64::new(0),
272            custom: Mutex::new(None),
273        }
274    }
275
276    /// Time elapsed since the attempt started, measured with the clock the
277    /// attempt was stamped with.
278    ///
279    /// A clock that goes backwards yields zero rather than panicking.
280    pub fn elapsed(&self) -> Duration {
281        self.clock.monotonic().saturating_sub(self.started_at)
282    }
283
284    /// Bytes read so far, keys and values summed.
285    pub fn bytes_read(&self) -> u64 {
286        self.bytes_read.load(Ordering::Relaxed)
287    }
288
289    /// Bytes written so far, keys, values and mutation parameters summed.
290    pub fn bytes_written(&self) -> u64 {
291        self.bytes_written.load(Ordering::Relaxed)
292    }
293
294    /// Takes a consistent-enough copy of every counter.
295    ///
296    /// Counters are read one after the other without a lock, so a snapshot
297    /// taken while operations are completing may mix values from slightly
298    /// different instants.
299    pub fn snapshot(&self) -> UsageSnapshot {
300        UsageSnapshot {
301            elapsed: self.elapsed(),
302            bytes_read: self.bytes_read.load(Ordering::Relaxed),
303            bytes_written: self.bytes_written.load(Ordering::Relaxed),
304            keys_values_fetched: self.keys_values_fetched.load(Ordering::Relaxed),
305            call_get: self.call_get.load(Ordering::Relaxed),
306            call_get_range: self.call_get_range.load(Ordering::Relaxed),
307            call_set: self.call_set.load(Ordering::Relaxed),
308            call_clear: self.call_clear.load(Ordering::Relaxed),
309            call_clear_range: self.call_clear_range.load(Ordering::Relaxed),
310            call_atomic_op: self.call_atomic_op.load(Ordering::Relaxed),
311        }
312    }
313
314    /// Records a resolved `get` or `get_key`.
315    pub(crate) fn record_get(&self, bytes: u64, keys_values: u64) {
316        self.bytes_read.fetch_add(bytes, Ordering::Relaxed);
317        self.keys_values_fetched
318            .fetch_add(keys_values, Ordering::Relaxed);
319        self.call_get.fetch_add(1, Ordering::Relaxed);
320    }
321
322    /// Records a resolved `get_range` batch.
323    pub(crate) fn record_get_range(&self, bytes: u64, keys_values: u64) {
324        self.bytes_read.fetch_add(bytes, Ordering::Relaxed);
325        self.keys_values_fetched
326            .fetch_add(keys_values, Ordering::Relaxed);
327        self.call_get_range.fetch_add(1, Ordering::Relaxed);
328    }
329
330    /// Records a `set`.
331    pub(crate) fn record_set(&self, bytes: u64) {
332        self.bytes_written.fetch_add(bytes, Ordering::Relaxed);
333        self.call_set.fetch_add(1, Ordering::Relaxed);
334    }
335
336    /// Records a `clear`.
337    pub(crate) fn record_clear(&self, bytes: u64) {
338        self.bytes_written.fetch_add(bytes, Ordering::Relaxed);
339        self.call_clear.fetch_add(1, Ordering::Relaxed);
340    }
341
342    /// Records a `clear_range`.
343    pub(crate) fn record_clear_range(&self, bytes: u64) {
344        self.bytes_written.fetch_add(bytes, Ordering::Relaxed);
345        self.call_clear_range.fetch_add(1, Ordering::Relaxed);
346    }
347
348    /// Records an `atomic_op`.
349    pub(crate) fn record_atomic_op(&self, bytes: u64) {
350        self.bytes_written.fetch_add(bytes, Ordering::Relaxed);
351        self.call_atomic_op.fetch_add(1, Ordering::Relaxed);
352    }
353
354    /// Sets an application metric of this attempt, replacing any previous value.
355    pub(crate) fn set_custom(&self, key: MetricKey, value: u64) {
356        self.with_custom(|custom| {
357            custom.insert(key, value);
358        });
359    }
360
361    /// Adds `amount` to an application metric of this attempt, starting from
362    /// zero if it was not recorded yet.
363    pub(crate) fn increment_custom(&self, key: MetricKey, amount: u64) {
364        self.with_custom(|custom| {
365            *custom.entry(key).or_insert(0) += amount;
366        });
367    }
368
369    /// The application metrics recorded during this attempt.
370    pub fn custom_metrics(&self) -> HashMap<MetricKey, u64> {
371        self.custom
372            .lock()
373            .unwrap_or_else(|poisoned| poisoned.into_inner())
374            .clone()
375            .unwrap_or_default()
376    }
377
378    fn with_custom<R>(&self, f: impl FnOnce(&mut HashMap<MetricKey, u64>) -> R) -> R {
379        let mut custom = self
380            .custom
381            .lock()
382            .unwrap_or_else(|poisoned| poisoned.into_inner());
383        f(custom.get_or_insert_with(HashMap::new))
384    }
385}
386
387/// A copy of the counters of a single transaction attempt, taken by
388/// [`AttemptUsage::snapshot`] or
389/// [`Transaction::attempt_usage`](crate::Transaction::attempt_usage).
390///
391/// Every byte count is a client-side estimate, see the
392/// [module documentation](self).
393#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
394pub struct UsageSnapshot {
395    /// Time elapsed since the attempt started.
396    pub elapsed: Duration,
397    /// Bytes read: for each resolved read, the key(s) requested plus the
398    /// key-values returned.
399    pub bytes_read: u64,
400    /// Bytes written: keys, values and mutation parameters handed to the
401    /// client. For `clear_range` only the two boundary keys are counted, **not**
402    /// the volume of data the range deletes.
403    pub bytes_written: u64,
404    /// Number of key-values returned by resolved reads.
405    pub keys_values_fetched: u64,
406    /// Number of `get` and `get_key` calls resolved.
407    pub call_get: u64,
408    /// Number of `get_range` batches resolved. A single `get_ranges` stream
409    /// counts once per underlying FoundationDB batch, not once per stream.
410    pub call_get_range: u64,
411    /// Number of `set` calls.
412    pub call_set: u64,
413    /// Number of `clear` calls.
414    pub call_clear: u64,
415    /// Number of `clear_range` calls.
416    pub call_clear_range: u64,
417    /// Number of `atomic_op` calls.
418    pub call_atomic_op: u64,
419}
420
421/// The accounting generation currently active on a transaction.
422///
423/// Operations clone the `Arc` out of the slot when they are **issued** and
424/// record into that clone when they complete. Starting a new attempt swaps the
425/// slot with a fresh [`AttemptUsage`] instead of zeroing the current one, so a
426/// read issued during attempt N that completes during attempt N+1 records into
427/// the counters of attempt N, which nobody reads anymore, and cannot pollute
428/// N+1.
429#[derive(Debug, Default)]
430pub(crate) struct UsageSlot(Mutex<Arc<AttemptUsage>>);
431
432impl UsageSlot {
433    /// The generation to record into for an operation issued now.
434    pub(crate) fn current(&self) -> Arc<AttemptUsage> {
435        self.0
436            .lock()
437            .unwrap_or_else(|poisoned| poisoned.into_inner())
438            .clone()
439    }
440
441    /// Starts a fresh generation measured with `clock` (the [`WallClock`] when
442    /// `None`), leaving the previous one to the in-flight operations still
443    /// holding it.
444    pub(crate) fn begin(&self, clock: Option<Arc<dyn Clock>>) {
445        let mut slot = self
446            .0
447            .lock()
448            .unwrap_or_else(|poisoned| poisoned.into_inner());
449        *slot = Arc::new(AttemptUsage::with_clock(clock));
450    }
451}
452
453#[cfg(test)]
454mod tests {
455    use super::*;
456
457    /// A clock the test moves by hand, in milliseconds.
458    #[derive(Debug, Clone, Default)]
459    struct FakeClock(Arc<AtomicU64>);
460
461    impl FakeClock {
462        fn set_millis(&self, millis: u64) {
463            self.0.store(millis, Ordering::Relaxed);
464        }
465    }
466
467    impl Clock for FakeClock {
468        fn monotonic(&self) -> Duration {
469            Duration::from_millis(self.0.load(Ordering::Relaxed))
470        }
471
472        fn wall(&self) -> Duration {
473            self.monotonic()
474        }
475    }
476
477    #[test]
478    fn budget_without_limits_never_fails() {
479        let usage = AttemptUsage::new();
480        usage.record_get(1_000_000, 10);
481        usage.record_set(1_000_000);
482
483        assert!(ClientBudget::default().check(&usage).is_ok());
484    }
485
486    #[test]
487    fn budget_allows_usage_up_to_the_limit() {
488        let budget = ClientBudget {
489            max_bytes_read: Some(10),
490            max_bytes_written: Some(10),
491            ..ClientBudget::default()
492        };
493
494        let usage = AttemptUsage::new();
495        usage.record_get(10, 1);
496        usage.record_set(10);
497
498        assert!(budget.check(&usage).is_ok());
499    }
500
501    #[test]
502    fn budget_reports_exceeded_bytes_read() {
503        let budget = ClientBudget {
504            max_bytes_read: Some(10),
505            ..ClientBudget::default()
506        };
507
508        let usage = AttemptUsage::new();
509        usage.record_get(7, 1);
510        assert!(budget.check(&usage).is_ok());
511        usage.record_get_range(5, 2);
512
513        let err = budget.check(&usage).unwrap_err();
514        assert_eq!(err.kind, BudgetKind::BytesRead);
515        assert_eq!(err.used, 12);
516        assert_eq!(err.limit, 10);
517        assert!(err.to_string().contains("client-side estimate"));
518    }
519
520    #[test]
521    fn budget_reports_exceeded_bytes_written() {
522        let budget = ClientBudget {
523            max_bytes_written: Some(4),
524            ..ClientBudget::default()
525        };
526
527        let usage = AttemptUsage::new();
528        usage.record_set(2);
529        usage.record_clear(1);
530        usage.record_clear_range(1);
531        assert!(budget.check(&usage).is_ok());
532        usage.record_atomic_op(1);
533
534        let err = budget.check(&usage).unwrap_err();
535        assert_eq!(err.kind, BudgetKind::BytesWritten);
536        assert_eq!(err.used, 5);
537        assert_eq!(err.limit, 4);
538    }
539
540    #[test]
541    fn budget_reports_exceeded_time() {
542        let clock = FakeClock::default();
543        clock.set_millis(1_000);
544
545        let budget = ClientBudget {
546            time_limit: Some(Duration::from_millis(20)),
547            ..ClientBudget::default()
548        }
549        .with_clock(clock.clone());
550
551        let usage = AttemptUsage::with_clock(budget.clock.clone());
552        clock.set_millis(1_020);
553        assert!(budget.check(&usage).is_ok(), "the limit itself is allowed");
554
555        clock.set_millis(1_035);
556
557        let err = budget.check(&usage).unwrap_err();
558        assert_eq!(err.kind, BudgetKind::Time);
559        assert_eq!(err.used, 35);
560        assert_eq!(err.limit, 20);
561        assert!(err.to_string().contains("client-side estimate"));
562    }
563
564    #[test]
565    fn elapsed_follows_a_custom_clock() {
566        let clock = FakeClock::default();
567        clock.set_millis(500);
568
569        let usage = AttemptUsage::with_clock(Some(Arc::new(clock.clone())));
570        assert_eq!(usage.elapsed(), Duration::ZERO);
571
572        clock.set_millis(700);
573        assert_eq!(usage.elapsed(), Duration::from_millis(200));
574        assert_eq!(usage.snapshot().elapsed, Duration::from_millis(200));
575    }
576
577    #[test]
578    fn a_clock_going_backwards_saturates_to_zero() {
579        let clock = FakeClock::default();
580        clock.set_millis(1_000);
581
582        let usage = AttemptUsage::with_clock(Some(Arc::new(clock.clone())));
583        clock.set_millis(10);
584
585        assert_eq!(usage.elapsed(), Duration::ZERO);
586    }
587
588    #[test]
589    fn a_new_generation_is_stamped_with_the_given_clock() {
590        let clock = FakeClock::default();
591        clock.set_millis(100);
592
593        let slot = UsageSlot::default();
594        let previous = slot.current();
595
596        slot.begin(Some(Arc::new(clock.clone())));
597        let current = slot.current();
598        clock.set_millis(160);
599
600        assert_eq!(current.elapsed(), Duration::from_millis(60));
601        // The previous generation keeps the wall clock it was stamped with.
602        assert!(previous.elapsed() < Duration::from_millis(60));
603    }
604
605    #[test]
606    fn snapshot_counts_every_operation() {
607        let usage = AttemptUsage::new();
608        usage.record_get(3, 1);
609        usage.record_get_range(7, 2);
610        usage.record_set(4);
611        usage.record_clear(5);
612        usage.record_clear_range(6);
613        usage.record_atomic_op(7);
614
615        let snapshot = usage.snapshot();
616        assert_eq!(snapshot.bytes_read, 10);
617        assert_eq!(snapshot.bytes_written, 22);
618        assert_eq!(snapshot.keys_values_fetched, 3);
619        assert_eq!(snapshot.call_get, 1);
620        assert_eq!(snapshot.call_get_range, 1);
621        assert_eq!(snapshot.call_set, 1);
622        assert_eq!(snapshot.call_clear, 1);
623        assert_eq!(snapshot.call_clear_range, 1);
624        assert_eq!(snapshot.call_atomic_op, 1);
625    }
626
627    #[test]
628    fn custom_metrics_are_scoped_to_the_generation() {
629        let slot = UsageSlot::default();
630        let key = MetricKey::new("documents", &[("kind", "user")]);
631
632        assert!(slot.current().custom_metrics().is_empty());
633
634        slot.current().set_custom(key.clone(), 2);
635        slot.current().increment_custom(key.clone(), 3);
636        assert_eq!(slot.current().custom_metrics().get(&key), Some(&5));
637
638        let previous = slot.current();
639        slot.begin(None);
640
641        assert!(slot.current().custom_metrics().is_empty());
642        assert_eq!(previous.custom_metrics().get(&key), Some(&5));
643    }
644
645    #[test]
646    fn a_new_generation_starts_empty() {
647        let slot = UsageSlot::default();
648        slot.current().record_set(42);
649        assert_eq!(slot.current().snapshot().bytes_written, 42);
650
651        slot.begin(None);
652
653        let snapshot = slot.current().snapshot();
654        assert_eq!(
655            UsageSnapshot {
656                elapsed: Duration::ZERO,
657                ..snapshot
658            },
659            UsageSnapshot::default()
660        );
661    }
662
663    #[test]
664    fn a_stale_generation_does_not_pollute_the_next_one() {
665        let slot = UsageSlot::default();
666
667        // An operation issued during the first attempt, still in flight.
668        let in_flight = slot.current();
669
670        slot.begin(None);
671        slot.current().record_get(10, 1);
672
673        // The stale operation completes and records into its own generation.
674        in_flight.record_get(999, 99);
675
676        let snapshot = slot.current().snapshot();
677        assert_eq!(snapshot.bytes_read, 10);
678        assert_eq!(snapshot.keys_values_fetched, 1);
679        assert_eq!(snapshot.call_get, 1);
680        assert_eq!(in_flight.snapshot().bytes_read, 999);
681    }
682}