foundationdb/
metrics.rs

1//! Per-attempt metrics collected by [`Database::instrumented_run`](crate::Database::instrumented_run).
2//!
3//! The report is a list of [`AttemptMetrics`], one per transaction attempt, in
4//! order: a retried transaction keeps everything it did in its earlier
5//! attempts. Operation counters come from the always-on
6//! [`UsageSnapshot`] of the attempt, so
7//! instrumentation only adds the timings, the outcome and the aggregates on
8//! top of what the binding already counts.
9
10use crate::budget::{AttemptUsage, UsageSnapshot};
11use crate::error::FdbError;
12use crate::transaction::ConflictingKeyRange;
13use std::collections::HashMap;
14use std::sync::{Arc, Mutex};
15use std::time::Duration;
16
17/// Label key-value pairs for metrics
18pub type Labels = Vec<(String, String)>;
19
20/// Unique key for a metric: name + labels
21#[derive(Clone, Hash, Eq, PartialEq, Debug)]
22pub struct MetricKey {
23    pub name: String,
24    pub labels: Labels,
25}
26
27impl MetricKey {
28    /// Create a new MetricKey
29    ///
30    /// # Arguments
31    /// * `name` - The name of the metric
32    /// * `labels` - Key-value pairs for labeling the metric
33    ///
34    /// # Returns
35    /// * `MetricKey` - A new MetricKey instance
36    pub fn new(name: &str, labels: &[(&str, &str)]) -> Self {
37        // Convert labels to owned strings
38        let mut sorted_labels: Labels = labels
39            .iter()
40            .map(|(k, v)| (k.to_string(), v.to_string()))
41            .collect();
42
43        // Sort labels by key to ensure consistent ordering
44        sorted_labels.sort_by(|a, b| a.0.cmp(&b.0));
45
46        Self {
47            name: name.to_string(),
48            labels: sorted_labels,
49        }
50    }
51}
52
53/// How a transaction attempt ended.
54///
55/// Marked non exhaustive: finer-grained failure reporting is expected to add
56/// variants.
57#[derive(Debug, Clone)]
58#[non_exhaustive]
59pub enum AttemptOutcome {
60    /// The attempt committed. For a run, only the last attempt can be in this
61    /// state.
62    Committed,
63    /// The attempt was retried, `cause` being the error handed to `on_error`.
64    /// A closure error asking for a retry without a native error underneath
65    /// surfaces here as code 1020 (`not_committed`).
66    Retried { cause: FdbError },
67    /// The attempt ended the run without committing: the error was not
68    /// retryable, or the retries were exhausted.
69    Failed,
70}
71
72/// The conflicting key ranges of a transaction attempt.
73///
74/// They are only read after a commit conflict, and only carry ranges when
75/// [`TransactionOption::ReportConflictingKeys`](crate::options::TransactionOption::ReportConflictingKeys)
76/// was set on the transaction.
77#[derive(Debug, Clone, Default)]
78pub enum ConflictKeys {
79    /// The attempt did not fail on a commit conflict, so nothing was read.
80    #[default]
81    NotRequested,
82    /// The special keyspace was read. The list is empty when
83    /// `ReportConflictingKeys` was not enabled.
84    Available(Vec<ConflictingKeyRange>),
85    /// The special keyspace read failed.
86    ReadFailed(FdbError),
87}
88
89impl ConflictKeys {
90    /// The conflicting ranges, empty unless they were read successfully.
91    pub fn ranges(&self) -> &[ConflictingKeyRange] {
92        match self {
93            ConflictKeys::Available(ranges) => ranges,
94            ConflictKeys::NotRequested | ConflictKeys::ReadFailed(_) => &[],
95        }
96    }
97}
98
99/// Everything recorded about a single transaction attempt.
100///
101/// One is pushed to [`MetricsReport::attempts`] when the attempt ends, so an
102/// attempt that was retried keeps its own counters, timings and custom metrics
103/// instead of being overwritten by the next one.
104#[derive(Debug, Clone)]
105pub struct AttemptMetrics {
106    /// Position of the attempt in the run, starting at 0.
107    pub index: usize,
108    /// Operation counters and bytes of the attempt, from the always-on
109    /// accounting. See [`crate::budget`] for how precise they are.
110    pub usage: UsageSnapshot,
111    /// Application metrics recorded during the attempt with
112    /// [`Transaction::set_custom_metric`](crate::Transaction::set_custom_metric).
113    pub custom_metrics: HashMap<MetricKey, u64>,
114    /// Wall clock of the attempt, measured from its start to the end of its
115    /// work: the retry backoff performed by `on_error` is **not** included.
116    pub duration: Option<Duration>,
117    /// Time spent in `commit`, `None` if the attempt never reached it.
118    pub commit_duration: Option<Duration>,
119    /// Time spent waiting on `get_read_version`, `None` if it was never called.
120    /// The first completion of the attempt is kept.
121    pub grv_duration: Option<Duration>,
122    /// Time spent in `on_error`, backoff included. `None` if `on_error` did not
123    /// run, which is the case for the last attempt of a run.
124    pub on_error_duration: Option<Duration>,
125    /// How the attempt ended.
126    pub outcome: AttemptOutcome,
127    /// Conflicting key ranges, only read after a commit conflict.
128    pub conflicting_keys: ConflictKeys,
129    /// Read version of the attempt, recorded when
130    /// [`Transaction::get_read_version`](crate::Transaction::get_read_version)
131    /// is called. The binding never fetches it on its own.
132    pub read_version: Option<i64>,
133}
134
135/// Transaction-level information that spans attempts
136#[derive(Debug, Default, Clone)]
137pub struct TransactionInfo {
138    /// Number of retries performed, that is `attempts.len() - 1` for a
139    /// completed run
140    pub retries: u64,
141    /// Number of retries caused by commit conflicts (`not_committed`, error 1020)
142    pub conflict_count: u64,
143    /// Last read version observed, see [`AttemptMetrics::read_version`]
144    pub read_version: Option<i64>,
145    /// Commit version of the committed attempt
146    pub commit_version: Option<i64>,
147}
148
149/// The metrics of a whole `instrumented_run`: the detail of every attempt plus
150/// the aggregates that only make sense for the run as a whole.
151#[derive(Debug, Clone, Default)]
152pub struct MetricsReport {
153    /// One entry per transaction attempt, in order. Nothing is lost across
154    /// retries.
155    pub attempts: Vec<AttemptMetrics>,
156    /// Wall clock of the whole run, every attempt and backoff included.
157    pub total_duration: Option<Duration>,
158    /// Information that spans attempts.
159    pub transaction: TransactionInfo,
160}
161
162impl MetricsReport {
163    /// Sums the usage of every attempt, retries included.
164    ///
165    /// `elapsed` is the sum of the attempt durations, which is shorter than
166    /// [`total_duration`](Self::total_duration): the retry backoffs are not
167    /// part of any attempt.
168    pub fn total_usage(&self) -> UsageSnapshot {
169        self.attempts
170            .iter()
171            .fold(UsageSnapshot::default(), |mut total, attempt| {
172                let usage = &attempt.usage;
173                total.elapsed += usage.elapsed;
174                total.bytes_read += usage.bytes_read;
175                total.bytes_written += usage.bytes_written;
176                total.keys_values_fetched += usage.keys_values_fetched;
177                total.call_get += usage.call_get;
178                total.call_get_range += usage.call_get_range;
179                total.call_set += usage.call_set;
180                total.call_clear += usage.call_clear;
181                total.call_clear_range += usage.call_clear_range;
182                total.call_atomic_op += usage.call_atomic_op;
183                total
184            })
185    }
186
187    /// The attempt that ended the run, `None` if no attempt was ever started.
188    pub fn last_attempt(&self) -> Option<&AttemptMetrics> {
189        self.attempts.last()
190    }
191}
192
193/// The attempt currently being recorded.
194///
195/// Holds the accounting generation of the transaction, so that the attempt can
196/// be pushed to the report even after the transaction itself is gone (a
197/// non-retryable `on_error` consumes it), and the pieces that are known before
198/// the attempt ends.
199#[derive(Debug, Default)]
200struct OpenAttempt {
201    usage: Option<Arc<AttemptUsage>>,
202    duration: Option<Duration>,
203    commit_duration: Option<Duration>,
204    grv_duration: Option<Duration>,
205    conflicting_keys: ConflictKeys,
206    read_version: Option<i64>,
207}
208
209/// Collects the metrics of a transaction, attempt after attempt.
210///
211/// This is the handle shared between the transaction, the runner hooks and the
212/// caller: cloning it clones the handle, not the data. The recording methods
213/// are called by the binding as the transaction progresses, the report is read
214/// back with [`get_metrics_data`](Self::get_metrics_data).
215#[derive(Debug, Clone, Default)]
216pub struct TransactionMetrics {
217    /// The report being built.
218    pub metrics: Arc<Mutex<MetricsReport>>,
219    /// The attempt being recorded, pushed to the report when it ends.
220    open: Arc<Mutex<OpenAttempt>>,
221}
222
223impl TransactionMetrics {
224    /// Create a new instance of TransactionMetrics
225    pub fn new() -> Self {
226        Self::default()
227    }
228
229    fn report(&self) -> std::sync::MutexGuard<'_, MetricsReport> {
230        self.metrics
231            .lock()
232            .unwrap_or_else(|poisoned| poisoned.into_inner())
233    }
234
235    fn open(&self) -> std::sync::MutexGuard<'_, OpenAttempt> {
236        self.open
237            .lock()
238            .unwrap_or_else(|poisoned| poisoned.into_inner())
239    }
240
241    /// Starts recording a new attempt, on the accounting generation the
242    /// transaction just entered.
243    ///
244    /// Anything recorded on a previous attempt that was not ended with
245    /// [`finish_attempt`](Self::finish_attempt) is dropped: the transaction
246    /// restarted without reaching a conclusion, typically because
247    /// [`Transaction::set_client_budget`](crate::Transaction::set_client_budget)
248    /// or [`Transaction::reset`](crate::Transaction::reset) was called.
249    pub(crate) fn begin_attempt(&self, usage: Arc<AttemptUsage>) {
250        let mut open = self.open();
251
252        #[cfg(feature = "trace")]
253        if let Some(dropped) = open.usage.as_ref() {
254            // `elapsed` always moves, so the counters are what tells whether
255            // the attempt being dropped did anything worth reporting. Some
256            // activity is recorded straight on `OpenAttempt` instead of
257            // `AttemptUsage` (read version, conflicting keys, commit
258            // duration, grv duration), so it has to be checked too.
259            let snapshot = dropped.snapshot();
260            let idle = snapshot
261                == UsageSnapshot {
262                    elapsed: snapshot.elapsed,
263                    ..UsageSnapshot::default()
264                }
265                && dropped.custom_metrics().is_empty()
266                && open.read_version.is_none()
267                && open.duration.is_none()
268                && open.commit_duration.is_none()
269                && open.grv_duration.is_none()
270                && matches!(open.conflicting_keys, ConflictKeys::NotRequested);
271
272            if !idle {
273                tracing::warn!(
274                    "a new attempt was opened while the previous one was still open, its usage and custom metrics are dropped (set_client_budget or reset called mid-attempt)"
275                );
276            }
277        }
278
279        *open = OpenAttempt {
280            usage: Some(usage),
281            ..OpenAttempt::default()
282        };
283    }
284
285    /// Freezes the duration of the attempt at the current elapsed time, so that
286    /// what happens next (retry backoff, error handling) is not counted as part
287    /// of it. Later calls do not move it.
288    pub(crate) fn mark_attempt_end(&self) {
289        let mut open = self.open();
290        if open.duration.is_none() {
291            open.duration = open.usage.as_ref().map(|usage| usage.elapsed());
292        }
293    }
294
295    /// Records the time spent in `commit`, whatever its result, and ends the
296    /// attempt's work.
297    pub(crate) fn record_commit(&self, duration: Duration) {
298        let mut open = self.open();
299        open.commit_duration = Some(duration);
300        if open.duration.is_none() {
301            open.duration = open.usage.as_ref().map(|usage| usage.elapsed());
302        }
303    }
304
305    /// Records a GRV completion only while its issuing attempt is still open.
306    /// The first timing of that attempt is kept, even when the GRV failed.
307    pub(crate) fn record_grv(
308        &self,
309        usage: &Arc<AttemptUsage>,
310        duration: Duration,
311        version: Option<i64>,
312    ) {
313        let mut open = self.open();
314        if !open
315            .usage
316            .as_ref()
317            .is_some_and(|current| Arc::ptr_eq(current, usage))
318        {
319            return;
320        }
321        if open.grv_duration.is_none() {
322            open.grv_duration = Some(duration);
323        }
324        if let Some(version) = version {
325            open.read_version = Some(version);
326            self.report().transaction.read_version = Some(version);
327        }
328    }
329
330    /// Ends the current attempt and pushes it to the report.
331    ///
332    /// Does nothing when no attempt is being recorded, which makes it safe to
333    /// call on the exit paths of a runner that may already have ended it.
334    pub fn finish_attempt(&self, outcome: AttemptOutcome) {
335        let (usage, open) = {
336            let mut open = self.open();
337            match open.usage.take() {
338                // `usage` was taken above, so this leaves the scratch empty.
339                Some(usage) => (usage, std::mem::take(&mut *open)),
340                None => return,
341            }
342        };
343
344        let retried = matches!(outcome, AttemptOutcome::Retried { .. });
345
346        let mut report = self.report();
347        let index = report.attempts.len();
348        report.attempts.push(AttemptMetrics {
349            index,
350            usage: usage.snapshot(),
351            custom_metrics: usage.custom_metrics(),
352            duration: open.duration.or_else(|| Some(usage.elapsed())),
353            commit_duration: open.commit_duration,
354            grv_duration: open.grv_duration,
355            on_error_duration: None,
356            outcome,
357            conflicting_keys: open.conflicting_keys,
358            read_version: open.read_version,
359        });
360
361        if retried {
362            report.transaction.retries += 1;
363        }
364    }
365
366    /// Records the time spent in `on_error` for the attempt that just ended.
367    ///
368    /// The runner measures it after `on_error` returned, hence after the
369    /// attempt was pushed: it lands on the last attempt of the report.
370    pub fn set_on_error_duration(&self, duration: Duration) {
371        if let Some(attempt) = self.report().attempts.last_mut() {
372            attempt.on_error_duration = Some(duration);
373        }
374    }
375
376    /// Records the total duration of the run.
377    pub fn set_total_duration(&self, duration: Duration) {
378        self.report().total_duration = Some(duration);
379    }
380
381    /// Records the read version of the current attempt.
382    pub fn set_read_version(&self, version: i64) {
383        self.open().read_version = Some(version);
384        self.report().transaction.read_version = Some(version);
385    }
386
387    /// Records the commit version of the transaction.
388    pub fn set_commit_version(&self, version: i64) {
389        self.report().transaction.commit_version = Some(version);
390    }
391
392    /// Records the conflicting key ranges of the current attempt.
393    pub fn set_conflicting_keys(&self, keys: ConflictKeys) {
394        self.open().conflicting_keys = keys;
395    }
396
397    /// Increment the conflict counter
398    pub fn increment_conflict_count(&self) {
399        self.report().transaction.conflict_count += 1;
400    }
401
402    /// Get the number of retries
403    pub fn get_retries(&self) -> u64 {
404        self.report().transaction.retries
405    }
406
407    /// Returns a clone of the report built so far.
408    pub fn get_metrics_data(&self) -> MetricsReport {
409        self.report().clone()
410    }
411
412    /// Returns a clone of the transaction information.
413    pub fn get_transaction_info(&self) -> TransactionInfo {
414        self.report().transaction.clone()
415    }
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421    use std::collections::HashSet;
422
423    #[test]
424    fn test_metric_key_equality() {
425        // Test that different combinations of the same labels are considered equal
426        let key1 = MetricKey::new("counter", &[("region", "us-west"), ("service", "api")]);
427        let key2 = MetricKey::new("counter", &[("service", "api"), ("region", "us-west")]);
428
429        // Same labels in different order should be considered equal
430        assert_eq!(key1, key2);
431
432        // Different label values should produce different keys
433        let key3 = MetricKey::new("counter", &[("region", "us-east"), ("service", "api")]);
434        assert_ne!(key1, key3);
435
436        // Different label keys should produce different keys
437        let key4 = MetricKey::new("counter", &[("zone", "us-west"), ("service", "api")]);
438        assert_ne!(key1, key4);
439
440        // Different metric names should produce different keys
441        let key5 = MetricKey::new("timer", &[("region", "us-west"), ("service", "api")]);
442        assert_ne!(key1, key5);
443    }
444
445    #[test]
446    fn test_metric_key_in_hashmap() {
447        let mut metrics = HashMap::new();
448
449        // Insert metrics with different label combinations
450        metrics.insert(
451            MetricKey::new("counter", &[("region", "us-west"), ("service", "api")]),
452            100,
453        );
454        metrics.insert(
455            MetricKey::new("counter", &[("region", "us-east"), ("service", "api")]),
456            200,
457        );
458        metrics.insert(
459            MetricKey::new("timer", &[("region", "us-west"), ("service", "api")]),
460            300,
461        );
462
463        // Verify we can retrieve metrics with the same label combinations
464        assert_eq!(
465            metrics.get(&MetricKey::new(
466                "counter",
467                &[("region", "us-west"), ("service", "api")]
468            )),
469            Some(&100)
470        );
471
472        // Verify we can retrieve metrics with labels in different order
473        assert_eq!(
474            metrics.get(&MetricKey::new(
475                "counter",
476                &[("service", "api"), ("region", "us-west")]
477            )),
478            Some(&100)
479        );
480
481        // Verify different label values produce different keys
482        assert_eq!(
483            metrics.get(&MetricKey::new(
484                "counter",
485                &[("region", "us-east"), ("service", "api")]
486            )),
487            Some(&200)
488        );
489
490        // Verify different metric names produce different keys
491        assert_eq!(
492            metrics.get(&MetricKey::new(
493                "timer",
494                &[("region", "us-west"), ("service", "api")]
495            )),
496            Some(&300)
497        );
498    }
499
500    #[test]
501    fn test_metric_key_label_order_independence() {
502        // Create a HashSet to verify uniqueness
503        let mut unique_keys = HashSet::new();
504
505        // Add keys with the same labels in different orders
506        unique_keys.insert(MetricKey::new(
507            "counter",
508            &[("a", "1"), ("b", "2"), ("c", "3")],
509        ));
510        unique_keys.insert(MetricKey::new(
511            "counter",
512            &[("a", "1"), ("c", "3"), ("b", "2")],
513        ));
514        unique_keys.insert(MetricKey::new(
515            "counter",
516            &[("b", "2"), ("a", "1"), ("c", "3")],
517        ));
518        unique_keys.insert(MetricKey::new(
519            "counter",
520            &[("b", "2"), ("c", "3"), ("a", "1")],
521        ));
522        unique_keys.insert(MetricKey::new(
523            "counter",
524            &[("c", "3"), ("a", "1"), ("b", "2")],
525        ));
526        unique_keys.insert(MetricKey::new(
527            "counter",
528            &[("c", "3"), ("b", "2"), ("a", "1")],
529        ));
530
531        // All permutations should be considered the same key
532        assert_eq!(unique_keys.len(), 1);
533    }
534
535    /// Every attempt keeps its own counters and custom metrics, and the report
536    /// aggregates them.
537    #[test]
538    fn attempts_are_recorded_one_by_one() {
539        let metrics = TransactionMetrics::new();
540        let key = MetricKey::new("documents", &[]);
541
542        // Nothing to push before an attempt is opened.
543        metrics.finish_attempt(AttemptOutcome::Failed);
544        assert!(metrics.get_metrics_data().attempts.is_empty());
545
546        let first = Arc::new(AttemptUsage::new());
547        metrics.begin_attempt(first.clone());
548        first.record_set(10);
549        first.increment_custom(key.clone(), 1);
550        metrics.record_commit(Duration::from_millis(3));
551        metrics.record_grv(&first, Duration::from_millis(5), None);
552        metrics.record_grv(&first, Duration::from_millis(1), Some(42)); // first sample wins
553        metrics.finish_attempt(AttemptOutcome::Retried {
554            cause: FdbError::from_code(1020),
555        });
556        metrics.set_on_error_duration(Duration::from_millis(7));
557
558        let second = Arc::new(AttemptUsage::new());
559        metrics.begin_attempt(second.clone());
560        metrics.record_grv(&first, Duration::from_millis(20), None);
561        second.record_set(4);
562        second.increment_custom(key.clone(), 2);
563        metrics.finish_attempt(AttemptOutcome::Committed);
564
565        // A second end is a no-op: an attempt is pushed exactly once.
566        metrics.finish_attempt(AttemptOutcome::Failed);
567
568        let report = metrics.get_metrics_data();
569        assert_eq!(report.attempts.len(), 2);
570        assert_eq!(report.transaction.retries, 1);
571        assert_eq!(report.transaction.read_version, Some(42));
572
573        let first = &report.attempts[0];
574        assert_eq!(first.index, 0);
575        assert_eq!(first.usage.bytes_written, 10);
576        assert_eq!(first.custom_metrics.get(&key), Some(&1));
577        assert_eq!(first.commit_duration, Some(Duration::from_millis(3)));
578        assert_eq!(first.grv_duration, Some(Duration::from_millis(5)));
579        assert_eq!(first.on_error_duration, Some(Duration::from_millis(7)));
580        assert!(matches!(
581            first.outcome,
582            AttemptOutcome::Retried { cause } if cause.code() == 1020
583        ));
584
585        let second = &report.attempts[1];
586        assert_eq!(second.index, 1);
587        assert_eq!(second.usage.bytes_written, 4);
588        assert_eq!(second.custom_metrics.get(&key), Some(&2));
589        assert!(second.commit_duration.is_none());
590        assert!(second.grv_duration.is_none());
591        assert!(second.read_version.is_none());
592        assert!(matches!(second.outcome, AttemptOutcome::Committed));
593
594        assert_eq!(report.total_usage().bytes_written, 14);
595        assert_eq!(report.total_usage().call_set, 2);
596    }
597
598    /// An attempt whose only activity was `set_read_version` (no counter
599    /// touched, no custom metric) must not be silently dropped: it should be
600    /// discarded the same way a busy attempt is when a new one starts before
601    /// `finish_attempt` was called, and the crate must keep working normally
602    /// afterwards. This covers the state side of the `idle` heuristic in
603    /// `begin_attempt`; the `#[cfg(feature = "trace")]` warning itself is not
604    /// asserted here as the crate has no test pattern capturing `tracing`
605    /// output content (see `tests/trace.rs`, which only checks the writer
606    /// doesn't panic).
607    #[test]
608    fn dropped_attempt_with_only_read_version_is_handled() {
609        let metrics = TransactionMetrics::new();
610
611        let first = Arc::new(AttemptUsage::new());
612        metrics.begin_attempt(first.clone());
613        metrics.set_read_version(42);
614
615        // A new attempt starts (e.g. `set_client_budget`/`reset`) without the
616        // first one ever reaching `finish_attempt`: it must be dropped
617        // cleanly, no panic.
618        let second = Arc::new(AttemptUsage::new());
619        metrics.begin_attempt(second.clone());
620        second.record_set(1);
621        metrics.finish_attempt(AttemptOutcome::Committed);
622
623        let report = metrics.get_metrics_data();
624        assert_eq!(report.attempts.len(), 1);
625        let only = &report.attempts[0];
626        assert_eq!(only.usage.bytes_written, 1);
627        assert!(matches!(only.outcome, AttemptOutcome::Committed));
628
629        // The read version set on the dropped attempt is still reflected at
630        // the transaction level.
631        assert_eq!(report.transaction.read_version, Some(42));
632    }
633}