foundationdb/
runner.rs

1// Copyright 2026 foundationdb-rs developers, https://github.com/Clikengo/foundationdb-rs/graphs/contributors
2//
3// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5// http://opensource.org/licenses/MIT>, at your option. This file may not be
6// copied, modified, or distributed except according to those terms.
7
8//! The retry runner behind [`Database::run`](crate::Database::run).
9//!
10//! The runner is built from three independent pieces:
11//!
12//! * the **attempt**: run the closure on a transaction, then commit it. One
13//!   attempt does exactly one pass and never retries.
14//! * the **hooks** ([`RunnerHooks`]): purely observational callbacks fired
15//!   around each attempt. They cannot change what the runner decides.
16//! * the **policy** ([`RetryPolicy`]): the only thing that can change a
17//!   decision. It sees the failure and the decision the runner proposes, and
18//!   returns the decision to apply.
19//!
20//! They are assembled by [`TransactionRunner`], the builder returned by
21//! [`Database::runner`](crate::Database::runner):
22//!
23//! ```no_run
24//! # use foundationdb::*;
25//! # use foundationdb::runner::MetricsHooks;
26//! # async fn example(db: &Database) -> Result<(), FdbBindingError> {
27//! let metrics = TransactionMetrics::new();
28//! db.runner()
29//!     .hooks(&MetricsHooks::new(&metrics))
30//!     .run(|trx, _maybe_committed| async move {
31//!         trx.set(b"key", b"value");
32//!         Ok::<_, FdbBindingError>(())
33//!     })
34//!     .await?;
35//! let report = metrics.get_metrics_data();
36//! # let _ = report;
37//! # Ok(())
38//! # }
39//! ```
40//!
41//! # Retry semantics
42//!
43//! `fdb_transaction_on_error` remains the single retry governor: backoff, max
44//! retry delay and [`TransactionOption::RetryLimit`](crate::options::TransactionOption::RetryLimit)
45//! are applied by the C API, never by a Rust-side budget. A closure error asking
46//! for a retry without a native error underneath
47//! ([`RetryDecision::Retry`]) is routed through `on_error` with code 1020
48//! (`not_committed`) so that it obeys the same budget.
49
50use std::time::{Duration, Instant};
51
52use crate::database::{Database, MaybeCommitted};
53use crate::error::{FdbBindingError, FdbError, FdbResult, RetryDecision, RetryableError};
54use crate::metrics::{AttemptOutcome, ConflictKeys, TransactionMetrics};
55use crate::transaction::{
56    RetryableTransaction, Transaction, TransactionCommitError, TransactionCommitted,
57};
58
59/// `not_committed`: the retryable error code a [`RetryDecision::Retry`] is
60/// routed through.
61const NOT_COMMITTED: i32 = 1020;
62
63/// Observation points of the retry runner.
64///
65/// Hooks are **purely observational**: nothing they do or return changes what
66/// the runner decides. Use a [`RetryPolicy`] to influence retries. Every method
67/// has a no-op default, so an implementation only overrides what it needs.
68///
69/// # Ordering
70///
71/// Every callback receives the index of the attempt it belongs to, starting at
72/// `0` and matching [`AttemptMetrics::index`](crate::metrics::AttemptMetrics::index).
73/// Per attempt, in order:
74///
75/// 1. [`on_attempt_start`](Self::on_attempt_start)
76/// 2. the closure runs
77/// 3. if the closure succeeded: [`before_commit`](Self::before_commit), then
78///    the commit
79///    * commit succeeded: [`on_commit_success`](Self::on_commit_success), the
80///      run is over
81///    * commit failed: [`on_commit_error`](Self::on_commit_error), then
82///      `on_error`, then [`on_error_duration`](Self::on_error_duration) and
83///      [`on_retry`](Self::on_retry) if the transaction was retried
84/// 4. if the closure failed: [`on_closure_error`](Self::on_closure_error) (only
85///    when the error maps to an [`FdbError`], that is when the runner is about
86///    to call `on_error`), then `on_error`, then
87///    [`on_error_duration`](Self::on_error_duration) and
88///    [`on_retry`](Self::on_retry) if the transaction was retried
89///
90/// [`on_complete`](Self::on_complete) fires exactly once per run, on **every**
91/// exit path: success, non-retryable error, exhausted retries or binding error.
92///
93/// The two fallible hooks ([`before_commit`](Self::before_commit) and
94/// [`on_commit_error`](Self::on_commit_error)) never abort the runner: their
95/// error is reported to [`on_hook_error`](Self::on_hook_error) and the run
96/// continues exactly as if they had succeeded.
97///
98/// # Transaction access
99///
100/// Hooks see a `&Transaction`, never the clonable
101/// [`RetryableTransaction`]: a hook that could
102/// keep a reference to it would make the runner fail with
103/// [`FdbBindingError::ReferenceToTransactionKept`].
104///
105/// # Composition
106///
107/// [`RunnerHooks`] is implemented for `()` (no-op), `&H`, `Option<H>` (`None`
108/// is a no-op) and `(A, B)`, which nest for any arity: `(a, (b, c))`. In a
109/// tuple, callbacks fire left to right, and both sides always run even if the
110/// left one returned an error, the first error being the one reported.
111pub trait RunnerHooks {
112    /// Called when an attempt starts, before invoking the closure to construct
113    /// its future.
114    fn on_attempt_start(&self, _trx: &Transaction, _attempt: usize) {}
115
116    /// Called after the closure succeeded, before the transaction is committed.
117    ///
118    /// This is the last point where the transaction can be read or written
119    /// inside the attempt. Returning an error does **not** abort the commit: it
120    /// is reported through [`on_hook_error`](Self::on_hook_error) and the
121    /// transaction is committed anyway.
122    fn before_commit(
123        &self,
124        _trx: &Transaction,
125        _attempt: usize,
126    ) -> impl Future<Output = FdbResult<()>> + Send {
127        async { Ok(()) }
128    }
129
130    /// Called with the error of a hook that failed, right after it failed.
131    fn on_hook_error(&self, _err: &FdbError, _attempt: usize) {}
132
133    /// Called after a successful commit, with the time the commit took.
134    fn on_commit_success(
135        &self,
136        _committed: &TransactionCommitted,
137        _commit_duration_ms: u64,
138        _attempt: usize,
139    ) {
140    }
141
142    /// Called when the commit failed, **before** `on_error` resets the
143    /// transaction. This is the only window to read conflicting keys or inspect
144    /// the state of the failed attempt.
145    ///
146    /// It fires whatever the runner does next, including when the
147    /// [`RetryPolicy`] turns the failure into a fatal one. An error returned
148    /// here is reported through [`on_hook_error`](Self::on_hook_error) and
149    /// changes nothing else.
150    fn on_commit_error(
151        &self,
152        _err: &TransactionCommitError,
153        _attempt: usize,
154    ) -> impl Future<Output = FdbResult<()>> + Send {
155        async { Ok(()) }
156    }
157
158    /// Called when a closure error is about to be handed to `on_error`.
159    ///
160    /// A closure error classified as [`RetryDecision::Retry`] surfaces here as
161    /// a synthetic `FdbError` with code 1020 (`not_committed`). A fatal error
162    /// never reaches this hook.
163    fn on_closure_error(&self, _err: &FdbError, _attempt: usize) {}
164
165    /// Called after `on_error` returned, with the time it took, backoff
166    /// included.
167    fn on_error_duration(&self, _duration_ms: u64, _attempt: usize) {}
168
169    /// Called when the attempt is about to be retried, that is once `on_error`
170    /// accepted to retry it.
171    fn on_retry(&self, _attempt: usize) {}
172
173    /// Called exactly once when the run ends, whatever its outcome.
174    fn on_complete(&self) {}
175}
176
177impl RunnerHooks for () {}
178
179impl<H: RunnerHooks + Sync> RunnerHooks for &H {
180    fn on_attempt_start(&self, trx: &Transaction, attempt: usize) {
181        (*self).on_attempt_start(trx, attempt)
182    }
183
184    fn before_commit(
185        &self,
186        trx: &Transaction,
187        attempt: usize,
188    ) -> impl Future<Output = FdbResult<()>> + Send {
189        (*self).before_commit(trx, attempt)
190    }
191
192    fn on_hook_error(&self, err: &FdbError, attempt: usize) {
193        (*self).on_hook_error(err, attempt)
194    }
195
196    fn on_commit_success(
197        &self,
198        committed: &TransactionCommitted,
199        commit_duration_ms: u64,
200        attempt: usize,
201    ) {
202        (*self).on_commit_success(committed, commit_duration_ms, attempt)
203    }
204
205    fn on_commit_error(
206        &self,
207        err: &TransactionCommitError,
208        attempt: usize,
209    ) -> impl Future<Output = FdbResult<()>> + Send {
210        (*self).on_commit_error(err, attempt)
211    }
212
213    fn on_closure_error(&self, err: &FdbError, attempt: usize) {
214        (*self).on_closure_error(err, attempt)
215    }
216
217    fn on_error_duration(&self, duration_ms: u64, attempt: usize) {
218        (*self).on_error_duration(duration_ms, attempt)
219    }
220
221    fn on_retry(&self, attempt: usize) {
222        (*self).on_retry(attempt)
223    }
224
225    fn on_complete(&self) {
226        (*self).on_complete()
227    }
228}
229
230impl<H: RunnerHooks + Sync> RunnerHooks for Option<H> {
231    fn on_attempt_start(&self, trx: &Transaction, attempt: usize) {
232        if let Some(hooks) = self {
233            hooks.on_attempt_start(trx, attempt);
234        }
235    }
236
237    async fn before_commit(&self, trx: &Transaction, attempt: usize) -> FdbResult<()> {
238        match self {
239            Some(hooks) => hooks.before_commit(trx, attempt).await,
240            None => Ok(()),
241        }
242    }
243
244    fn on_hook_error(&self, err: &FdbError, attempt: usize) {
245        if let Some(hooks) = self {
246            hooks.on_hook_error(err, attempt);
247        }
248    }
249
250    fn on_commit_success(
251        &self,
252        committed: &TransactionCommitted,
253        commit_duration_ms: u64,
254        attempt: usize,
255    ) {
256        if let Some(hooks) = self {
257            hooks.on_commit_success(committed, commit_duration_ms, attempt);
258        }
259    }
260
261    async fn on_commit_error(&self, err: &TransactionCommitError, attempt: usize) -> FdbResult<()> {
262        match self {
263            Some(hooks) => hooks.on_commit_error(err, attempt).await,
264            None => Ok(()),
265        }
266    }
267
268    fn on_closure_error(&self, err: &FdbError, attempt: usize) {
269        if let Some(hooks) = self {
270            hooks.on_closure_error(err, attempt);
271        }
272    }
273
274    fn on_error_duration(&self, duration_ms: u64, attempt: usize) {
275        if let Some(hooks) = self {
276            hooks.on_error_duration(duration_ms, attempt);
277        }
278    }
279
280    fn on_retry(&self, attempt: usize) {
281        if let Some(hooks) = self {
282            hooks.on_retry(attempt);
283        }
284    }
285
286    fn on_complete(&self) {
287        if let Some(hooks) = self {
288            hooks.on_complete();
289        }
290    }
291}
292
293impl<A: RunnerHooks + Sync, B: RunnerHooks + Sync> RunnerHooks for (A, B) {
294    fn on_attempt_start(&self, trx: &Transaction, attempt: usize) {
295        self.0.on_attempt_start(trx, attempt);
296        self.1.on_attempt_start(trx, attempt);
297    }
298
299    async fn before_commit(&self, trx: &Transaction, attempt: usize) -> FdbResult<()> {
300        let first = self.0.before_commit(trx, attempt).await;
301        let second = self.1.before_commit(trx, attempt).await;
302        first.and(second)
303    }
304
305    fn on_hook_error(&self, err: &FdbError, attempt: usize) {
306        self.0.on_hook_error(err, attempt);
307        self.1.on_hook_error(err, attempt);
308    }
309
310    fn on_commit_success(
311        &self,
312        committed: &TransactionCommitted,
313        commit_duration_ms: u64,
314        attempt: usize,
315    ) {
316        self.0
317            .on_commit_success(committed, commit_duration_ms, attempt);
318        self.1
319            .on_commit_success(committed, commit_duration_ms, attempt);
320    }
321
322    async fn on_commit_error(&self, err: &TransactionCommitError, attempt: usize) -> FdbResult<()> {
323        let first = self.0.on_commit_error(err, attempt).await;
324        let second = self.1.on_commit_error(err, attempt).await;
325        first.and(second)
326    }
327
328    fn on_closure_error(&self, err: &FdbError, attempt: usize) {
329        self.0.on_closure_error(err, attempt);
330        self.1.on_closure_error(err, attempt);
331    }
332
333    fn on_error_duration(&self, duration_ms: u64, attempt: usize) {
334        self.0.on_error_duration(duration_ms, attempt);
335        self.1.on_error_duration(duration_ms, attempt);
336    }
337
338    fn on_retry(&self, attempt: usize) {
339        self.0.on_retry(attempt);
340        self.1.on_retry(attempt);
341    }
342
343    fn on_complete(&self) {
344        self.0.on_complete();
345        self.1.on_complete();
346    }
347}
348
349/// Hooks collecting a [`MetricsReport`](crate::metrics::MetricsReport) into a
350/// [`TransactionMetrics`].
351///
352/// They are what [`Database::instrumented_run`](crate::Database::instrumented_run)
353/// is made of, and can be stacked on any other hooks to get the same report out
354/// of a plain [`run`](crate::Database::run):
355///
356/// ```no_run
357/// # use foundationdb::*;
358/// # use foundationdb::runner::MetricsHooks;
359/// # struct MyHooks;
360/// # impl foundationdb::runner::RunnerHooks for MyHooks {}
361/// # async fn example(db: &Database) -> Result<(), FdbBindingError> {
362/// let metrics = TransactionMetrics::new();
363/// db.run_with_hooks(&(MetricsHooks::new(&metrics), MyHooks), |trx, _| async move {
364///     trx.set(b"key", b"value");
365///     Ok::<_, FdbBindingError>(())
366/// })
367/// .await?;
368///
369/// let report = metrics.get_metrics_data();
370/// # let _ = report;
371/// # Ok(())
372/// # }
373/// ```
374///
375/// The run is timed from the construction of the hooks, so build one per run.
376///
377/// The collector is attached to the transaction when the first attempt starts,
378/// which is what makes the counters of a plain `run` land in the report. A
379/// transaction only ever reports to one collector: stacking two `MetricsHooks`
380/// leaves the second one empty.
381pub struct MetricsHooks {
382    metrics: TransactionMetrics,
383    start: Instant,
384}
385
386impl MetricsHooks {
387    /// Collects the metrics of a run into `metrics`.
388    ///
389    /// The handle is cloned, so the caller keeps reading the report from its
390    /// own [`TransactionMetrics`] once the run is over.
391    pub fn new(metrics: &TransactionMetrics) -> Self {
392        Self {
393            metrics: metrics.clone(),
394            start: Instant::now(),
395        }
396    }
397}
398
399impl RunnerHooks for MetricsHooks {
400    fn on_attempt_start(&self, trx: &Transaction, _attempt: usize) {
401        // Wires the collector onto the transaction the runner created, which is
402        // what makes the counters of every attempt land in the report. A no-op
403        // from the second attempt on: the transaction keeps its collector
404        // across retries.
405        trx.attach_metrics(&self.metrics);
406    }
407
408    async fn on_commit_error(
409        &self,
410        err: &TransactionCommitError,
411        _attempt: usize,
412    ) -> FdbResult<()> {
413        // not_committed (1020) = commit conflict
414        if err.code() == NOT_COMMITTED {
415            self.metrics.increment_conflict_count();
416        }
417        // Reading from the \xff\xff/transaction/conflicting_keys/ special keyspace is
418        // resolved client-side — no network round-trip to the cluster. The future still
419        // goes through the FDB network thread event loop, but the data comes from an
420        // in-memory map populated during the commit response. Returns empty if
421        // ReportConflictingKeys was not set.
422        match err.conflicting_keys().await {
423            Ok(keys) => {
424                self.metrics
425                    .set_conflicting_keys(ConflictKeys::Available(keys));
426                Ok(())
427            }
428            Err(read_error) => {
429                self.metrics
430                    .set_conflicting_keys(ConflictKeys::ReadFailed(read_error));
431                Err(read_error)
432            }
433        }
434    }
435
436    fn on_error_duration(&self, duration_ms: u64, _attempt: usize) {
437        // `on_error` already pushed the attempt it ended, so this lands on the
438        // last attempt of the report.
439        self.metrics
440            .set_on_error_duration(Duration::from_millis(duration_ms));
441    }
442
443    fn on_commit_success(
444        &self,
445        committed: &TransactionCommitted,
446        _commit_duration_ms: u64,
447        _attempt: usize,
448    ) {
449        // The attempt itself was pushed by `Transaction::commit`, which also
450        // recorded how long the commit took.
451        if let Ok(version) = committed.committed_version() {
452            self.metrics.set_commit_version(version);
453        }
454    }
455
456    fn on_complete(&self) {
457        // An attempt still open when the run ends is the one that failed:
458        // attempts that committed or were retried were pushed by the
459        // transaction itself, which makes this a no-op for them.
460        self.metrics.finish_attempt(AttemptOutcome::Failed);
461        self.metrics.set_total_duration(self.start.elapsed());
462    }
463}
464
465/// The failure a [`RetryPolicy`] is asked about.
466#[derive(Debug)]
467#[non_exhaustive]
468pub enum AttemptFailure<'a, E> {
469    /// The closure returned an error, the transaction was not committed.
470    Closure(&'a E),
471    /// The closure succeeded but the commit failed.
472    Commit(&'a TransactionCommitError),
473}
474
475/// Decides what the runner does with a failed attempt.
476///
477/// The runner classifies the failure on its own and passes its verdict as
478/// `proposed`; the policy returns the decision to apply, which is `proposed`
479/// itself unless it wants to override it:
480///
481/// * [`RetryDecision::Fatal`] ends the run and returns the original error, the
482///   one the closure produced or the commit error, never something the policy
483///   made up.
484/// * [`RetryDecision::Fdb`] hands that error to `fdb_transaction_on_error`,
485///   which judges retryability and applies the backoff.
486/// * [`RetryDecision::Retry`] does the same with code 1020 (`not_committed`),
487///   which is retryable by definition.
488///
489/// A policy cannot clear the [`MaybeCommitted`] flag handed to the closure:
490/// it accumulates the original errors from all attempts before the policy is
491/// consulted.
492///
493/// # Example: cap the number of attempts
494///
495/// ```
496/// use foundationdb::runner::{AttemptFailure, RetryPolicy};
497/// use foundationdb::{FdbBindingError, RetryDecision};
498///
499/// struct MaxAttempts(usize);
500///
501/// impl RetryPolicy<FdbBindingError> for MaxAttempts {
502///     fn decide(
503///         &self,
504///         _failure: AttemptFailure<'_, FdbBindingError>,
505///         proposed: RetryDecision,
506///         attempt: usize,
507///     ) -> RetryDecision {
508///         if attempt + 1 >= self.0 {
509///             RetryDecision::Fatal
510///         } else {
511///             proposed
512///         }
513///     }
514/// }
515/// ```
516pub trait RetryPolicy<E> {
517    /// Returns the decision to apply for `failure`, `proposed` being what the
518    /// runner would do on its own.
519    fn decide(
520        &self,
521        failure: AttemptFailure<'_, E>,
522        proposed: RetryDecision,
523        attempt: usize,
524    ) -> RetryDecision;
525}
526
527/// The default policy: it always applies what the runner proposes, leaving
528/// `fdb_transaction_on_error` the last word on retries.
529#[derive(Debug, Clone, Copy, Default)]
530pub struct NativeRetryPolicy;
531
532impl<E> RetryPolicy<E> for NativeRetryPolicy {
533    fn decide(
534        &self,
535        _failure: AttemptFailure<'_, E>,
536        proposed: RetryDecision,
537        _attempt: usize,
538    ) -> RetryDecision {
539        proposed
540    }
541}
542
543/// How a single attempt ended, carrying whatever still owns the transaction.
544enum Attempt<T, E> {
545    /// The closure succeeded and the transaction committed.
546    Committed {
547        committed: TransactionCommitted,
548        value: T,
549        commit_duration_ms: u64,
550    },
551    /// The closure failed, the transaction is untouched and can be retried.
552    ClosureFailed {
553        transaction: RetryableTransaction,
554        error: E,
555    },
556    /// The commit failed, the transaction is owned by the error.
557    CommitFailed(TransactionCommitError),
558    /// The runner could not get the transaction back from the closure.
559    BindingFailed(FdbBindingError),
560}
561
562/// Awaits the closure future and commits, without any retry logic.
563///
564/// The closure future is built by the caller so that the runner keeps owning
565/// the closure itself: an attempt that borrowed it would make every `run`
566/// future require `F: Sync`.
567async fn run_attempt<Fut, T, E, H>(
568    transaction: RetryableTransaction,
569    hooks: &H,
570    closure_result: Fut,
571    attempt: usize,
572) -> Attempt<T, E>
573where
574    Fut: Future<Output = Result<T, E>>,
575    H: RunnerHooks,
576{
577    let value = match closure_result.await {
578        Ok(value) => value,
579        Err(error) => return Attempt::ClosureFailed { transaction, error },
580    };
581
582    // Observational: a failing hook is reported, the commit happens anyway.
583    if let Err(hook_error) = hooks.before_commit(&transaction, attempt).await {
584        hooks.on_hook_error(&hook_error, attempt);
585    }
586
587    let now_commit = Instant::now();
588    match transaction.commit().await {
589        Ok(Ok(committed)) => Attempt::Committed {
590            committed,
591            value,
592            commit_duration_ms: now_commit.elapsed().as_millis() as u64,
593        },
594        Ok(Err(commit_error)) => Attempt::CommitFailed(commit_error),
595        Err(binding_error) => Attempt::BindingFailed(binding_error),
596    }
597}
598
599/// The retry loop: classification, policy, `on_error` and backoff on top of
600/// [`run_attempt`].
601///
602/// Wrapped by [`run_with_hooks`], which owns firing
603/// [`RunnerHooks::on_complete`] on every exit path.
604async fn retry_loop<F, Fut, T, E, H, P>(
605    initial_transaction: RetryableTransaction,
606    hooks: &H,
607    policy: &P,
608    closure: F,
609) -> Result<T, E>
610where
611    F: Fn(RetryableTransaction, MaybeCommitted) -> Fut,
612    Fut: Future<Output = Result<T, E>>,
613    E: RetryableError,
614    H: RunnerHooks,
615    P: RetryPolicy<E>,
616{
617    let mut maybe_committed = false;
618    let mut transaction = initial_transaction;
619    let mut attempt: usize = 0;
620
621    loop {
622        // The closure can issue operations while constructing its future.
623        hooks.on_attempt_start(&transaction, attempt);
624        let closure_result = closure(transaction.clone(), MaybeCommitted::new(maybe_committed));
625
626        let (retried_transaction, _fdb_err) =
627            match run_attempt(transaction, hooks, closure_result, attempt).await {
628                Attempt::Committed {
629                    committed,
630                    value,
631                    commit_duration_ms,
632                } => {
633                    hooks.on_commit_success(&committed, commit_duration_ms, attempt);
634
635                    #[cfg(feature = "trace")]
636                    tracing::info!(attempt, "success, returning result");
637
638                    return Ok(value);
639                }
640
641                Attempt::BindingFailed(binding_error) => {
642                    #[cfg(feature = "trace")]
643                    tracing::error!(attempt, "transaction reference kept, aborting transaction");
644
645                    return Err(E::from(binding_error));
646                }
647
648                Attempt::ClosureFailed { transaction, error } => {
649                    let proposed = error.retry_decision();
650                    // Later failures cannot resolve an earlier uncertain commit.
651                    // Preserve it regardless of the policy's replacement error.
652                    if let RetryDecision::Fdb(fdb_err) = proposed {
653                        maybe_committed |= fdb_err.is_maybe_committed();
654                    }
655
656                    let fdb_err =
657                        match policy.decide(AttemptFailure::Closure(&error), proposed, attempt) {
658                            RetryDecision::Fdb(fdb_err) => fdb_err,
659                            // Synthetic retryable code (1020, not_committed): backoff and
660                            // RetryLimit stay governed by fdb_transaction_on_error.
661                            RetryDecision::Retry => FdbError::from_code(NOT_COMMITTED),
662                            RetryDecision::Fatal => return Err(error),
663                        };
664                    hooks.on_closure_error(&fdb_err, attempt);
665
666                    let now_on_error = Instant::now();
667                    match transaction.on_error(fdb_err).await {
668                        Ok(Ok(transaction)) => {
669                            hooks.on_error_duration(
670                                now_on_error.elapsed().as_millis() as u64,
671                                attempt,
672                            );
673                            (transaction, fdb_err)
674                        }
675                        // Retries exhausted or non-retryable: return the original
676                        // closure error, which carries the caller's context.
677                        Ok(Err(_non_retryable)) => return Err(error),
678                        Err(binding_error) => return Err(E::from(binding_error)),
679                    }
680                }
681
682                Attempt::CommitFailed(commit_error) => {
683                    maybe_committed |= commit_error.is_maybe_committed();
684                    let proposed = RetryDecision::Fdb(*commit_error);
685
686                    // Fires before the policy is consulted: a hook reading the
687                    // conflicting keys must see them even when the run is about
688                    // to end.
689                    if let Err(hook_error) = hooks.on_commit_error(&commit_error, attempt).await {
690                        hooks.on_hook_error(&hook_error, attempt);
691                    }
692
693                    let fdb_err = match policy.decide(
694                        AttemptFailure::Commit(&commit_error),
695                        proposed,
696                        attempt,
697                    ) {
698                        RetryDecision::Fdb(fdb_err) => fdb_err,
699                        RetryDecision::Retry => FdbError::from_code(NOT_COMMITTED),
700                        RetryDecision::Fatal => {
701                            return Err(E::from(FdbBindingError::from(*commit_error)));
702                        }
703                    };
704
705                    // The transaction is taken out of the error so that the
706                    // error handed to `on_error` is the one the policy chose.
707                    let (transaction, _commit_err) = commit_error.into_parts();
708                    let now_on_error = Instant::now();
709                    match transaction.on_error(fdb_err).await {
710                        Ok(transaction) => {
711                            hooks.on_error_duration(
712                                now_on_error.elapsed().as_millis() as u64,
713                                attempt,
714                            );
715                            (RetryableTransaction::new(transaction), fdb_err)
716                        }
717                        Err(non_retryable) => {
718                            #[cfg(feature = "trace")]
719                            tracing::error!(
720                                attempt,
721                                error_code = non_retryable.code(),
722                                "could not commit, non retryable error"
723                            );
724
725                            return Err(E::from(FdbBindingError::from(non_retryable)));
726                        }
727                    }
728                }
729            };
730
731        #[cfg(feature = "trace")]
732        tracing::warn!(
733            attempt,
734            error_code = _fdb_err.code(),
735            "restarting transaction"
736        );
737
738        hooks.on_retry(attempt);
739        transaction = retried_transaction;
740        attempt += 1;
741    }
742}
743
744/// The retry runner: [`retry_loop`] plus the guarantee that
745/// [`RunnerHooks::on_complete`] fires exactly once.
746#[cfg_attr(
747    feature = "trace",
748    tracing::instrument(level = "debug", skip(initial_transaction, hooks, policy, closure))
749)]
750pub(crate) async fn run_with_hooks<F, Fut, T, E, H, P>(
751    initial_transaction: RetryableTransaction,
752    hooks: &H,
753    policy: &P,
754    closure: F,
755) -> Result<T, E>
756where
757    F: Fn(RetryableTransaction, MaybeCommitted) -> Fut,
758    Fut: Future<Output = Result<T, E>>,
759    E: RetryableError,
760    H: RunnerHooks,
761    P: RetryPolicy<E>,
762{
763    let result = retry_loop(initial_transaction, hooks, policy, closure).await;
764    hooks.on_complete();
765    result
766}
767
768/// Builder for a transactional run, returned by
769/// [`Database::runner`](crate::Database::runner).
770///
771/// It assembles the [`RunnerHooks`] and the [`RetryPolicy`] of a single call to
772/// [`run`](Self::run): no hooks and [`NativeRetryPolicy`] by default. Both are
773/// borrowed, so the same hooks and policy can be reused across runs (mind
774/// [`MetricsHooks`], which times the run from its own construction).
775///
776/// ```no_run
777/// # use foundationdb::*;
778/// # async fn example(db: &Database) -> Result<(), FdbBindingError> {
779/// db.runner()
780///     .run(|trx, _| async move {
781///         trx.set(b"key", b"value");
782///         Ok::<_, FdbBindingError>(())
783///     })
784///     .await
785/// # }
786/// ```
787pub struct TransactionRunner<'a, H = (), P = NativeRetryPolicy> {
788    db: &'a Database,
789    hooks: &'a H,
790    policy: &'a P,
791}
792
793impl<'a> TransactionRunner<'a, (), NativeRetryPolicy> {
794    /// A runner on `db` with no hooks and the native retry policy.
795    pub(crate) fn new(db: &'a Database) -> Self {
796        Self {
797            db,
798            hooks: &(),
799            policy: &NativeRetryPolicy,
800        }
801    }
802}
803
804impl<'a, H, P> TransactionRunner<'a, H, P> {
805    /// Observes the run with `hooks`, replacing any hooks set before.
806    ///
807    /// Stack several of them with a tuple: `.hooks(&(first, second))`.
808    pub fn hooks<H2>(self, hooks: &'a H2) -> TransactionRunner<'a, H2, P> {
809        TransactionRunner {
810            db: self.db,
811            hooks,
812            policy: self.policy,
813        }
814    }
815
816    /// Decides the retries with `policy` instead of [`NativeRetryPolicy`].
817    pub fn retry_policy<P2>(self, policy: &'a P2) -> TransactionRunner<'a, H, P2> {
818        TransactionRunner {
819            db: self.db,
820            hooks: self.hooks,
821            policy,
822        }
823    }
824
825    /// Runs `closure` in a transaction, retrying it until it commits or the
826    /// error is fatal.
827    ///
828    /// See [`Database::run`](crate::Database::run) for the semantics of the
829    /// closure, its error type and the retries.
830    #[cfg_attr(
831        feature = "trace",
832        tracing::instrument(level = "debug", skip(self, closure))
833    )]
834    pub async fn run<F, Fut, T, E>(self, closure: F) -> Result<T, E>
835    where
836        F: Fn(RetryableTransaction, MaybeCommitted) -> Fut,
837        Fut: Future<Output = Result<T, E>>,
838        E: RetryableError,
839        H: RunnerHooks,
840        P: RetryPolicy<E>,
841    {
842        let transaction = match self.db.create_retryable_trx() {
843            Ok(transaction) => transaction,
844            // The run is over before it started, but it did start: hooks that
845            // report on completion must still be called exactly once.
846            Err(err) => {
847                self.hooks.on_complete();
848                return Err(E::from(err));
849            }
850        };
851
852        run_with_hooks(transaction, self.hooks, self.policy, closure).await
853    }
854}