foundationdb/runner.rs
1// Copyright 2026 foundationdb-rs developers, https://github.com/Clikengo/foundationdb-rs/graphs/contributors
2//
3// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5// http://opensource.org/licenses/MIT>, at your option. This file may not be
6// copied, modified, or distributed except according to those terms.
7
8//! The retry runner behind [`Database::run`](crate::Database::run).
9//!
10//! The runner is built from three independent pieces:
11//!
12//! * the **attempt**: run the closure on a transaction, then commit it. One
13//! attempt does exactly one pass and never retries.
14//! * the **hooks** ([`RunnerHooks`]): purely observational callbacks fired
15//! around each attempt. They cannot change what the runner decides.
16//! * the **policy** ([`RetryPolicy`]): the only thing that can change a
17//! decision. It sees the failure and the decision the runner proposes, and
18//! returns the decision to apply.
19//!
20//! They are assembled by [`TransactionRunner`], the builder returned by
21//! [`Database::runner`](crate::Database::runner):
22//!
23//! ```no_run
24//! # use foundationdb::*;
25//! # use foundationdb::runner::MetricsHooks;
26//! # async fn example(db: &Database) -> Result<(), FdbBindingError> {
27//! let metrics = TransactionMetrics::new();
28//! db.runner()
29//! .hooks(&MetricsHooks::new(&metrics))
30//! .run(|trx, _maybe_committed| async move {
31//! trx.set(b"key", b"value");
32//! Ok::<_, FdbBindingError>(())
33//! })
34//! .await?;
35//! let report = metrics.get_metrics_data();
36//! # let _ = report;
37//! # Ok(())
38//! # }
39//! ```
40//!
41//! # Retry semantics
42//!
43//! `fdb_transaction_on_error` remains the single retry governor: backoff, max
44//! retry delay and [`TransactionOption::RetryLimit`](crate::options::TransactionOption::RetryLimit)
45//! are applied by the C API, never by a Rust-side budget. A closure error asking
46//! for a retry without a native error underneath
47//! ([`RetryDecision::Retry`]) is routed through `on_error` with code 1020
48//! (`not_committed`) so that it obeys the same budget.
49
50use std::time::{Duration, Instant};
51
52use crate::database::{Database, MaybeCommitted};
53use crate::error::{FdbBindingError, FdbError, FdbResult, RetryDecision, RetryableError};
54use crate::metrics::{AttemptOutcome, ConflictKeys, TransactionMetrics};
55use crate::transaction::{
56 RetryableTransaction, Transaction, TransactionCommitError, TransactionCommitted,
57};
58
59/// `not_committed`: the retryable error code a [`RetryDecision::Retry`] is
60/// routed through.
61const NOT_COMMITTED: i32 = 1020;
62
63/// Observation points of the retry runner.
64///
65/// Hooks are **purely observational**: nothing they do or return changes what
66/// the runner decides. Use a [`RetryPolicy`] to influence retries. Every method
67/// has a no-op default, so an implementation only overrides what it needs.
68///
69/// # Ordering
70///
71/// Every callback receives the index of the attempt it belongs to, starting at
72/// `0` and matching [`AttemptMetrics::index`](crate::metrics::AttemptMetrics::index).
73/// Per attempt, in order:
74///
75/// 1. [`on_attempt_start`](Self::on_attempt_start)
76/// 2. the closure runs
77/// 3. if the closure succeeded: [`before_commit`](Self::before_commit), then
78/// the commit
79/// * commit succeeded: [`on_commit_success`](Self::on_commit_success), the
80/// run is over
81/// * commit failed: [`on_commit_error`](Self::on_commit_error), then
82/// `on_error`, then [`on_error_duration`](Self::on_error_duration) and
83/// [`on_retry`](Self::on_retry) if the transaction was retried
84/// 4. if the closure failed: [`on_closure_error`](Self::on_closure_error) (only
85/// when the error maps to an [`FdbError`], that is when the runner is about
86/// to call `on_error`), then `on_error`, then
87/// [`on_error_duration`](Self::on_error_duration) and
88/// [`on_retry`](Self::on_retry) if the transaction was retried
89///
90/// [`on_complete`](Self::on_complete) fires exactly once per run, on **every**
91/// exit path: success, non-retryable error, exhausted retries or binding error.
92///
93/// The two fallible hooks ([`before_commit`](Self::before_commit) and
94/// [`on_commit_error`](Self::on_commit_error)) never abort the runner: their
95/// error is reported to [`on_hook_error`](Self::on_hook_error) and the run
96/// continues exactly as if they had succeeded.
97///
98/// # Transaction access
99///
100/// Hooks see a `&Transaction`, never the clonable
101/// [`RetryableTransaction`]: a hook that could
102/// keep a reference to it would make the runner fail with
103/// [`FdbBindingError::ReferenceToTransactionKept`].
104///
105/// # Composition
106///
107/// [`RunnerHooks`] is implemented for `()` (no-op), `&H`, `Option<H>` (`None`
108/// is a no-op) and `(A, B)`, which nest for any arity: `(a, (b, c))`. In a
109/// tuple, callbacks fire left to right, and both sides always run even if the
110/// left one returned an error, the first error being the one reported.
111pub trait RunnerHooks {
112 /// Called when an attempt starts, before invoking the closure to construct
113 /// its future.
114 fn on_attempt_start(&self, _trx: &Transaction, _attempt: usize) {}
115
116 /// Called after the closure succeeded, before the transaction is committed.
117 ///
118 /// This is the last point where the transaction can be read or written
119 /// inside the attempt. Returning an error does **not** abort the commit: it
120 /// is reported through [`on_hook_error`](Self::on_hook_error) and the
121 /// transaction is committed anyway.
122 fn before_commit(
123 &self,
124 _trx: &Transaction,
125 _attempt: usize,
126 ) -> impl Future<Output = FdbResult<()>> + Send {
127 async { Ok(()) }
128 }
129
130 /// Called with the error of a hook that failed, right after it failed.
131 fn on_hook_error(&self, _err: &FdbError, _attempt: usize) {}
132
133 /// Called after a successful commit, with the time the commit took.
134 fn on_commit_success(
135 &self,
136 _committed: &TransactionCommitted,
137 _commit_duration_ms: u64,
138 _attempt: usize,
139 ) {
140 }
141
142 /// Called when the commit failed, **before** `on_error` resets the
143 /// transaction. This is the only window to read conflicting keys or inspect
144 /// the state of the failed attempt.
145 ///
146 /// It fires whatever the runner does next, including when the
147 /// [`RetryPolicy`] turns the failure into a fatal one. An error returned
148 /// here is reported through [`on_hook_error`](Self::on_hook_error) and
149 /// changes nothing else.
150 fn on_commit_error(
151 &self,
152 _err: &TransactionCommitError,
153 _attempt: usize,
154 ) -> impl Future<Output = FdbResult<()>> + Send {
155 async { Ok(()) }
156 }
157
158 /// Called when a closure error is about to be handed to `on_error`.
159 ///
160 /// A closure error classified as [`RetryDecision::Retry`] surfaces here as
161 /// a synthetic `FdbError` with code 1020 (`not_committed`). A fatal error
162 /// never reaches this hook.
163 fn on_closure_error(&self, _err: &FdbError, _attempt: usize) {}
164
165 /// Called after `on_error` returned, with the time it took, backoff
166 /// included.
167 fn on_error_duration(&self, _duration_ms: u64, _attempt: usize) {}
168
169 /// Called when the attempt is about to be retried, that is once `on_error`
170 /// accepted to retry it.
171 fn on_retry(&self, _attempt: usize) {}
172
173 /// Called exactly once when the run ends, whatever its outcome.
174 fn on_complete(&self) {}
175}
176
177impl RunnerHooks for () {}
178
179impl<H: RunnerHooks + Sync> RunnerHooks for &H {
180 fn on_attempt_start(&self, trx: &Transaction, attempt: usize) {
181 (*self).on_attempt_start(trx, attempt)
182 }
183
184 fn before_commit(
185 &self,
186 trx: &Transaction,
187 attempt: usize,
188 ) -> impl Future<Output = FdbResult<()>> + Send {
189 (*self).before_commit(trx, attempt)
190 }
191
192 fn on_hook_error(&self, err: &FdbError, attempt: usize) {
193 (*self).on_hook_error(err, attempt)
194 }
195
196 fn on_commit_success(
197 &self,
198 committed: &TransactionCommitted,
199 commit_duration_ms: u64,
200 attempt: usize,
201 ) {
202 (*self).on_commit_success(committed, commit_duration_ms, attempt)
203 }
204
205 fn on_commit_error(
206 &self,
207 err: &TransactionCommitError,
208 attempt: usize,
209 ) -> impl Future<Output = FdbResult<()>> + Send {
210 (*self).on_commit_error(err, attempt)
211 }
212
213 fn on_closure_error(&self, err: &FdbError, attempt: usize) {
214 (*self).on_closure_error(err, attempt)
215 }
216
217 fn on_error_duration(&self, duration_ms: u64, attempt: usize) {
218 (*self).on_error_duration(duration_ms, attempt)
219 }
220
221 fn on_retry(&self, attempt: usize) {
222 (*self).on_retry(attempt)
223 }
224
225 fn on_complete(&self) {
226 (*self).on_complete()
227 }
228}
229
230impl<H: RunnerHooks + Sync> RunnerHooks for Option<H> {
231 fn on_attempt_start(&self, trx: &Transaction, attempt: usize) {
232 if let Some(hooks) = self {
233 hooks.on_attempt_start(trx, attempt);
234 }
235 }
236
237 async fn before_commit(&self, trx: &Transaction, attempt: usize) -> FdbResult<()> {
238 match self {
239 Some(hooks) => hooks.before_commit(trx, attempt).await,
240 None => Ok(()),
241 }
242 }
243
244 fn on_hook_error(&self, err: &FdbError, attempt: usize) {
245 if let Some(hooks) = self {
246 hooks.on_hook_error(err, attempt);
247 }
248 }
249
250 fn on_commit_success(
251 &self,
252 committed: &TransactionCommitted,
253 commit_duration_ms: u64,
254 attempt: usize,
255 ) {
256 if let Some(hooks) = self {
257 hooks.on_commit_success(committed, commit_duration_ms, attempt);
258 }
259 }
260
261 async fn on_commit_error(&self, err: &TransactionCommitError, attempt: usize) -> FdbResult<()> {
262 match self {
263 Some(hooks) => hooks.on_commit_error(err, attempt).await,
264 None => Ok(()),
265 }
266 }
267
268 fn on_closure_error(&self, err: &FdbError, attempt: usize) {
269 if let Some(hooks) = self {
270 hooks.on_closure_error(err, attempt);
271 }
272 }
273
274 fn on_error_duration(&self, duration_ms: u64, attempt: usize) {
275 if let Some(hooks) = self {
276 hooks.on_error_duration(duration_ms, attempt);
277 }
278 }
279
280 fn on_retry(&self, attempt: usize) {
281 if let Some(hooks) = self {
282 hooks.on_retry(attempt);
283 }
284 }
285
286 fn on_complete(&self) {
287 if let Some(hooks) = self {
288 hooks.on_complete();
289 }
290 }
291}
292
293impl<A: RunnerHooks + Sync, B: RunnerHooks + Sync> RunnerHooks for (A, B) {
294 fn on_attempt_start(&self, trx: &Transaction, attempt: usize) {
295 self.0.on_attempt_start(trx, attempt);
296 self.1.on_attempt_start(trx, attempt);
297 }
298
299 async fn before_commit(&self, trx: &Transaction, attempt: usize) -> FdbResult<()> {
300 let first = self.0.before_commit(trx, attempt).await;
301 let second = self.1.before_commit(trx, attempt).await;
302 first.and(second)
303 }
304
305 fn on_hook_error(&self, err: &FdbError, attempt: usize) {
306 self.0.on_hook_error(err, attempt);
307 self.1.on_hook_error(err, attempt);
308 }
309
310 fn on_commit_success(
311 &self,
312 committed: &TransactionCommitted,
313 commit_duration_ms: u64,
314 attempt: usize,
315 ) {
316 self.0
317 .on_commit_success(committed, commit_duration_ms, attempt);
318 self.1
319 .on_commit_success(committed, commit_duration_ms, attempt);
320 }
321
322 async fn on_commit_error(&self, err: &TransactionCommitError, attempt: usize) -> FdbResult<()> {
323 let first = self.0.on_commit_error(err, attempt).await;
324 let second = self.1.on_commit_error(err, attempt).await;
325 first.and(second)
326 }
327
328 fn on_closure_error(&self, err: &FdbError, attempt: usize) {
329 self.0.on_closure_error(err, attempt);
330 self.1.on_closure_error(err, attempt);
331 }
332
333 fn on_error_duration(&self, duration_ms: u64, attempt: usize) {
334 self.0.on_error_duration(duration_ms, attempt);
335 self.1.on_error_duration(duration_ms, attempt);
336 }
337
338 fn on_retry(&self, attempt: usize) {
339 self.0.on_retry(attempt);
340 self.1.on_retry(attempt);
341 }
342
343 fn on_complete(&self) {
344 self.0.on_complete();
345 self.1.on_complete();
346 }
347}
348
349/// Hooks collecting a [`MetricsReport`](crate::metrics::MetricsReport) into a
350/// [`TransactionMetrics`].
351///
352/// They are what [`Database::instrumented_run`](crate::Database::instrumented_run)
353/// is made of, and can be stacked on any other hooks to get the same report out
354/// of a plain [`run`](crate::Database::run):
355///
356/// ```no_run
357/// # use foundationdb::*;
358/// # use foundationdb::runner::MetricsHooks;
359/// # struct MyHooks;
360/// # impl foundationdb::runner::RunnerHooks for MyHooks {}
361/// # async fn example(db: &Database) -> Result<(), FdbBindingError> {
362/// let metrics = TransactionMetrics::new();
363/// db.run_with_hooks(&(MetricsHooks::new(&metrics), MyHooks), |trx, _| async move {
364/// trx.set(b"key", b"value");
365/// Ok::<_, FdbBindingError>(())
366/// })
367/// .await?;
368///
369/// let report = metrics.get_metrics_data();
370/// # let _ = report;
371/// # Ok(())
372/// # }
373/// ```
374///
375/// The run is timed from the construction of the hooks, so build one per run.
376///
377/// The collector is attached to the transaction when the first attempt starts,
378/// which is what makes the counters of a plain `run` land in the report. A
379/// transaction only ever reports to one collector: stacking two `MetricsHooks`
380/// leaves the second one empty.
381pub struct MetricsHooks {
382 metrics: TransactionMetrics,
383 start: Instant,
384}
385
386impl MetricsHooks {
387 /// Collects the metrics of a run into `metrics`.
388 ///
389 /// The handle is cloned, so the caller keeps reading the report from its
390 /// own [`TransactionMetrics`] once the run is over.
391 pub fn new(metrics: &TransactionMetrics) -> Self {
392 Self {
393 metrics: metrics.clone(),
394 start: Instant::now(),
395 }
396 }
397}
398
399impl RunnerHooks for MetricsHooks {
400 fn on_attempt_start(&self, trx: &Transaction, _attempt: usize) {
401 // Wires the collector onto the transaction the runner created, which is
402 // what makes the counters of every attempt land in the report. A no-op
403 // from the second attempt on: the transaction keeps its collector
404 // across retries.
405 trx.attach_metrics(&self.metrics);
406 }
407
408 async fn on_commit_error(
409 &self,
410 err: &TransactionCommitError,
411 _attempt: usize,
412 ) -> FdbResult<()> {
413 // not_committed (1020) = commit conflict
414 if err.code() == NOT_COMMITTED {
415 self.metrics.increment_conflict_count();
416 }
417 // Reading from the \xff\xff/transaction/conflicting_keys/ special keyspace is
418 // resolved client-side — no network round-trip to the cluster. The future still
419 // goes through the FDB network thread event loop, but the data comes from an
420 // in-memory map populated during the commit response. Returns empty if
421 // ReportConflictingKeys was not set.
422 match err.conflicting_keys().await {
423 Ok(keys) => {
424 self.metrics
425 .set_conflicting_keys(ConflictKeys::Available(keys));
426 Ok(())
427 }
428 Err(read_error) => {
429 self.metrics
430 .set_conflicting_keys(ConflictKeys::ReadFailed(read_error));
431 Err(read_error)
432 }
433 }
434 }
435
436 fn on_error_duration(&self, duration_ms: u64, _attempt: usize) {
437 // `on_error` already pushed the attempt it ended, so this lands on the
438 // last attempt of the report.
439 self.metrics
440 .set_on_error_duration(Duration::from_millis(duration_ms));
441 }
442
443 fn on_commit_success(
444 &self,
445 committed: &TransactionCommitted,
446 _commit_duration_ms: u64,
447 _attempt: usize,
448 ) {
449 // The attempt itself was pushed by `Transaction::commit`, which also
450 // recorded how long the commit took.
451 if let Ok(version) = committed.committed_version() {
452 self.metrics.set_commit_version(version);
453 }
454 }
455
456 fn on_complete(&self) {
457 // An attempt still open when the run ends is the one that failed:
458 // attempts that committed or were retried were pushed by the
459 // transaction itself, which makes this a no-op for them.
460 self.metrics.finish_attempt(AttemptOutcome::Failed);
461 self.metrics.set_total_duration(self.start.elapsed());
462 }
463}
464
465/// The failure a [`RetryPolicy`] is asked about.
466#[derive(Debug)]
467#[non_exhaustive]
468pub enum AttemptFailure<'a, E> {
469 /// The closure returned an error, the transaction was not committed.
470 Closure(&'a E),
471 /// The closure succeeded but the commit failed.
472 Commit(&'a TransactionCommitError),
473}
474
475/// Decides what the runner does with a failed attempt.
476///
477/// The runner classifies the failure on its own and passes its verdict as
478/// `proposed`; the policy returns the decision to apply, which is `proposed`
479/// itself unless it wants to override it:
480///
481/// * [`RetryDecision::Fatal`] ends the run and returns the original error, the
482/// one the closure produced or the commit error, never something the policy
483/// made up.
484/// * [`RetryDecision::Fdb`] hands that error to `fdb_transaction_on_error`,
485/// which judges retryability and applies the backoff.
486/// * [`RetryDecision::Retry`] does the same with code 1020 (`not_committed`),
487/// which is retryable by definition.
488///
489/// A policy cannot clear the [`MaybeCommitted`] flag handed to the closure:
490/// it accumulates the original errors from all attempts before the policy is
491/// consulted.
492///
493/// # Example: cap the number of attempts
494///
495/// ```
496/// use foundationdb::runner::{AttemptFailure, RetryPolicy};
497/// use foundationdb::{FdbBindingError, RetryDecision};
498///
499/// struct MaxAttempts(usize);
500///
501/// impl RetryPolicy<FdbBindingError> for MaxAttempts {
502/// fn decide(
503/// &self,
504/// _failure: AttemptFailure<'_, FdbBindingError>,
505/// proposed: RetryDecision,
506/// attempt: usize,
507/// ) -> RetryDecision {
508/// if attempt + 1 >= self.0 {
509/// RetryDecision::Fatal
510/// } else {
511/// proposed
512/// }
513/// }
514/// }
515/// ```
516pub trait RetryPolicy<E> {
517 /// Returns the decision to apply for `failure`, `proposed` being what the
518 /// runner would do on its own.
519 fn decide(
520 &self,
521 failure: AttemptFailure<'_, E>,
522 proposed: RetryDecision,
523 attempt: usize,
524 ) -> RetryDecision;
525}
526
527/// The default policy: it always applies what the runner proposes, leaving
528/// `fdb_transaction_on_error` the last word on retries.
529#[derive(Debug, Clone, Copy, Default)]
530pub struct NativeRetryPolicy;
531
532impl<E> RetryPolicy<E> for NativeRetryPolicy {
533 fn decide(
534 &self,
535 _failure: AttemptFailure<'_, E>,
536 proposed: RetryDecision,
537 _attempt: usize,
538 ) -> RetryDecision {
539 proposed
540 }
541}
542
543/// How a single attempt ended, carrying whatever still owns the transaction.
544enum Attempt<T, E> {
545 /// The closure succeeded and the transaction committed.
546 Committed {
547 committed: TransactionCommitted,
548 value: T,
549 commit_duration_ms: u64,
550 },
551 /// The closure failed, the transaction is untouched and can be retried.
552 ClosureFailed {
553 transaction: RetryableTransaction,
554 error: E,
555 },
556 /// The commit failed, the transaction is owned by the error.
557 CommitFailed(TransactionCommitError),
558 /// The runner could not get the transaction back from the closure.
559 BindingFailed(FdbBindingError),
560}
561
562/// Awaits the closure future and commits, without any retry logic.
563///
564/// The closure future is built by the caller so that the runner keeps owning
565/// the closure itself: an attempt that borrowed it would make every `run`
566/// future require `F: Sync`.
567async fn run_attempt<Fut, T, E, H>(
568 transaction: RetryableTransaction,
569 hooks: &H,
570 closure_result: Fut,
571 attempt: usize,
572) -> Attempt<T, E>
573where
574 Fut: Future<Output = Result<T, E>>,
575 H: RunnerHooks,
576{
577 let value = match closure_result.await {
578 Ok(value) => value,
579 Err(error) => return Attempt::ClosureFailed { transaction, error },
580 };
581
582 // Observational: a failing hook is reported, the commit happens anyway.
583 if let Err(hook_error) = hooks.before_commit(&transaction, attempt).await {
584 hooks.on_hook_error(&hook_error, attempt);
585 }
586
587 let now_commit = Instant::now();
588 match transaction.commit().await {
589 Ok(Ok(committed)) => Attempt::Committed {
590 committed,
591 value,
592 commit_duration_ms: now_commit.elapsed().as_millis() as u64,
593 },
594 Ok(Err(commit_error)) => Attempt::CommitFailed(commit_error),
595 Err(binding_error) => Attempt::BindingFailed(binding_error),
596 }
597}
598
599/// The retry loop: classification, policy, `on_error` and backoff on top of
600/// [`run_attempt`].
601///
602/// Wrapped by [`run_with_hooks`], which owns firing
603/// [`RunnerHooks::on_complete`] on every exit path.
604async fn retry_loop<F, Fut, T, E, H, P>(
605 initial_transaction: RetryableTransaction,
606 hooks: &H,
607 policy: &P,
608 closure: F,
609) -> Result<T, E>
610where
611 F: Fn(RetryableTransaction, MaybeCommitted) -> Fut,
612 Fut: Future<Output = Result<T, E>>,
613 E: RetryableError,
614 H: RunnerHooks,
615 P: RetryPolicy<E>,
616{
617 let mut maybe_committed = false;
618 let mut transaction = initial_transaction;
619 let mut attempt: usize = 0;
620
621 loop {
622 // The closure can issue operations while constructing its future.
623 hooks.on_attempt_start(&transaction, attempt);
624 let closure_result = closure(transaction.clone(), MaybeCommitted::new(maybe_committed));
625
626 let (retried_transaction, _fdb_err) =
627 match run_attempt(transaction, hooks, closure_result, attempt).await {
628 Attempt::Committed {
629 committed,
630 value,
631 commit_duration_ms,
632 } => {
633 hooks.on_commit_success(&committed, commit_duration_ms, attempt);
634
635 #[cfg(feature = "trace")]
636 tracing::info!(attempt, "success, returning result");
637
638 return Ok(value);
639 }
640
641 Attempt::BindingFailed(binding_error) => {
642 #[cfg(feature = "trace")]
643 tracing::error!(attempt, "transaction reference kept, aborting transaction");
644
645 return Err(E::from(binding_error));
646 }
647
648 Attempt::ClosureFailed { transaction, error } => {
649 let proposed = error.retry_decision();
650 // Later failures cannot resolve an earlier uncertain commit.
651 // Preserve it regardless of the policy's replacement error.
652 if let RetryDecision::Fdb(fdb_err) = proposed {
653 maybe_committed |= fdb_err.is_maybe_committed();
654 }
655
656 let fdb_err =
657 match policy.decide(AttemptFailure::Closure(&error), proposed, attempt) {
658 RetryDecision::Fdb(fdb_err) => fdb_err,
659 // Synthetic retryable code (1020, not_committed): backoff and
660 // RetryLimit stay governed by fdb_transaction_on_error.
661 RetryDecision::Retry => FdbError::from_code(NOT_COMMITTED),
662 RetryDecision::Fatal => return Err(error),
663 };
664 hooks.on_closure_error(&fdb_err, attempt);
665
666 let now_on_error = Instant::now();
667 match transaction.on_error(fdb_err).await {
668 Ok(Ok(transaction)) => {
669 hooks.on_error_duration(
670 now_on_error.elapsed().as_millis() as u64,
671 attempt,
672 );
673 (transaction, fdb_err)
674 }
675 // Retries exhausted or non-retryable: return the original
676 // closure error, which carries the caller's context.
677 Ok(Err(_non_retryable)) => return Err(error),
678 Err(binding_error) => return Err(E::from(binding_error)),
679 }
680 }
681
682 Attempt::CommitFailed(commit_error) => {
683 maybe_committed |= commit_error.is_maybe_committed();
684 let proposed = RetryDecision::Fdb(*commit_error);
685
686 // Fires before the policy is consulted: a hook reading the
687 // conflicting keys must see them even when the run is about
688 // to end.
689 if let Err(hook_error) = hooks.on_commit_error(&commit_error, attempt).await {
690 hooks.on_hook_error(&hook_error, attempt);
691 }
692
693 let fdb_err = match policy.decide(
694 AttemptFailure::Commit(&commit_error),
695 proposed,
696 attempt,
697 ) {
698 RetryDecision::Fdb(fdb_err) => fdb_err,
699 RetryDecision::Retry => FdbError::from_code(NOT_COMMITTED),
700 RetryDecision::Fatal => {
701 return Err(E::from(FdbBindingError::from(*commit_error)));
702 }
703 };
704
705 // The transaction is taken out of the error so that the
706 // error handed to `on_error` is the one the policy chose.
707 let (transaction, _commit_err) = commit_error.into_parts();
708 let now_on_error = Instant::now();
709 match transaction.on_error(fdb_err).await {
710 Ok(transaction) => {
711 hooks.on_error_duration(
712 now_on_error.elapsed().as_millis() as u64,
713 attempt,
714 );
715 (RetryableTransaction::new(transaction), fdb_err)
716 }
717 Err(non_retryable) => {
718 #[cfg(feature = "trace")]
719 tracing::error!(
720 attempt,
721 error_code = non_retryable.code(),
722 "could not commit, non retryable error"
723 );
724
725 return Err(E::from(FdbBindingError::from(non_retryable)));
726 }
727 }
728 }
729 };
730
731 #[cfg(feature = "trace")]
732 tracing::warn!(
733 attempt,
734 error_code = _fdb_err.code(),
735 "restarting transaction"
736 );
737
738 hooks.on_retry(attempt);
739 transaction = retried_transaction;
740 attempt += 1;
741 }
742}
743
744/// The retry runner: [`retry_loop`] plus the guarantee that
745/// [`RunnerHooks::on_complete`] fires exactly once.
746#[cfg_attr(
747 feature = "trace",
748 tracing::instrument(level = "debug", skip(initial_transaction, hooks, policy, closure))
749)]
750pub(crate) async fn run_with_hooks<F, Fut, T, E, H, P>(
751 initial_transaction: RetryableTransaction,
752 hooks: &H,
753 policy: &P,
754 closure: F,
755) -> Result<T, E>
756where
757 F: Fn(RetryableTransaction, MaybeCommitted) -> Fut,
758 Fut: Future<Output = Result<T, E>>,
759 E: RetryableError,
760 H: RunnerHooks,
761 P: RetryPolicy<E>,
762{
763 let result = retry_loop(initial_transaction, hooks, policy, closure).await;
764 hooks.on_complete();
765 result
766}
767
768/// Builder for a transactional run, returned by
769/// [`Database::runner`](crate::Database::runner).
770///
771/// It assembles the [`RunnerHooks`] and the [`RetryPolicy`] of a single call to
772/// [`run`](Self::run): no hooks and [`NativeRetryPolicy`] by default. Both are
773/// borrowed, so the same hooks and policy can be reused across runs (mind
774/// [`MetricsHooks`], which times the run from its own construction).
775///
776/// ```no_run
777/// # use foundationdb::*;
778/// # async fn example(db: &Database) -> Result<(), FdbBindingError> {
779/// db.runner()
780/// .run(|trx, _| async move {
781/// trx.set(b"key", b"value");
782/// Ok::<_, FdbBindingError>(())
783/// })
784/// .await
785/// # }
786/// ```
787pub struct TransactionRunner<'a, H = (), P = NativeRetryPolicy> {
788 db: &'a Database,
789 hooks: &'a H,
790 policy: &'a P,
791}
792
793impl<'a> TransactionRunner<'a, (), NativeRetryPolicy> {
794 /// A runner on `db` with no hooks and the native retry policy.
795 pub(crate) fn new(db: &'a Database) -> Self {
796 Self {
797 db,
798 hooks: &(),
799 policy: &NativeRetryPolicy,
800 }
801 }
802}
803
804impl<'a, H, P> TransactionRunner<'a, H, P> {
805 /// Observes the run with `hooks`, replacing any hooks set before.
806 ///
807 /// Stack several of them with a tuple: `.hooks(&(first, second))`.
808 pub fn hooks<H2>(self, hooks: &'a H2) -> TransactionRunner<'a, H2, P> {
809 TransactionRunner {
810 db: self.db,
811 hooks,
812 policy: self.policy,
813 }
814 }
815
816 /// Decides the retries with `policy` instead of [`NativeRetryPolicy`].
817 pub fn retry_policy<P2>(self, policy: &'a P2) -> TransactionRunner<'a, H, P2> {
818 TransactionRunner {
819 db: self.db,
820 hooks: self.hooks,
821 policy,
822 }
823 }
824
825 /// Runs `closure` in a transaction, retrying it until it commits or the
826 /// error is fatal.
827 ///
828 /// See [`Database::run`](crate::Database::run) for the semantics of the
829 /// closure, its error type and the retries.
830 #[cfg_attr(
831 feature = "trace",
832 tracing::instrument(level = "debug", skip(self, closure))
833 )]
834 pub async fn run<F, Fut, T, E>(self, closure: F) -> Result<T, E>
835 where
836 F: Fn(RetryableTransaction, MaybeCommitted) -> Fut,
837 Fut: Future<Output = Result<T, E>>,
838 E: RetryableError,
839 H: RunnerHooks,
840 P: RetryPolicy<E>,
841 {
842 let transaction = match self.db.create_retryable_trx() {
843 Ok(transaction) => transaction,
844 // The run is over before it started, but it did start: hooks that
845 // report on completion must still be called exactly once.
846 Err(err) => {
847 self.hooks.on_complete();
848 return Err(E::from(err));
849 }
850 };
851
852 run_with_hooks(transaction, self.hooks, self.policy, closure).await
853 }
854}