grafos_pipeline/
lib.rs

1//! Relocatable stage edges with generation-tracked handoff and durable
2//! checkpoints for grafOS pipelines.
3//!
4//! This crate provides the core primitives for building pipelines whose
5//! inter-stage edges can relocate transparently on failure. The key pattern:
6//!
7//! 1. A **producer** pushes items into a
8//!    [`grafos_collections::queue::FabricQueue`] via a
9//!    [`RelocatableQueueEdge`].
10//! 2. When the queue's lease expires or the connection drops, the producer
11//!    allocates a new queue, publishes the new
12//!    [`grafos_locator::locator::QueueLocator`] via a
13//!    [`grafos_locator::handoff::HandoffWriter`], and resumes pushing.
14//! 3. A **consumer** polls a [`grafos_locator::handoff::HandoffReader`] to detect the generation
15//!    bump, rebinds to the new queue, and resumes popping.
16//!
17//! Stage state can be durably checkpointed to block storage via
18//! [`CheckpointedStageState`], enabling at-least-once delivery with
19//! idempotent sinks.
20//!
21//! `RelocatableQueueEdge` is a local handoff/rebind primitive. It does not
22//! make a `FabricQueue` cross-domain or quorum-replicated by itself. Phase 215
23//! replicated queues build the user-facing failure-domain abstraction above
24//! these mechanics using `grafos_replicated` resource policy, locators,
25//! idempotency, and failure observations.
26//!
27//! # Feature flags
28//!
29//! | Feature | Default | Effect |
30//! |---------|---------|--------|
31//! | `std` | Yes | Enables `std` across all grafos dependencies |
32//! | `observe` | No | Enables `grafos-observe` integration |
33
34#![cfg_attr(not(feature = "std"), no_std)]
35
36extern crate alloc;
37
38mod checkpoint;
39mod edge;
40mod error;
41mod stage_id;
42
43pub use checkpoint::{CheckpointFlushObserver, CheckpointedStageState};
44pub use edge::RelocatableQueueEdge;
45pub use error::EdgeError;
46pub use stage_id::StageId;
47
48/// Alias for [`grafos_fence::FenceEpoch`] used as the edge generation counter.
49pub type EdgeGeneration = grafos_fence::FenceEpoch;
50
51/// Checkpoint recording how far a stage has progressed through its
52/// input and output queues.
53#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
54pub struct EdgeCheckpoint {
55    /// How far we read from the input queue.
56    pub input_offset: u64,
57    /// How far we committed to the output queue.
58    pub output_commit: u64,
59}
grafos_pipeline/lib.rs

grafos_pipeline/
lib.rs
