From 7e311cd04eac1b2583c7a51403acc707e26d2633 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:02 -0500 Subject: [PATCH 01/49] feat(storage): add the Postgres-to-LMDB migration seam --- crates/nostream-storage/src/lib.rs | 23 +- crates/nostream-storage/src/migration.rs | 269 +++++++++++++++++++++++ 2 files changed, 284 insertions(+), 8 deletions(-) create mode 100644 crates/nostream-storage/src/migration.rs diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index ddee1d90..6b61e3b2 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -16,20 +16,27 @@ //! - `src/factories/event-repository-factory.ts` //! //! What it deliberately does **not** own: the relay handlers that call these traits, the -//! payments service, the daemon's backend factory and the `nostream-cli migrate pg-to-lmdb` -//! tool. The cache is not a cache server: no Redis client is added and -//! `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not ported — the decision and -//! what would overturn it are ADR `0010-services-worker-model-and-cache.md`. Both backends are -//! complete: Postgres against the 37-migration schema, and LMDB against the same column -//! contract for 9 of the capture's 10 repositories — `user_subscriptions` has a TypeScript -//! repository and no trait yet, and `events.first_seen` is stored but never indexed -//! (`docs/rewrite/decisions/0011-lmdb-backend.md`). +//! payments service, the daemon's backend factory and the tool behind +//! `nostream-cli migrate pg-to-lmdb`. It owns that tool's seam — [`migration`], the paged source +//! read and the exact-per-row target write — which exists for that one caller and for no other; +//! the copy, its manifest and its verification live in the CLI. The cache is not a cache server: +//! no Redis client is added and `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not +//! ported — the decision and what would overturn it are ADR +//! `0010-services-worker-model-and-cache.md`. Both backends are complete: Postgres against the +//! 37-migration schema, and LMDB against the same column contract for 9 of the capture's 10 +//! repositories — `user_subscriptions` has a TypeScript repository and no trait yet, and is the +//! one of those ten the migration does not move (`event_tags` is not moved either: it is the +//! trigger-maintained index whose LMDB `tags` database the event write path keeps), and +//! `events.first_seen` is stored but never indexed +//! (`docs/rewrite/decisions/0011-lmdb-backend.md`, +//! `docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`). pub mod backend; pub mod error; pub mod event_store; #[cfg(feature = "lmdb")] pub mod lmdb; +pub mod migration; #[cfg(feature = "postgres")] pub mod postgres; pub mod records; diff --git a/crates/nostream-storage/src/migration.rs b/crates/nostream-storage/src/migration.rs new file mode 100644 index 00000000..e14809a2 --- /dev/null +++ b/crates/nostream-storage/src/migration.rs @@ -0,0 +1,269 @@ +//! The Postgres-to-LMDB migration seam. +//! +//! This module exists for one caller — `nostream-cli migrate pg-to-lmdb` — and is **not** a +//! general read or write surface. Nothing in the relay, the daemon or the repository traits may +//! use it, and a second caller needs a decision record first +//! (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D1). +//! +//! It lives here rather than in the CLI because the pieces a copy needs are in this crate: the +//! Postgres row decoders (`crate::postgres::rows`), the event column list, and the LMDB +//! per-table write paths with their index entries. A CLI-side copy would restate nine column +//! lists and nine index write paths and drift from them. +//! +//! The seam is two traits and the rows they move. [`MigrationSource`] reads a table one keyset +//! page at a time with no filter of any kind; [`MigrationTarget`] writes a page exactly as the +//! source carries it and reads a page back for the verification pass. Neither goes through the +//! nine repository traits: no trait enumerates a table, and every trait write stamps, clears or +//! drops at least one column, so a trait-shaped copy would be both lossy and partial. +//! +//! Every type here compiles with no driver. The two implementations are `#[cfg(feature)]`, and +//! this vocabulary must stay buildable with neither. + +use chrono::{DateTime, Utc}; + +use crate::error::StorageError; +use crate::repositories::{ + DeliveryLogEntry, DvmJob, EventRecord, InviteCode, Invoice, Nip05Verification, OutboxMessage, + Report, StorageFuture, User, +}; + +/// One table the migration moves, in the order the copy walks them. +/// +/// The nine are the tables the store has a repository and a database for. Two of the capture's +/// ten repository tables are not here: `event_tags`, whose LMDB `tags` database the event write +/// path maintains itself, and `user_subscriptions`, which has no trait and no database +/// (`MigratedTable::NOT_MIGRATED` records both). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum MigratedTable { + DvmJobs, + Events, + InviteCodes, + Invoices, + Nip05Verifications, + NotificationDeliveryLog, + NotificationOutbox, + Reports, + Users, +} + +impl MigratedTable { + /// Every table, in the order the copy walks them. + pub const ALL: [Self; 9] = [ + Self::DvmJobs, + Self::Events, + Self::InviteCodes, + Self::Invoices, + Self::Nip05Verifications, + Self::NotificationDeliveryLog, + Self::NotificationOutbox, + Self::Reports, + Self::Users, + ]; + + /// The source tables this tool does not move, which the manifest prints as a stated limit. + pub const NOT_MIGRATED: [&'static str; 2] = ["event_tags", "user_subscriptions"]; + + /// The table's name, spelled as Postgres and the manifest spell it. + pub const fn name(self) -> &'static str { + match self { + Self::DvmJobs => "dvm_jobs", + Self::Events => "events", + Self::InviteCodes => "invite_codes", + Self::Invoices => "invoices", + Self::Nip05Verifications => "nip05_verifications", + Self::NotificationDeliveryLog => "notification_delivery_log", + Self::NotificationOutbox => "notification_outbox", + Self::Reports => "reports", + Self::Users => "users", + } + } +} + +/// A `users` row with the one column the seam's [`User`] omits. +/// +/// `fromDBUser` drops `tos_accepted_at` from the read shape (`src/utils/transform.ts:137-144`), +/// so no trait write can carry an operator's terms-of-service record; the migration does. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct MigratedUser { + pub user: User, + /// `users.tos_accepted_at`. + pub tos_accepted_at: Option>, +} + +/// One row of one table, as the migration moves it. +/// +/// One tuple variant per table, so a row is built as `MigrationRow::Report(Report { .. })` and +/// never as a struct variant; the variant tag is part of the canonical bytes. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub enum MigrationRow { + DvmJob(DvmJob), + Event(EventRecord), + InviteCode(InviteCode), + Invoice(Invoice), + Nip05Verification(Nip05Verification), + DeliveryLogEntry(DeliveryLogEntry), + OutboxMessage(OutboxMessage), + Report(Report), + User(MigratedUser), +} + +impl MigrationRow { + /// The table this row belongs to. + pub const fn table(&self) -> MigratedTable { + match self { + Self::DvmJob(_) => MigratedTable::DvmJobs, + Self::Event(_) => MigratedTable::Events, + Self::InviteCode(_) => MigratedTable::InviteCodes, + Self::Invoice(_) => MigratedTable::Invoices, + Self::Nip05Verification(_) => MigratedTable::Nip05Verifications, + Self::DeliveryLogEntry(_) => MigratedTable::NotificationDeliveryLog, + Self::OutboxMessage(_) => MigratedTable::NotificationOutbox, + Self::Report(_) => MigratedTable::Reports, + Self::User(_) => MigratedTable::Users, + } + } + + /// The row's key, in the byte form the LMDB database it belongs to is written under: the + /// 32 raw bytes of an id or a pubkey, the UTF-8 bytes of a text key, and the flip-signed + /// `i32` of `reports.id`. + pub fn key(&self) -> Vec { + match self { + Self::DvmJob(job) => job.id.as_bytes().to_vec(), + Self::Event(record) => record.event.id.as_bytes().to_vec(), + Self::InviteCode(invite) => invite.code.as_bytes().to_vec(), + Self::Invoice(invoice) => invoice.id.as_bytes().to_vec(), + Self::Nip05Verification(verification) => verification.pubkey.as_bytes().to_vec(), + Self::DeliveryLogEntry(entry) => entry.id.as_bytes().to_vec(), + Self::OutboxMessage(message) => message.id.as_bytes().to_vec(), + Self::Report(report) => report_key(report.id).to_vec(), + Self::User(user) => user.user.pubkey.as_bytes().to_vec(), + } + } + + /// The row's canonical bytes: `serde_json` of the row with the two normalisations applied + /// that make a source row and the same row read back from the store project identically. + /// + /// [`EventRecord::row_id`] is set to `None` — it is the one column that is not stored + /// (`crate::lmdb::rows` sets it to `None` on every read) — and every instant is truncated + /// to the microsecond precision a PostgreSQL `timestamp` column has. + pub fn canonical(&self) -> Result, StorageError> { + serde_json::to_vec(&self.normalised()).map_err(|error| StorageError::InvalidRow { + table: self.table().name(), + column: "value", + value: error.to_string(), + }) + } + + /// The row as the target stores it and as [`canonical`](Self::canonical) digests it. + pub(crate) fn normalised(&self) -> Self { + match self { + Self::DvmJob(job) => Self::DvmJob(DvmJob { + picked_up_at: job.picked_up_at.map(micros), + completed_at: job.completed_at.map(micros), + created_at: micros(job.created_at), + updated_at: micros(job.updated_at), + ..job.clone() + }), + Self::Event(record) => Self::Event(EventRecord { + row_id: None, + first_seen: record.first_seen.map(micros), + deleted_at: record.deleted_at.map(micros), + ..record.clone() + }), + Self::InviteCode(invite) => Self::InviteCode(InviteCode { + expires_at: invite.expires_at.map(micros), + created_at: micros(invite.created_at), + updated_at: micros(invite.updated_at), + ..invite.clone() + }), + Self::Invoice(invoice) => Self::Invoice(Invoice { + confirmed_at: invoice.confirmed_at.map(micros), + expires_at: invoice.expires_at.map(micros), + updated_at: micros(invoice.updated_at), + created_at: micros(invoice.created_at), + ..invoice.clone() + }), + Self::Nip05Verification(verification) => Self::Nip05Verification(Nip05Verification { + last_verified_at: verification.last_verified_at.map(micros), + last_checked_at: micros(verification.last_checked_at), + created_at: micros(verification.created_at), + updated_at: micros(verification.updated_at), + ..verification.clone() + }), + Self::DeliveryLogEntry(entry) => Self::DeliveryLogEntry(DeliveryLogEntry { + created_at: micros(entry.created_at), + ..entry.clone() + }), + Self::OutboxMessage(message) => Self::OutboxMessage(OutboxMessage { + available_at: micros(message.available_at), + delivered_at: message.delivered_at.map(micros), + created_at: micros(message.created_at), + updated_at: micros(message.updated_at), + ..message.clone() + }), + Self::Report(report) => Self::Report(Report { + created_at: micros(report.created_at), + ..report.clone() + }), + Self::User(user) => Self::User(MigratedUser { + user: User { + created_at: micros(user.user.created_at), + updated_at: micros(user.user.updated_at), + ..user.user.clone() + }, + tos_accepted_at: user.tos_accepted_at.map(micros), + }), + } + } +} + +/// Reads one table of the source, a keyset page at a time. +/// +/// The implementation issues no statement but these reads: the migration never writes to the +/// source (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D12). +pub trait MigrationSource: Send + Sync { + /// The rows of `table` after `after`, in the table's key order, at most `limit` of them. + /// + /// `after` is exclusive and `None` starts at the table's first row. A page shorter than + /// `limit` means the table has no more rows. + fn read_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec>; +} + +/// Writes one page of the source's rows into the store, and reads a page back. +pub trait MigrationTarget: Send + Sync { + /// Writes every row in `rows`, which all belong to one table, in one transaction, and + /// answers how many rows it wrote. + fn write_batch<'a>(&'a self, rows: &'a [MigrationRow]) -> StorageFuture<'a, u64>; + + /// The rows of `table` after `after`, in the store's key order, at most `limit` of them. + /// + /// This is the read the verification pass folds: it takes the rows a copy would have + /// written, not the rows a repository read would return. + fn scan_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec>; +} + +/// The instant at the microsecond precision every PostgreSQL `timestamp` column has. +/// +/// The store's value codec truncates the same way (`lmdb::encoding::micros`); this module +/// restates it because it has to compile with no driver. +fn micros(value: DateTime) -> DateTime { + DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) +} + +/// The `reports.id` key: an `i32` ordered by its bytes, the sign bit flipped so a negative value +/// sorts before zero. +/// +/// The store's `lmdb::encoding::i32_key`, restated for the same reason as [`micros`]. +fn report_key(id: i32) -> [u8; 4] { + ((id as u32) ^ (1 << 31)).to_be_bytes() +} From 3448639a25d99b5a4bb8b129869046cfb790da4c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:02 -0500 Subject: [PATCH 02/49] feat(storage): read the migration's source from Postgres, one keyset page at a time --- .../nostream-storage/src/postgres/invoices.rs | 2 +- .../nostream-storage/src/postgres/metadata.rs | 8 +- .../src/postgres/migration.rs | 186 ++++++++++++++++++ crates/nostream-storage/src/postgres/mod.rs | 1 + .../src/postgres/notifications.rs | 4 +- crates/nostream-storage/src/postgres/sql.rs | 2 +- 6 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 crates/nostream-storage/src/postgres/migration.rs diff --git a/crates/nostream-storage/src/postgres/invoices.rs b/crates/nostream-storage/src/postgres/invoices.rs index 6a5dc7f7..74c32aff 100644 --- a/crates/nostream-storage/src/postgres/invoices.rs +++ b/crates/nostream-storage/src/postgres/invoices.rs @@ -8,7 +8,7 @@ use super::{Arg, Args, EventQuery}; use crate::error::StorageError; use crate::repositories::{Invoice, InvoiceStatus}; -const INVOICE_COLUMNS: &str = "id, pubkey, bolt11, amount_requested, amount_paid, unit, status, \ +pub(super) const INVOICE_COLUMNS: &str = "id, pubkey, bolt11, amount_requested, amount_paid, unit, status, \ description, confirmed_at, expires_at, updated_at, created_at, verify_url"; /// `findById`: one row by primary key. diff --git a/crates/nostream-storage/src/postgres/metadata.rs b/crates/nostream-storage/src/postgres/metadata.rs index 8a92908a..5e359226 100644 --- a/crates/nostream-storage/src/postgres/metadata.rs +++ b/crates/nostream-storage/src/postgres/metadata.rs @@ -14,16 +14,16 @@ use crate::repositories::{ InviteCode, NewReport, Nip05Verification, Report, }; -const NIP05_COLUMNS: &str = "pubkey, nip05, domain, is_verified, last_verified_at, \ +pub(super) const NIP05_COLUMNS: &str = "pubkey, nip05, domain, is_verified, last_verified_at, \ last_checked_at, failure_count, created_at, updated_at"; -const INVITE_COLUMNS: &str = +pub(super) const INVITE_COLUMNS: &str = "code, created_by, claimed_by, expires_at, remaining_uses, created_at, updated_at"; -const DVM_COLUMNS: &str = "id, requester_pubkey, kind, worker_index, status, result_event_id, \ +pub(super) const DVM_COLUMNS: &str = "id, requester_pubkey, kind, worker_index, status, result_event_id, \ error, picked_up_at, completed_at, created_at, updated_at"; -const REPORT_COLUMNS: &str = "id, event_id, reporter_pubkey, reported_pubkey, \ +pub(super) const REPORT_COLUMNS: &str = "id, event_id, reporter_pubkey, reported_pubkey, \ reported_event_id, report_type, weight, actionable, created_at"; /// `Nip05VerificationRepository::find_by_pubkey`. diff --git a/crates/nostream-storage/src/postgres/migration.rs b/crates/nostream-storage/src/postgres/migration.rs new file mode 100644 index 00000000..e19b5e61 --- /dev/null +++ b/crates/nostream-storage/src/postgres/migration.rs @@ -0,0 +1,186 @@ +//! `MigrationSource` against Postgres: one keyset statement per table, decoded by the +//! backend's own row decoders. +//! +//! Every statement reads a whole table. It carries no filter of any kind, because the trait +//! reads cannot see the rows a migration has to move: `find_by_filters` adds +//! `deleted_at IS NULL AND (expires_at IS NULL OR expires_at > $now)`, so a soft-deleted or +//! expired event is unreachable through it, and no trait read enumerates a table at all. +//! +//! Nothing here writes. The migration issues no `INSERT`, `UPDATE`, `DELETE` or DDL against the +//! source, and never runs the embedded migrator: a source that has not been migrated fails with +//! the driver's own error (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D12). + +use sqlx::Row; +use sqlx::postgres::PgRow; + +use super::invoices::INVOICE_COLUMNS; +use super::metadata::{DVM_COLUMNS, INVITE_COLUMNS, NIP05_COLUMNS, REPORT_COLUMNS}; +use super::notifications::{DELIVERY_LOG_COLUMNS, OUTBOX_COLUMNS}; +use super::sql::EVENT_COLUMNS; +use super::{Arg, Args, EventQuery, PostgresBackend, rows}; +use crate::error::StorageError; +use crate::migration::{MigratedTable, MigratedUser, MigrationRow, MigrationSource}; +use crate::repositories::StorageFuture; + +/// `users` columns: [`super::users::find_by_pubkey`]'s list plus `tos_accepted_at`, which the +/// seam's [`crate::repositories::User`] omits and a migration still has to carry. +const USER_COLUMNS: &str = "pubkey, COALESCE(is_admitted, false) AS is_admitted, \ +COALESCE(is_vanished, false) AS is_vanished, COALESCE(balance, 0) AS balance, created_at, \ +updated_at, tos_accepted_at"; + +impl MigrationSource for PostgresBackend { + fn read_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let (sql, args) = statement(table, after, limit)?; + let rows = EventQuery { sql, args } + .into_query() + .fetch_all(self.master()) + .await?; + decode(table, &rows) + }) + } +} + +/// How a table's unique key is bound as a keyset cursor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Cursor { + /// A `bytea` key: the row's key bytes are the column's value. + Bytes, + /// A `uuid` key: the row's key bytes are its canonical text. + Uuid, + /// A `text`/`varchar` key: the row's key bytes are the column's value. + Text, + /// `reports.id`: the row's key is [`crate::migration::MigrationRow::key`]'s flip-signed + /// `i32`, which the column holds unflipped. + Int32Key, +} + +/// `SELECT FROM [WHERE > $1] ORDER BY LIMIT $n`. +/// +/// The cursor is the table's unique key, which is what makes the paging stable across a resume: +/// the copy's own `after` is that key in exactly this form. +fn statement( + table: MigratedTable, + after: Option<&[u8]>, + limit: usize, +) -> Result<(String, Args), StorageError> { + let (columns, key, cursor) = match table { + MigratedTable::DvmJobs => (DVM_COLUMNS, "id", Cursor::Bytes), + MigratedTable::Events => (EVENT_COLUMNS, "event_id", Cursor::Bytes), + MigratedTable::InviteCodes => (INVITE_COLUMNS, "code", Cursor::Text), + MigratedTable::Invoices => (INVOICE_COLUMNS, "id", Cursor::Uuid), + MigratedTable::Nip05Verifications => (NIP05_COLUMNS, "pubkey", Cursor::Bytes), + MigratedTable::NotificationDeliveryLog => (DELIVERY_LOG_COLUMNS, "id", Cursor::Uuid), + MigratedTable::NotificationOutbox => (OUTBOX_COLUMNS, "id", Cursor::Uuid), + MigratedTable::Reports => (REPORT_COLUMNS, "id", Cursor::Int32Key), + MigratedTable::Users => (USER_COLUMNS, "pubkey", Cursor::Bytes), + }; + + let mut args = Args::default(); + let filter = match after { + None => String::new(), + Some(after) => { + let comparison = match cursor { + Cursor::Bytes => { + let slot = args.push(Arg::Bytes(after.to_vec())); + format!("{key} > {slot}") + } + Cursor::Uuid => { + let slot = args.push(Arg::Text(text(table, after)?)); + // An explicit cast: the bound value is text, the column is `uuid`. + format!("{key} > {slot}::uuid") + } + Cursor::Text => { + let slot = args.push(Arg::Text(text(table, after)?)); + format!("{key} > {slot}") + } + Cursor::Int32Key => { + let slot = args.push(Arg::Int(int32_key(table, after)?)); + format!("{key} > {slot}") + } + }; + format!(" WHERE {comparison}") + } + }; + let limit_slot = args.push(Arg::BigInt(i64::try_from(limit).unwrap_or(i64::MAX))); + let table_name = table.name(); + + Ok(( + format!("SELECT {columns} FROM {table_name} {filter} ORDER BY {key} LIMIT {limit_slot}"), + args, + )) +} + +/// One cursor as the text a `uuid`, `text` or `varchar` column holds. +fn text(table: MigratedTable, after: &[u8]) -> Result { + std::str::from_utf8(after) + .map(str::to_owned) + .map_err(|_| StorageError::InvalidRow { + table: table.name(), + column: "key", + value: "the resume cursor is not utf-8".to_owned(), + }) +} + +/// One cursor as the `i32` `reports.id` holds, undoing the key's sign flip. +fn int32_key(table: MigratedTable, after: &[u8]) -> Result { + let bytes: [u8; 4] = after.try_into().map_err(|_| StorageError::InvalidRow { + table: table.name(), + column: "key", + value: format!("{} bytes are not a reports key", after.len()), + })?; + Ok((u32::from_be_bytes(bytes) ^ (1 << 31)) as i32) +} + +/// One page of rows, through the decoder the backend already has for that table. +fn decode(table: MigratedTable, rows: &[PgRow]) -> Result, StorageError> { + match table { + MigratedTable::DvmJobs => rows + .iter() + .map(|row| rows::dvm_job(row).map(MigrationRow::DvmJob)) + .collect(), + MigratedTable::Events => rows + .iter() + .map(|row| rows::event_record(row).map(MigrationRow::Event)) + .collect(), + MigratedTable::InviteCodes => rows + .iter() + .map(|row| rows::invite_code(row).map(MigrationRow::InviteCode)) + .collect(), + MigratedTable::Invoices => rows + .iter() + .map(|row| rows::invoice(row).map(MigrationRow::Invoice)) + .collect(), + MigratedTable::Nip05Verifications => rows + .iter() + .map(|row| rows::nip05_verification(row).map(MigrationRow::Nip05Verification)) + .collect(), + MigratedTable::NotificationDeliveryLog => rows + .iter() + .map(|row| rows::delivery_log_entry(row).map(MigrationRow::DeliveryLogEntry)) + .collect(), + MigratedTable::NotificationOutbox => rows + .iter() + .map(|row| rows::outbox_message(row).map(MigrationRow::OutboxMessage)) + .collect(), + MigratedTable::Reports => rows + .iter() + .map(|row| rows::report(row).map(MigrationRow::Report)) + .collect(), + MigratedTable::Users => rows.iter().map(user).collect(), + } +} + +/// One user row: [`rows::user`] plus the column the seam's read shape drops. +fn user(row: &PgRow) -> Result { + let tos_accepted_at: Option = row.try_get("tos_accepted_at")?; + Ok(MigrationRow::User(MigratedUser { + user: rows::user(row)?, + tos_accepted_at: tos_accepted_at.map(crate::records::utc_from_naive), + })) +} diff --git a/crates/nostream-storage/src/postgres/mod.rs b/crates/nostream-storage/src/postgres/mod.rs index ed949010..d81506fe 100644 --- a/crates/nostream-storage/src/postgres/mod.rs +++ b/crates/nostream-storage/src/postgres/mod.rs @@ -10,6 +10,7 @@ mod events; mod invoices; mod metadata; +mod migration; mod notifications; mod repository; mod rows; diff --git a/crates/nostream-storage/src/postgres/notifications.rs b/crates/nostream-storage/src/postgres/notifications.rs index bd017e4a..579e3d81 100644 --- a/crates/nostream-storage/src/postgres/notifications.rs +++ b/crates/nostream-storage/src/postgres/notifications.rs @@ -13,10 +13,10 @@ use crate::repositories::{ NewDeliveryLogEntry, OutboxMessage, OutboxPayload, OutboxStatus, }; -const OUTBOX_COLUMNS: &str = "id::text AS id, event_type, payload, status, attempt_count, \ +pub(super) const OUTBOX_COLUMNS: &str = "id::text AS id, event_type, payload, status, attempt_count, \ available_at, last_error, delivered_at, created_at, updated_at"; -const DELIVERY_LOG_COLUMNS: &str = "id::text AS id, outbox_id::text AS outbox_id, event_type, \ +pub(super) const DELIVERY_LOG_COLUMNS: &str = "id::text AS id, outbox_id::text AS outbox_id, event_type, \ target_id, target_type, status, attempt_number, error_snippet, created_at"; /// `append`: one delivery attempt, timestamped now. diff --git a/crates/nostream-storage/src/postgres/sql.rs b/crates/nostream-storage/src/postgres/sql.rs index 098745c8..563b0c0c 100644 --- a/crates/nostream-storage/src/postgres/sql.rs +++ b/crates/nostream-storage/src/postgres/sql.rs @@ -11,7 +11,7 @@ use super::{DEFAULT_FILTER_LIMIT, DEFAULT_TS_CONFIG}; use crate::error::StorageError; /// The columns one stored event row is read from. -const EVENT_COLUMNS: &str = "events.id::text AS id, events.event_id, events.event_pubkey, \ +pub(super) const EVENT_COLUMNS: &str = "events.id::text AS id, events.event_id, events.event_pubkey, \ events.event_kind, events.event_created_at, events.event_content, events.event_tags, \ events.event_signature, events.first_seen, events.deleted_at, events.event_deduplication, \ host(events.remote_address) AS remote_address, events.expires_at"; From 98d94d6525e367dd435e443ab50e9d30d345bb7d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:02 -0500 Subject: [PATCH 03/49] feat(storage): write the migration's rows into LMDB exactly as the source carries them --- crates/nostream-storage/src/lmdb/events.rs | 33 ++++ crates/nostream-storage/src/lmdb/invoices.rs | 5 +- crates/nostream-storage/src/lmdb/metadata.rs | 46 ++++- crates/nostream-storage/src/lmdb/migration.rs | 179 ++++++++++++++++++ crates/nostream-storage/src/lmdb/mod.rs | 1 + .../src/lmdb/notifications.rs | 27 ++- 6 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 crates/nostream-storage/src/lmdb/migration.rs diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index c1281b4f..b262a842 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -247,6 +247,39 @@ fn write_event( Ok(()) } +/// Writes one migrated event: every entry [`write_event`] writes, the `replaceable` pointer +/// [`write_upserted`] writes, and the `events_deleted` entry [`soft_delete`] writes when the row +/// carries `deleted_at`. +/// +/// Nothing is stamped, cleared or defaulted: a migrated row keeps the `first_seen`, +/// `deleted_at`, `expires_at`, `event_deduplication` and `remote_address` the source carries. A +/// row that arrives soft-deleted or expired never passed through [`soft_delete`], so both tail +/// entries are this writer's own work. +pub(super) fn write_migrated( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + write_event(txn, databases, record)?; + let kind = kind_of(record)?; + if encoding::replaceable_kind(kind) { + let deduplication = record.deduplication(); + databases.replaceable.put( + txn, + &encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?, + record.event.id.as_bytes(), + )?; + } + if let Some(deleted_at) = record.deleted_at { + databases.events_deleted.put( + txn, + &encoding::time_key(deleted_at.timestamp(), &record.event.id), + &[], + )?; + } + Ok(()) +} + /// Writes only the `events` value, which is what a soft delete changes. fn put_value( txn: &mut RwTxn<'_>, diff --git a/crates/nostream-storage/src/lmdb/invoices.rs b/crates/nostream-storage/src/lmdb/invoices.rs index 4bfd667f..287e7893 100644 --- a/crates/nostream-storage/src/lmdb/invoices.rs +++ b/crates/nostream-storage/src/lmdb/invoices.rs @@ -33,7 +33,10 @@ fn pending_key(created_at: DateTime, id: &str) -> Vec { } /// Writes a row and moves its pending entry, when the status or the row changed. -fn write( +/// +/// The migration's writer too ([`super::migration`]), with no previous row: it puts the source's +/// own `amount_paid` and `confirmed_at` and the pending entry the row's status asks for. +pub(super) fn write( txn: &mut RwTxn<'_>, databases: &Databases, invoice: &Invoice, diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs index bfcd2807..f8b4ef9c 100644 --- a/crates/nostream-storage/src/lmdb/metadata.rs +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -83,6 +83,25 @@ pub(crate) fn nip05_upsert( Ok(1) } +/// Writes one verification row and its pending-check entry as the source carries them. +/// +/// [`nip05_upsert`] keeps a stored `created_at` and stamps `updated_at`; a migrated row arrives +/// with both of its own, so this puts the row it is given and its own index entry. +pub(super) fn nip05_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + verification: &Nip05Verification, +) -> Result<(), StorageError> { + let value = rows::encode("nip05_verifications", verification)?; + databases + .nip05 + .put(txn, &nip05_key(&verification.pubkey), &value)?; + databases + .nip05_by_last_checked_at + .put(txn, &nip05_pending_key(verification), &[])?; + Ok(()) +} + /// `Nip05VerificationRepository::find_pending_verifications`: rows last checked before the /// cutoff with fewer than `max_failures`, oldest check first. pub(crate) fn nip05_find_pending( @@ -162,7 +181,7 @@ pub(crate) fn invite_find_by_code( rows::decode("invite_codes", value).map(Some) } -fn invite_write( +pub(super) fn invite_write( txn: &mut RwTxn<'_>, databases: &Databases, invite: &InviteCode, @@ -346,7 +365,7 @@ pub(crate) fn dvm_find_by_id( rows::decode("dvm_jobs", value).map(Some) } -fn dvm_write( +pub(super) fn dvm_write( txn: &mut RwTxn<'_>, databases: &Databases, job: &DvmJob, @@ -549,7 +568,28 @@ fn next_report_id(txn: &mut RwTxn<'_>, databases: &Databases) -> Result, + databases: &Databases, + id: i32, +) -> Result<(), StorageError> { + let wanted = u64::try_from(id).unwrap_or(0); + let current: u64 = match databases.meta.get(txn, REPORT_SEQUENCE)? { + Some(value) => rows::decode("meta", value)?, + None => 0, + }; + if wanted > current { + let value = rows::encode("meta", &wanted)?; + databases.meta.put(txn, REPORT_SEQUENCE, &value)?; + } + Ok(()) +} + +pub(super) fn report_write( txn: &mut RwTxn<'_>, databases: &Databases, report: &Report, diff --git a/crates/nostream-storage/src/lmdb/migration.rs b/crates/nostream-storage/src/lmdb/migration.rs new file mode 100644 index 00000000..ab7c5957 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/migration.rs @@ -0,0 +1,179 @@ +//! `MigrationTarget` against LMDB: one write transaction per batch, one migration-only writer +//! per table. +//! +//! Every writer puts the source's own key and value and the index entries that table's existing +//! write path writes, and stamps, clears and defaults nothing. That is what makes a replayed +//! batch idempotent: the same row written twice is a put of the same key with the same value. +//! +//! What each writer is handed is the row's normalised form ([`MigrationRow::normalised`]), so +//! every instant is at the microsecond precision the value codec stores, and an event's +//! `row_id` — the one column the store does not keep — is already `None`. + +use heed::types::Bytes; +use heed::{Database, RoTxn, RwTxn}; + +use super::rows::StoredUser; +use super::{Databases, LmdbBackend, encoding, events, invoices, metadata, notifications, rows}; +use crate::error::StorageError; +use crate::migration::{MigratedTable, MigratedUser, MigrationRow, MigrationTarget}; +use crate::repositories::StorageFuture; + +impl MigrationTarget for LmdbBackend { + fn write_batch<'a>(&'a self, rows: &'a [MigrationRow]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let Some(first) = rows.first() else { + return Ok(0); + }; + let table = first.table(); + if rows.iter().any(|row| row.table() != table) { + return Err(StorageError::InvalidArgument { + message: "a migration batch must hold one table's rows", + }); + } + + // The blocking closure owns what it touches, so the batch moves into it: one + // transaction for the whole page, which either lands or does not. + let batch = rows.to_vec(); + self.write(move |txn, databases| write_batch(txn, databases, &batch)) + .await?; + Ok(rows.len() as u64) + }) + } + + fn scan_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let after = after.map(<[u8]>::to_vec); + self.read(move |txn, databases| scan(txn, databases, table, after.as_deref(), limit)) + .await + }) + } +} + +/// Writes one batch, and raises `meta`'s `reports.next_id` past the batch's largest report so a +/// later insert cannot collide with a migrated row. +fn write_batch( + txn: &mut RwTxn<'_>, + databases: &Databases, + batch: &[MigrationRow], +) -> Result<(), StorageError> { + let mut largest_report = 0; + for row in batch { + match row.normalised() { + MigrationRow::DvmJob(job) => metadata::dvm_write(txn, databases, &job, None)?, + MigrationRow::Event(record) => events::write_migrated(txn, databases, &record)?, + MigrationRow::InviteCode(invite) => metadata::invite_write(txn, databases, &invite)?, + MigrationRow::Invoice(invoice) => invoices::write(txn, databases, &invoice, None)?, + MigrationRow::Nip05Verification(verification) => { + metadata::nip05_write(txn, databases, &verification)?; + } + MigrationRow::DeliveryLogEntry(entry) => { + notifications::delivery_write(txn, databases, &entry)?; + } + MigrationRow::OutboxMessage(message) => { + notifications::outbox_write(txn, databases, &message, None)?; + } + MigrationRow::Report(report) => { + metadata::report_write(txn, databases, &report)?; + largest_report = largest_report.max(report.id); + } + MigrationRow::User(user) => { + users_store(txn, databases, &user)?; + } + } + } + metadata::raise_report_sequence(txn, databases, largest_report) +} + +/// Writes one `users` row: the whole [`StoredUser`], including the `tos_accepted_at` no trait +/// write carries. +fn users_store( + txn: &mut RwTxn<'_>, + databases: &Databases, + migrated: &MigratedUser, +) -> Result<(), StorageError> { + let user = &migrated.user; + super::users::store( + txn, + databases, + &StoredUser { + pubkey: user.pubkey, + is_admitted: user.is_admitted, + is_vanished: user.is_vanished, + balance: user.balance, + tos_accepted_at: migrated.tos_accepted_at, + created_at: user.created_at, + updated_at: user.updated_at, + }, + ) +} + +/// One page of a table, in the store's key order, resuming exclusively after `after`. +fn scan( + txn: &RoTxn<'_>, + databases: &Databases, + table: MigratedTable, + after: Option<&[u8]>, + limit: usize, +) -> Result, StorageError> { + let database = database(databases, table); + let bounds = encoding::resume_bounds(after, b"", b""); + let mut rows = Vec::new(); + for entry in database.range(txn, &bounds)? { + if rows.len() >= limit { + break; + } + let (_, value) = entry?; + rows.push(decode(table, value)?); + } + Ok(rows) +} + +/// The database one table's rows live in. +fn database(databases: &Databases, table: MigratedTable) -> Database { + match table { + MigratedTable::DvmJobs => databases.dvm_jobs, + MigratedTable::Events => databases.events, + MigratedTable::InviteCodes => databases.invite_codes, + MigratedTable::Invoices => databases.invoices, + MigratedTable::Nip05Verifications => databases.nip05, + MigratedTable::NotificationDeliveryLog => databases.notification_delivery_log, + MigratedTable::NotificationOutbox => databases.outbox, + MigratedTable::Reports => databases.reports, + MigratedTable::Users => databases.users, + } +} + +/// One stored value, as the row the seam carries. +fn decode(table: MigratedTable, value: &[u8]) -> Result { + match table { + MigratedTable::DvmJobs => rows::decode("dvm_jobs", value).map(MigrationRow::DvmJob), + MigratedTable::Events => rows::decode_event(value).map(MigrationRow::Event), + MigratedTable::InviteCodes => { + rows::decode("invite_codes", value).map(MigrationRow::InviteCode) + } + MigratedTable::Invoices => rows::decode("invoices", value).map(MigrationRow::Invoice), + MigratedTable::Nip05Verifications => { + rows::decode("nip05_verifications", value).map(MigrationRow::Nip05Verification) + } + MigratedTable::NotificationDeliveryLog => { + rows::decode("notification_delivery_log", value).map(MigrationRow::DeliveryLogEntry) + } + MigratedTable::NotificationOutbox => { + rows::decode("notification_outbox", value).map(MigrationRow::OutboxMessage) + } + MigratedTable::Reports => rows::decode("reports", value).map(MigrationRow::Report), + MigratedTable::Users => { + let stored: StoredUser = rows::decode("users", value)?; + let tos_accepted_at = stored.tos_accepted_at; + Ok(MigrationRow::User(MigratedUser { + user: stored.user(), + tos_accepted_at, + })) + } + } +} diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 61b89184..d976de36 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -11,6 +11,7 @@ mod encoding; mod events; mod invoices; mod metadata; +mod migration; mod notifications; mod rows; mod users; diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index 3c73734c..62264b63 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -152,6 +152,31 @@ pub(crate) fn append( Ok(()) } +/// Writes one delivery-log row exactly as the source carries it. +/// +/// [`append`] mints the id and the timestamp and checks the outbox foreign key; a migrated row +/// arrives with its own id, its own `created_at` and an outbox row that is in the source. The +/// value and the two index keys are the same set `append` writes. +pub(super) fn delivery_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + entry: &DeliveryLogEntry, +) -> Result<(), StorageError> { + let value = rows::encode("notification_delivery_log", entry)?; + databases + .notification_delivery_log + .put(txn, entry.id.as_bytes(), &value)?; + databases + .delivery_log_by_created_at + .put(txn, &delivery_created_key(entry), &[])?; + if let Some(outbox_id) = &entry.outbox_id { + databases + .delivery_log_by_outbox + .put(txn, &delivery_outbox_key(outbox_id, entry)?, &[])?; + } + Ok(()) +} + /// `find_recent`: newest first, optionally filtered by status and event type. pub(crate) fn find_recent( txn: &RoTxn<'_>, @@ -288,7 +313,7 @@ fn outbox_load( } /// Writes a message and its two status indexes, removing the entries of the row it replaces. -fn outbox_write( +pub(super) fn outbox_write( txn: &mut RwTxn<'_>, databases: &Databases, message: &OutboxMessage, From f5bddc9b81de808a3b7ea6c8ca2ef0631da46ab4 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:06 -0500 Subject: [PATCH 04/49] feat(cli): add the migrate pg-to-lmdb subcommand with a resumable manifest --- Cargo.lock | 5 + crates/nostream-cli/Cargo.toml | 5 + crates/nostream-cli/src/commands/migrate.rs | 431 ++++++++++++++++++++ crates/nostream-cli/src/commands/mod.rs | 1 + crates/nostream-cli/src/error.rs | 5 + crates/nostream-cli/src/main.rs | 51 +++ 6 files changed, 498 insertions(+) create mode 100644 crates/nostream-cli/src/commands/migrate.rs diff --git a/Cargo.lock b/Cargo.lock index ef684388..062a8c2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1244,13 +1244,18 @@ name = "nostream-cli" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", + "heed", + "hex", "nostream-config", "nostream-core", "nostream-services", "nostream-storage", + "serde", "serde_json", "serde_yaml", + "sha2 0.11.1", "tempfile", "thiserror", "tokio", diff --git a/crates/nostream-cli/Cargo.toml b/crates/nostream-cli/Cargo.toml index 0aa379ff..fca0a721 100644 --- a/crates/nostream-cli/Cargo.toml +++ b/crates/nostream-cli/Cargo.toml @@ -14,16 +14,21 @@ path = "src/main.rs" [dependencies] clap.workspace = true anyhow = { workspace = true } +hex.workspace = true nostream-config.workspace = true nostream-core.workspace = true nostream-services.workspace = true nostream-storage.workspace = true +serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true +sha2.workspace = true thiserror.workspace = true tokio.workspace = true [dev-dependencies] +chrono.workspace = true +heed.workspace = true tempfile.workspace = true [lints] diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs new file mode 100644 index 00000000..4367a424 --- /dev/null +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -0,0 +1,431 @@ +//! `migrate pg-to-lmdb`: the copy of the Postgres store into an LMDB one. +//! +//! The seam is `nostream_storage::migration` — a paged read of each source table and an exact +//! write of each row, neither of which the nine repository traits can express. This module owns +//! what the seam's caller owns: the manifest, the checksum, resume, the verification pass and +//! the run's own measurement (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`). +//! +//! The batch commits to the store first and the manifest is replaced atomically second, so a +//! crash between them replays one batch: every write is a put of the source's own key and +//! value, so the replay is idempotent and the merged entry is unchanged. + +use std::collections::BTreeMap; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; +use nostream_storage::{LmdbBackend, PostgresBackend}; +use sha2::{Digest, Sha256}; + +use crate::config::CommandOutput; +use crate::error::CliError; + +/// The manifest's file name under the target root. +const MANIFEST_FILE: &str = "migration-manifest.json"; + +/// The manifest format this tool writes. +const MANIFEST_FORMAT: u32 = 1; + +/// The options one run carries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MigrateOptions { + /// The store root: the environment is `/lmdb`, the manifest `/migration-manifest.json`. + pub target: PathBuf, + /// Rows per batch, which is also the page size verify reads with. + pub batch_size: usize, + /// Verify the target against its manifest and copy nothing. + pub verify: bool, +} + +/// `migrate pg-to-lmdb`: the copy, or the verification pass when `verify` is set. +pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| CliError::Config(format!("tokio runtime: {error}")))?; + + runtime.block_on(async move { + if options.verify { + // Answered before the store is opened, so a verify of a directory that holds no + // manifest leaves it exactly as it found it. + if read_manifest(&options.target)?.is_none() { + return no_manifest(&options.target, out); + } + let target = open_target(&options.target).await?; + return verify_with(&target, &options, out).await; + } + + // A complete manifest is verified rather than copied, and the source is never opened: + // a finished migration must not need Postgres to still be reachable. + if read_manifest(&options.target)?.is_some_and(|manifest| manifest.complete()) { + let target = open_target(&options.target).await?; + return verify_with(&target, &options, out).await; + } + + let source = PostgresBackend::from_env() + .await + .map_err(|error| CliError::Migrate(error.to_string()))?; + let target = open_target(&options.target).await?; + migrate_with(&source, &target, &options, out).await + }) +} + +/// The copy over an injected source, target and writer, which is how the whole run is tested +/// without a process and without a database. +/// +/// Every batch's rows are written to the store and then folded into the manifest, which is +/// replaced after each batch; a batch that fails leaves the previous batch's `after`, `rows` and +/// `checksum` in place, so the next run resumes at the refused row. +pub async fn migrate_with( + source: &dyn MigrationSource, + target: &dyn MigrationTarget, + options: &MigrateOptions, + out: &mut dyn Write, +) -> Result { + let mut manifest = match read_manifest(&options.target)? { + Some(manifest) => manifest, + None => Manifest::fresh(), + }; + if manifest.complete() { + return verify_with(target, options, out).await; + } + + let started = Instant::now(); + // The wall clock the summary reports: the last value is the one taken just after the last + // `write_batch` of this run. + let mut measured: Option = None; + let mut total = 0u64; + let mut events = 0u64; + + for table in MigratedTable::ALL { + let entry = manifest.get(table); + if entry.complete { + continue; + } + let mut rows = entry.rows; + let mut sum = RowSum::from_hex(&entry.checksum)?; + let mut after = decode_key(&entry.after)?; + + loop { + let batch = source + .read_batch(table, after.as_deref(), options.batch_size) + .await + .map_err(|error| table_failed(table, &error))?; + let count = batch.len(); + let written = target + .write_batch(&batch) + .await + .map_err(|error| table_failed(table, &error))?; + measured = Some(started.elapsed()); + + for row in &batch { + sum.add(row)?; + } + rows += written; + total += written; + if table == MigratedTable::Events { + events += written; + } + if let Some(last) = batch.last() { + after = Some(last.key()); + } + + let complete = count < options.batch_size; + let entry = manifest.entry(table); + entry.rows = rows; + entry.checksum = sum.to_hex(); + entry.after = after.as_deref().map(hex::encode).unwrap_or_default(); + entry.complete = complete; + write_manifest(&options.target, &manifest)?; + + if complete { + break; + } + } + } + + let ms = u64::try_from(measured.unwrap_or_default().as_millis()).unwrap_or(u64::MAX); + let rate = events.saturating_mul(1_000) / ms.max(1); + writeln!(out, "Migrated {total} rows in {ms} ms").map_err(write_failed)?; + writeln!(out, "Events throughput: {rate} events/s").map_err(write_failed)?; + + Ok(CommandOutput::default()) +} + +/// The verification pass: fold each table in the store the way the copy folded it, and compare +/// the nine entries with the manifest. +/// +/// Every table is paged to its end, so a table longer than one page is read rather than cut at +/// its first. The pass writes nothing to the store and never opens the source. +pub async fn verify_with( + target: &dyn MigrationTarget, + options: &MigrateOptions, + out: &mut dyn Write, +) -> Result { + let Some(manifest) = read_manifest(&options.target)? else { + return no_manifest(&options.target, out); + }; + + let mut failures: Vec = Vec::new(); + let mut total = 0u64; + + for table in MigratedTable::ALL { + let entry = manifest.get(table); + let mut rows = 0u64; + let mut sum = RowSum::default(); + let mut after: Option> = None; + + loop { + let page = target + .scan_batch(table, after.as_deref(), options.batch_size) + .await + .map_err(|error| table_failed(table, &error))?; + let count = page.len(); + for row in &page { + sum.add(row)?; + } + rows += count as u64; + if let Some(last) = page.last() { + after = Some(last.key()); + } + if count < options.batch_size { + break; + } + } + total += rows; + + if rows != entry.rows { + failures.push(format!( + "[{}] manifest {} rows, target {rows} rows", + table.name(), + entry.rows + )); + } else if sum.to_hex() != entry.checksum { + failures.push(format!( + "[{}] checksum mismatch (manifest {}, target {})", + table.name(), + entry.checksum, + sum.to_hex() + )); + } else if !entry.complete { + failures.push(format!("[{}] the manifest is incomplete", table.name())); + } + } + + if failures.is_empty() { + writeln!( + out, + "Verified {} tables, {total} rows", + MigratedTable::ALL.len() + ) + .map_err(write_failed)?; + return Ok(CommandOutput::default()); + } + + for line in &failures { + writeln!(out, "{line}").map_err(write_failed)?; + } + Ok(CommandOutput { + lines: Vec::new(), + exit_code: 1, + }) +} + +/// The manifest: one entry per table, and the two tables this tool does not move. +/// +/// It carries no timestamp, rate or path, so two runs over the same source produce the same +/// bytes and a resumed run's manifest is the clean run's. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct Manifest { + format: u32, + not_migrated: Vec, + tables: BTreeMap, +} + +impl Manifest { + /// The manifest a first run starts from: every table empty and incomplete. + fn fresh() -> Self { + Self { + format: MANIFEST_FORMAT, + not_migrated: MigratedTable::NOT_MIGRATED + .iter() + .map(|name| (*name).to_owned()) + .collect(), + tables: MigratedTable::ALL + .iter() + .map(|table| (table.name().to_owned(), TableEntry::default())) + .collect(), + } + } + + /// This table's entry, as the empty one when the manifest carries none. + fn get(&self, table: MigratedTable) -> TableEntry { + self.tables.get(table.name()).cloned().unwrap_or_default() + } + + /// This table's entry, created empty when the manifest carries none. + fn entry(&mut self, table: MigratedTable) -> &mut TableEntry { + self.tables.entry(table.name().to_owned()).or_default() + } + + /// Whether every table has been copied to its end. + fn complete(&self) -> bool { + MigratedTable::ALL + .iter() + .all(|table| self.get(*table).complete) + } +} + +/// One table's accumulated state: what the copy has written and where it resumes. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct TableEntry { + /// The rows the table's batches have written. + rows: u64, + /// The hex of the table's running row sum; sixty-four `0` characters when it is empty. + checksum: String, + /// The hex of the last written key; empty before the first batch. + after: String, + /// True once a read returned fewer rows than the batch size. + complete: bool, +} + +impl Default for TableEntry { + fn default() -> Self { + Self { + rows: 0, + checksum: RowSum::default().to_hex(), + after: String::new(), + complete: false, + } + } +} + +/// The manifest's 256-bit row sum. +/// +/// For every row it adds `SHA-256(len32 ‖ key ‖ canonical)` into a wrapping 256-bit sum. The sum +/// is order-independent, needs constant memory, and composes across a resume +/// (`sum(A ∪ B) = sum(A) + sum(B)`); it detects an added, removed or altered row and is stated +/// as a corruption and completeness check, not as a cryptographic commitment +/// (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct RowSum([u64; 4]); + +impl RowSum { + /// Adds one row's digest. + fn add(&mut self, row: &MigrationRow) -> Result<(), CliError> { + let key = row.key(); + let length = u32::try_from(key.len()) + .map_err(|_| CliError::Migrate("a row key is longer than 4 GiB".to_owned()))?; + let canonical = row + .canonical() + .map_err(|error| CliError::Migrate(error.to_string()))?; + + let mut hasher = Sha256::new(); + hasher.update(length.to_be_bytes()); + hasher.update(&key); + hasher.update(&canonical); + let digest = hasher.finalize(); + let (words, _) = digest.as_chunks::<8>(); + for (limb, word) in self.0.iter_mut().zip(words) { + *limb = limb.wrapping_add(u64::from_be_bytes(*word)); + } + Ok(()) + } + + /// The sum a manifest entry carries, as its sixty-four hex characters. + fn from_hex(text: &str) -> Result { + let mut bytes = [0u8; 32]; + hex::decode_to_slice(text, &mut bytes).map_err(|error| { + CliError::Migrate(format!( + "the manifest's checksum is not 64 hexadecimal characters: {error}" + )) + })?; + let mut limbs = [0u64; 4]; + let (words, _) = bytes.as_chunks::<8>(); + for (limb, word) in limbs.iter_mut().zip(words) { + *limb = u64::from_be_bytes(*word); + } + Ok(Self(limbs)) + } + + /// The sum as sixty-four lowercase hex characters. + fn to_hex(self) -> String { + let mut bytes = [0u8; 32]; + let (words, _) = bytes.as_chunks_mut::<8>(); + for (word, limb) in words.iter_mut().zip(self.0) { + *word = limb.to_be_bytes(); + } + hex::encode(bytes) + } +} + +/// The manifest's path under a target root. +fn manifest_path(target: &Path) -> PathBuf { + target.join(MANIFEST_FILE) +} + +/// The manifest at the target, or `None` when there is none. +fn read_manifest(target: &Path) -> Result, CliError> { + let path = manifest_path(target); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(CliError::Migrate(format!("{}: {error}", path.display()))), + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display()))) +} + +/// Replaces the manifest through a temporary file, so a reader never sees a half-written one. +fn write_manifest(target: &Path, manifest: &Manifest) -> Result<(), CliError> { + let path = manifest_path(target); + let temporary = path.with_extension("json.tmp"); + let bytes = serde_json::to_vec_pretty(manifest) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + fs::write(&temporary, &bytes) + .map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?; + fs::rename(&temporary, &path) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display()))) +} + +/// The line and exit code a verify answers when the target has no manifest. +fn no_manifest(target: &Path, out: &mut dyn Write) -> Result { + writeln!(out, "No manifest at {}", manifest_path(target).display()).map_err(write_failed)?; + Ok(CommandOutput { + lines: Vec::new(), + exit_code: 1, + }) +} + +/// Opens the store a run reads or writes. +async fn open_target(root: &Path) -> Result { + LmdbBackend::open(root) + .await + .map_err(|error| CliError::Migrate(error.to_string())) +} + +/// A failure of one table's read or write, naming the table so an operator knows which row +/// stopped the run. +fn table_failed(table: MigratedTable, error: &dyn std::fmt::Display) -> CliError { + CliError::Migrate(format!("{}: {error}", table.name())) +} + +/// One hex resume cursor, or `None` before the table's first batch. +fn decode_key(text: &str) -> Result>, CliError> { + if text.is_empty() { + return Ok(None); + } + hex::decode(text).map(Some).map_err(|error| { + CliError::Migrate(format!( + "the manifest's resume cursor is not hexadecimal: {error}" + )) + }) +} + +/// A write failure is an error, not a success with nothing delivered. +fn write_failed(error: std::io::Error) -> CliError { + CliError::Migrate(format!("unable to write the migration's output: {error}")) +} diff --git a/crates/nostream-cli/src/commands/mod.rs b/crates/nostream-cli/src/commands/mod.rs index 3fe4416e..645c0dc8 100644 --- a/crates/nostream-cli/src/commands/mod.rs +++ b/crates/nostream-cli/src/commands/mod.rs @@ -7,6 +7,7 @@ pub mod export; pub mod import; pub mod info; pub mod invite; +pub mod migrate; pub mod seed; pub mod setup; pub mod start; diff --git a/crates/nostream-cli/src/error.rs b/crates/nostream-cli/src/error.rs index 3263dcb3..96a5826a 100644 --- a/crates/nostream-cli/src/error.rs +++ b/crates/nostream-cli/src/error.rs @@ -16,4 +16,9 @@ pub enum CliError { /// The import could not run: a bad argument, a missing file or a storage failure. #[error("{0}")] Import(String), + + /// The migration could not run: a refused row, an unusable manifest or a store that would + /// not open. + #[error("{0}")] + Migrate(String), } diff --git a/crates/nostream-cli/src/main.rs b/crates/nostream-cli/src/main.rs index c04af460..07f2cec6 100644 --- a/crates/nostream-cli/src/main.rs +++ b/crates/nostream-cli/src/main.rs @@ -49,6 +49,11 @@ enum Command { Info, /// Mint and manage invite codes. Invite, + /// Copy the Postgres store into an LMDB store. + Migrate { + #[command(subcommand)] + command: MigrateCommand, + }, /// Seed the database. Seed, /// Run the first-time setup flow. @@ -87,6 +92,33 @@ enum ConfigCommand { }, } +#[derive(Debug, Subcommand)] +enum MigrateCommand { + /// Copy every table from Postgres into an LMDB store. + /// + /// The source is Postgres from `DB_URI`, else the `DB_*` parts. The target is a store root: + /// the environment is `/lmdb` and the manifest `/migration-manifest.json`. + /// + /// Nothing may write to either store while a run is in progress, so + /// the relay must be stopped on both before the first batch is read. + PgToLmdb { + /// The store root (the environment lives under `/lmdb`). + #[arg(long, value_name = "DIR")] + target: PathBuf, + /// Rows per batch, and the page size verify reads with. + #[arg( + long, + value_name = "N", + default_value_t = 1_000, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..) + )] + batch_size: usize, + /// Verify the store against its manifest and copy nothing. + #[arg(long)] + verify: bool, + }, +} + fn main() -> ExitCode { let cli = Cli::parse(); let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); @@ -109,6 +141,21 @@ fn main() -> ExitCode { } Command::Info => commands::info::run().map(|_| CommandOutput::default()), Command::Invite => commands::invite::run().map(|_| CommandOutput::default()), + Command::Migrate { command } => match command { + MigrateCommand::PgToLmdb { + target, + batch_size, + verify, + } => { + let options = commands::migrate::MigrateOptions { + target, + batch_size, + verify, + }; + let mut stdout = std::io::stdout().lock(); + commands::migrate::run(options, &mut stdout) + } + }, Command::Seed => commands::seed::run().map(|_| CommandOutput::default()), Command::Setup => commands::setup::run().map(|_| CommandOutput::default()), Command::Start => commands::start::run().map(|_| CommandOutput::default()), @@ -136,5 +183,9 @@ fn main() -> ExitCode { eprintln!("Import failed: {message}"); ExitCode::from(1) } + Err(CliError::Migrate(message)) => { + eprintln!("Migration failed: {message}"); + ExitCode::from(1) + } } } From 94edd8ff5614c153a020a10767a1d905a6fe0a23 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:06 -0500 Subject: [PATCH 05/49] test(cli): pin the migrate command's copy, resume, verify and throughput --- crates/nostream-cli/tests/migrate_cli.rs | 1086 ++++++++++++++++++++++ 1 file changed, 1086 insertions(+) create mode 100644 crates/nostream-cli/tests/migrate_cli.rs diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs new file mode 100644 index 00000000..b04ec2dd --- /dev/null +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -0,0 +1,1086 @@ +//! The `migrate pg-to-lmdb` command's contract: the copy, the manifest, resume, verify and the +//! run's own throughput measurement. +//! +//! Every test drives `migrate_with`/`verify_with` over a constructed source and a real +//! `LmdbBackend` on a `TempDir`, so no database and no process is involved; the one test that +//! needs a process runs the binary and reads its stderr. +//! +//! These targets may unwrap and assert by panicking: a failed assertion is a failed test. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::process::Command; +use std::sync::Mutex; +use std::time::Instant; + +use chrono::{DateTime, TimeZone, Utc}; +use heed::types::Bytes; +use serde_json::{Value, json}; + +use nostream_cli::commands::migrate::{self, MigrateOptions}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::StorageError; +use nostream_storage::migration::{ + MigratedTable, MigratedUser, MigrationRow, MigrationSource, MigrationTarget, +}; +use nostream_storage::repositories::{ + DeliveryLogEntry, DeliveryLogFilters, DvmJob, DvmJobRepository, DvmJobStatus, EventRecord, + EventRepository, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, + InvoiceUnit, NewReport, Nip05Verification, Nip05VerificationRepository, + NotificationDeliveryLogRepository, NotificationDeliveryStatus, NotificationTargetType, + OutboxMessage, OutboxStatus, Report, ReportRepository, ReportType, StorageFuture, User, + UserRepository, collect_records, +}; + +// --------------------------------------------------------------------------------------- +// Fixtures. +// --------------------------------------------------------------------------------------- + +/// One whole-second instant, 2020-01-01T00:00:00Z plus `offset` seconds, so every fixture +/// timestamp is at the microsecond precision the store keeps. +fn at(offset: i64) -> DateTime { + Utc.timestamp_opt(1_577_836_800 + offset, 0) + .single() + .expect("a whole second") +} + +fn instant(text: &str) -> DateTime { + DateTime::parse_from_rfc3339(text) + .expect("a fixture instant") + .with_timezone(&Utc) +} + +fn event_id(seed: u8) -> EventId { + EventId::from_byte_array([seed; 32]) +} + +fn pubkey(seed: u8) -> Pubkey { + Pubkey::from_byte_array([seed; 32]) +} + +/// A live, unexpiring text note. +fn event_record(seed: u8, content: &str) -> EventRecord { + EventRecord { + row_id: None, + event: Event { + id: event_id(seed), + pubkey: pubkey(1), + created_at: 1_600_000_000 + i64::from(seed), + kind: Kind::TEXT_NOTE, + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([9; 64]), + }, + first_seen: Some(at(0)), + deleted_at: None, + expires_at: None, + event_deduplication: None, + remote_address: None, + } +} + +fn dvm_job(seed: u8) -> DvmJob { + DvmJob { + id: event_id(seed), + requester_pubkey: pubkey(2), + kind: Kind(5100), + worker_index: Some(1), + status: DvmJobStatus::Completed, + result_event_id: Some(event_id(200 + seed)), + error: None, + picked_up_at: Some(at(10)), + completed_at: Some(at(20)), + created_at: at(i64::from(seed)), + updated_at: at(30), + } +} + +fn invite(seed: u8) -> InviteCode { + InviteCode { + code: format!("code-{seed}"), + created_by: Some(pubkey(3)), + claimed_by: None, + expires_at: Some(at(100)), + remaining_uses: Some(2), + created_at: at(i64::from(seed)), + updated_at: at(40), + } +} + +/// The third invoice is the pending one, which is the row the pending index has to hold. +fn invoice(seed: u8) -> Invoice { + let pending = seed == 3; + Invoice { + id: format!("00000000-0000-4000-8000-00000000000{seed}"), + pubkey: pubkey(4), + bolt11: format!("lnbc1invoice{seed}"), + amount_requested: 1_000, + amount_paid: (!pending).then_some(1_000), + unit: InvoiceUnit::Msats, + status: if pending { + InvoiceStatus::Pending + } else { + InvoiceStatus::Completed + }, + description: "an invoice".to_owned(), + confirmed_at: (!pending).then(|| at(50)), + expires_at: Some(at(60)), + updated_at: at(70), + created_at: at(i64::from(seed)), + verify_url: None, + } +} + +fn nip05(seed: u8) -> Nip05Verification { + Nip05Verification { + pubkey: pubkey(20 + seed), + nip05: format!("user{seed}@example.com"), + domain: "example.com".to_owned(), + is_verified: true, + last_verified_at: Some(at(80)), + last_checked_at: at(90), + failure_count: 1, + created_at: at(i64::from(seed)), + updated_at: at(100), + } +} + +fn outbox_id(seed: u8) -> String { + format!("10000000-0000-4000-8000-00000000000{seed}") +} + +fn outbox(seed: u8) -> OutboxMessage { + OutboxMessage { + id: outbox_id(seed), + event_type: "invoice.paid".to_owned(), + payload: json!({ "amount": 1_000 }), + status: OutboxStatus::Delivered, + attempt_count: 1, + available_at: at(120), + last_error: None, + delivered_at: Some(at(130)), + created_at: at(i64::from(seed)), + updated_at: at(140), + } +} + +fn delivery_log(seed: u8) -> DeliveryLogEntry { + DeliveryLogEntry { + id: format!("20000000-0000-4000-8000-00000000000{seed}"), + outbox_id: Some(outbox_id(seed)), + event_type: "invoice.paid".to_owned(), + target_id: "telegram:7175022".to_owned(), + target_type: NotificationTargetType::Telegram, + status: NotificationDeliveryStatus::Success, + attempt_number: 1, + error_snippet: None, + created_at: at(110 + i64::from(seed)), + } +} + +fn report(seed: u8) -> Report { + Report { + id: i32::from(seed), + event_id: event_id(40 + seed), + reporter_pubkey: pubkey(5), + reported_pubkey: Some(pubkey(6)), + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.5, + actionable: true, + created_at: at(i64::from(seed)), + } +} + +fn user(seed: u8) -> MigratedUser { + MigratedUser { + user: User { + pubkey: pubkey(30 + seed), + is_admitted: true, + is_vanished: false, + balance: 0, + created_at: at(i64::from(seed)), + updated_at: at(150), + }, + tos_accepted_at: None, + } +} + +/// Three rows per table, which is the source every whole-run test copies. +fn source_rows() -> Vec { + let mut rows = Vec::new(); + for seed in 1..=3 { + rows.push(MigrationRow::DvmJob(dvm_job(seed))); + rows.push(MigrationRow::Event(event_record(seed, "hello"))); + rows.push(MigrationRow::InviteCode(invite(seed))); + rows.push(MigrationRow::Invoice(invoice(seed))); + rows.push(MigrationRow::Nip05Verification(nip05(seed))); + rows.push(MigrationRow::DeliveryLogEntry(delivery_log(seed))); + rows.push(MigrationRow::OutboxMessage(outbox(seed))); + rows.push(MigrationRow::Report(report(seed))); + rows.push(MigrationRow::User(user(seed))); + } + rows +} + +fn rows_of(rows: &[MigrationRow], table: MigratedTable) -> Vec { + let mut selected: Vec = rows + .iter() + .filter(|row| row.table() == table) + .cloned() + .collect(); + selected.sort_by_key(MigrationRow::key); + selected +} + +// --------------------------------------------------------------------------------------- +// The injected source. +// --------------------------------------------------------------------------------------- + +/// A source that serves a fixed row set, in key order, and records every call it is asked for. +struct StubSource { + rows: Vec, + /// The ordinal of the call that answers `Err`, if any. + fail_at: Option, + calls: Mutex>)>>, +} + +impl StubSource { + fn new(rows: Vec) -> Self { + Self { + rows, + fail_at: None, + calls: Mutex::new(Vec::new()), + } + } + + fn failing_at(rows: Vec, ordinal: usize) -> Self { + Self { + fail_at: Some(ordinal), + ..Self::new(rows) + } + } + + fn calls(&self) -> Vec<(MigratedTable, Option>)> { + self.calls + .lock() + .expect("the counter is not poisoned") + .clone() + } +} + +impl MigrationSource for StubSource { + fn read_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let ordinal = { + let mut calls = self.calls.lock().expect("the counter is not poisoned"); + calls.push((table, after.map(<[u8]>::to_vec))); + calls.len() - 1 + }; + if self.fail_at == Some(ordinal) { + return Err(StorageError::InvalidArgument { + message: "the stub's read failed", + }); + } + + let mut rows = rows_of(&self.rows, table); + if let Some(after) = after { + rows.retain(|row| row.key().as_slice() > after); + } + rows.truncate(limit); + Ok(rows) + }) + } +} + +/// A target a verify must never touch: every method panics. +struct UntouchedTarget; + +impl MigrationTarget for UntouchedTarget { + fn write_batch<'a>(&'a self, _rows: &'a [MigrationRow]) -> StorageFuture<'a, u64> { + panic!("verify must not write to the store") + } + + fn scan_batch<'a>( + &'a self, + _table: MigratedTable, + _after: Option<&'a [u8]>, + _limit: usize, + ) -> StorageFuture<'a, Vec> { + panic!("verify must answer a missing manifest before it scans") + } +} + +// --------------------------------------------------------------------------------------- +// Helpers. +// --------------------------------------------------------------------------------------- + +fn options(directory: &std::path::Path, batch_size: usize) -> MigrateOptions { + MigrateOptions { + target: directory.to_owned(), + batch_size, + verify: false, + } +} + +fn manifest_bytes(directory: &std::path::Path) -> Vec { + std::fs::read(directory.join("migration-manifest.json")).expect("the manifest is written") +} + +fn manifest(directory: &std::path::Path) -> Value { + serde_json::from_slice(&manifest_bytes(directory)).expect("the manifest is JSON") +} + +fn output_text(out: Vec) -> String { + String::from_utf8(out).expect("the command writes utf-8") +} + +/// `time_key`: the flip-signed seconds, then the event id — the shape the two event tail +/// indexes are keyed by. +fn time_key(seconds: i64, id: EventId) -> Vec { + let mut key = ((seconds as u64) ^ (1 << 63)).to_be_bytes().to_vec(); + key.extend_from_slice(id.as_bytes()); + key +} + +/// Every key in one named database, through the public environment handle. +fn database_keys(backend: &LmdbBackend, name: &str) -> Vec> { + let txn = backend.env().read_txn().expect("a read transaction"); + let database = backend + .env() + .open_database::(&txn, Some(name)) + .expect("the database opens") + .expect("the database exists"); + database + .iter(&txn) + .expect("the database iterates") + .map(|entry| entry.expect("a stored entry").0.to_vec()) + .collect() +} + +/// Pages one table of the store to its end. +async fn scan_all(target: &dyn MigrationTarget, table: MigratedTable) -> Vec { + let mut rows = Vec::new(); + let mut after: Option> = None; + loop { + let page = target + .scan_batch(table, after.as_deref(), 100) + .await + .expect("the store scans"); + let count = page.len(); + if let Some(last) = page.last() { + after = Some(last.key()); + } + rows.extend(page); + if count < 100 { + return rows; + } + } +} + +// --------------------------------------------------------------------------------------- +// AC2: the whole copy, and a second run that verifies. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_clean_run_copies_every_table_and_a_second_run_writes_nothing() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let source = StubSource::new(source_rows()); + let options = options(directory.path(), 1_000); + + let mut out = Vec::new(); + let output = migrate::migrate_with(&source, &target, &options, &mut out) + .await + .expect("the copy runs"); + assert_eq!(output.exit_code, 0); + + let written = manifest(directory.path()); + assert_eq!(written["format"], 1); + assert_eq!( + written["not_migrated"], + json!(["event_tags", "user_subscriptions"]) + ); + let tables = written["tables"] + .as_object() + .expect("the manifest names tables"); + assert_eq!(tables.len(), 9, "one entry per migrated table"); + for table in MigratedTable::ALL { + let entry = &tables[table.name()]; + assert_eq!(entry["rows"], 3, "{} rows", table.name()); + assert_eq!(entry["complete"], true, "{} is complete", table.name()); + } + + // Each table scans to exactly the source's rows. + for table in MigratedTable::ALL { + assert_eq!( + scan_all(&target, table).await, + rows_of(&source_rows(), table), + "{} scans to the source's rows", + table.name() + ); + } + + // One row per table read back through that table's own read. + let filters = [Filter { + limit: Some(10), + ..Filter::default() + }]; + let events = collect_records( + EventRepository::find_by_filters(&target, &filters) + .await + .expect("the filter read opens"), + ) + .await + .expect("the filter read drains"); + assert_eq!(events.len(), 3); + for seed in 1..=3 { + assert!( + events.contains(&event_record(seed, "hello")), + "event {seed} reads back" + ); + } + + assert_eq!( + UserRepository::find_by_pubkey(&target, &pubkey(31)) + .await + .expect("the user reads") + .expect("the user is stored"), + user(1).user + ); + assert_eq!( + InvoiceRepository::find_by_id(&target, &invoice(1).id) + .await + .expect("the invoice reads") + .expect("the invoice is stored"), + invoice(1) + ); + assert_eq!( + InvoiceRepository::find_pending_invoices(&target, 0, 10) + .await + .expect("the pending index reads"), + vec![invoice(3)] + ); + assert_eq!( + Nip05VerificationRepository::find_by_pubkey(&target, &nip05(1).pubkey) + .await + .expect("the verification reads"), + Some(nip05(1)) + ); + assert_eq!( + InviteCodeRepository::find_by_code(&target, &invite(1).code) + .await + .expect("the code reads"), + Some(invite(1)) + ); + assert_eq!( + DvmJobRepository::find_by_id(&target, &dvm_job(1).id) + .await + .expect("the job reads"), + Some(dvm_job(1)) + ); + assert_eq!( + ReportRepository::find_by_event_id(&target, &report(1).event_id) + .await + .expect("the report reads"), + vec![report(1)] + ); + let mut logged: Vec = + NotificationDeliveryLogRepository::find_recent(&target, 10, &DeliveryLogFilters::default()) + .await + .expect("the log reads") + .into_iter() + .map(|entry| entry.id) + .collect(); + logged.sort(); + assert_eq!( + logged, + vec![delivery_log(1).id, delivery_log(2).id, delivery_log(3).id] + ); + + // The second run: a source that would answer rows and counts its calls. + let counting = StubSource::new(Vec::new()); + let before = manifest_bytes(directory.path()); + let usage = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + let mut out = Vec::new(); + let output = migrate::migrate_with(&counting, &target, &options, &mut out) + .await + .expect("the second run succeeds"); + assert_eq!(output.exit_code, 0); + assert_eq!( + counting.calls().len(), + 0, + "the second run never asked the source for a row" + ); + assert_eq!( + manifest_bytes(directory.path()), + before, + "the manifest is unchanged" + ); + for table in MigratedTable::ALL { + assert_eq!( + scan_all(&target, table).await.len(), + 3, + "{} still holds three rows", + table.name() + ); + } + assert_eq!( + target + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + usage, + "the store was not written" + ); + assert!( + output_text(out).contains("Verified 9 tables"), + "the second run verified rather than short-circuiting" + ); + + // The one read that writes is made after that second run, because it would move a checksum. + let created = ReportRepository::create( + &target, + &NewReport { + event_id: event_id(1), + reporter_pubkey: pubkey(2), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }, + ) + .await + .expect("a report is created"); + assert_eq!(created.id, 4, "the migrated ids raised the sequence"); +} + +// --------------------------------------------------------------------------------------- +// AC3: resume. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn an_interrupted_run_resumes_to_the_clean_run_manifest() { + // The clean run over source S at --batch-size 1. + let clean_directory = tempfile::tempdir().expect("a temporary directory"); + let clean_target = LmdbBackend::open(clean_directory.path()) + .await + .expect("the store opens"); + let clean_options = options(clean_directory.path(), 1); + migrate::migrate_with( + &StubSource::new(source_rows()), + &clean_target, + &clean_options, + &mut Vec::new(), + ) + .await + .expect("the clean run succeeds"); + let clean = manifest_bytes(clean_directory.path()); + + // The interrupted run: the second read_batch answers Err. + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let options = options(directory.path(), 1); + let failing = StubSource::failing_at(source_rows(), 1); + let error = migrate::migrate_with(&failing, &target, &options, &mut Vec::new()) + .await + .expect_err("the second read fails"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a migration failure is the variant `main` prints as `Migration failed:` and exits 1 for" + ); + assert_eq!( + error.to_string(), + "dvm_jobs: invalid argument: the stub's read failed" + ); + + let interrupted = manifest(directory.path()); + let dvm = &interrupted["tables"]["dvm_jobs"]; + assert_eq!(dvm["rows"], 1); + assert_eq!(dvm["complete"], false); + let first_key = dvm_job(1).id.as_bytes().to_vec(); + assert_eq!(dvm["after"], hex::encode(&first_key)); + + // The healthy source over the same target, at --batch-size 1 too. + let resumed = StubSource::new(source_rows()); + let mut out = Vec::new(); + let output = migrate::migrate_with(&resumed, &target, &options, &mut out) + .await + .expect("the resumed run succeeds"); + let calls = resumed.calls(); + + // 1. The run measured the resume rather than restarting the table. + assert_eq!(calls[0].0, MigratedTable::DvmJobs); + assert_eq!( + calls[0].1, + Some(first_key), + "the first call resumes at the manifest's cursor" + ); + // 2. The cursor advances per batch. + assert_eq!(calls[1].0, MigratedTable::DvmJobs); + assert_eq!(calls[1].1, Some(dvm_job(2).id.as_bytes().to_vec())); + + // 3. The resumed run lands on the clean run's manifest, byte for byte. + assert_eq!(output.exit_code, 0); + assert_eq!(manifest_bytes(directory.path()), clean); +} + +// --------------------------------------------------------------------------------------- +// AC4: one changed column moves only its table's checksum. +// --------------------------------------------------------------------------------------- + +/// Copies `rows` into a fresh store and answers its manifest. +async fn manifest_of(rows: Vec) -> Value { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::new(rows), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + manifest(directory.path()) +} + +#[tokio::test] +async fn one_changed_source_row_changes_only_its_table_checksum() { + let clean = manifest_of(source_rows()).await; + + let mut richer = source_rows(); + for row in &mut richer { + if let MigrationRow::User(migrated) = row + && migrated.user.pubkey == user(1).user.pubkey + { + migrated.user.balance = 999; + } + } + let balance = manifest_of(richer).await; + + let mut accepted = source_rows(); + for row in &mut accepted { + if let MigrationRow::User(migrated) = row + && migrated.user.pubkey == user(1).user.pubkey + { + migrated.tos_accepted_at = Some(at(7)); + } + } + let tos = manifest_of(accepted).await; + + for variant in [&balance, &tos] { + assert_eq!( + variant["tables"]["users"]["rows"], + clean["tables"]["users"]["rows"] + ); + assert_ne!( + variant["tables"]["users"]["checksum"], + clean["tables"]["users"]["checksum"] + ); + for table in MigratedTable::ALL { + if table == MigratedTable::Users { + continue; + } + assert_eq!( + variant["tables"][table.name()]["checksum"], + clean["tables"][table.name()]["checksum"], + "{} is unchanged", + table.name() + ); + } + } +} + +// --------------------------------------------------------------------------------------- +// AC5: verify pages the whole table. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn verify_accepts_the_runs_own_target_and_rejects_a_changed_row() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let rows: Vec = (1..=5) + .map(|seed| MigrationRow::Event(event_record(seed, "hello"))) + .collect(); + // Five events at --batch-size 2 is three pages, so a verify that reads one page sees two. + let options = options(directory.path(), 2); + migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut Vec::new()) + .await + .expect("the copy succeeds"); + + let before = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + let mut out = Vec::new(); + let output = migrate::verify_with(&target, &options, &mut out) + .await + .expect("the verify runs"); + assert_eq!(output.exit_code, 0); + assert!(output_text(out).contains("Verified 9 tables")); + + // A same-length content change under the same key: the count cannot move. + let mut changed = event_record(1, "hello"); + changed.event.content = "jello".to_owned(); + target + .write_batch(&[MigrationRow::Event(changed)]) + .await + .expect("the changed row is written"); + + let mut out = Vec::new(); + let output = migrate::verify_with(&target, &options, &mut out) + .await + .expect("the verify runs"); + assert_eq!(output.exit_code, 1); + let text = output_text(out); + let line = text + .lines() + .find(|line| line.starts_with("[events] checksum mismatch (manifest ")) + .unwrap_or_else(|| panic!("a checksum line is printed, got:\n{text}")); + let body = line + .strip_prefix("[events] checksum mismatch (manifest ") + .expect("the line's prefix") + .trim_end_matches(')'); + let (manifest_hex, target_hex) = body.split_once(", target ").expect("both checksums"); + assert_eq!(manifest_hex.len(), 64, "the manifest's checksum: {line}"); + assert_eq!(target_hex.len(), 64, "the target's checksum: {line}"); + + assert_eq!( + target + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + before, + "verify wrote nothing to the store" + ); +} + +// --------------------------------------------------------------------------------------- +// AC6: no manifest. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn verify_rejects_a_missing_manifest_with_a_message_and_exit_1() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let options = MigrateOptions { + verify: true, + ..options(directory.path(), 1_000) + }; + + let mut out = Vec::new(); + let output = migrate::verify_with(&UntouchedTarget, &options, &mut out) + .await + .expect("a missing manifest is an answer, not a failure"); + assert_eq!(output.exit_code, 1); + + let text = output_text(out); + assert!(text.contains("No manifest at"), "got: {text}"); + assert!( + text.contains( + &directory + .path() + .join("migration-manifest.json") + .display() + .to_string() + ), + "the line names the path it looked for: {text}" + ); + assert_eq!( + std::fs::read_dir(directory.path()) + .expect("the directory reads") + .count(), + 0, + "verify created nothing" + ); +} + +// --------------------------------------------------------------------------------------- +// AC7: the run's own measurement. +// --------------------------------------------------------------------------------------- + +/// Copies `count` events and answers the two summary numbers, the test's own measurement, and +/// the row total. +async fn measured_copy(count: usize) -> (u64, u64, u64, u128) { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let rows: Vec = (0..count) + .map(|index| { + let mut record = event_record(1, "hello"); + let mut id = [0u8; 32]; + id[..8].copy_from_slice(&(index as u64).to_be_bytes()); + record.event.id = EventId::from_byte_array(id); + MigrationRow::Event(record) + }) + .collect(); + let options = options(directory.path(), 1_000); + + let mut out = Vec::new(); + let started = Instant::now(); + migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut out) + .await + .expect("the copy succeeds"); + let measured = started.elapsed().as_millis(); + + let text = output_text(out); + let mut lines = text.lines(); + let first = lines.next().expect("the rows line"); + let second = lines.next().expect("the throughput line"); + assert_eq!(lines.next(), None, "exactly two summary lines"); + + let (rows_text, rest) = first + .strip_prefix("Migrated ") + .and_then(|rest| rest.split_once(" rows in ")) + .expect("the rows line's shape"); + let ms_text = rest.strip_suffix(" ms").expect("the milliseconds"); + let rate_text = second + .strip_prefix("Events throughput: ") + .and_then(|rest| rest.strip_suffix(" events/s")) + .expect("the throughput line's shape"); + + ( + rows_text.parse().expect("a row count"), + ms_text.parse().expect("a millisecond count"), + rate_text.parse().expect("a rate"), + measured, + ) +} + +#[tokio::test] +async fn the_run_prints_a_rate_consistent_with_its_own_events_and_milliseconds() { + let (rows, ms, rate, measured) = measured_copy(1_000).await; + assert_eq!(rows, 1_000); + assert!(ms > 0, "the run took a measurable time"); + let difference = i128::from(ms) - i128::try_from(measured).expect("a bounded duration"); + assert!( + difference.abs() <= 20, + "the printed {ms} ms is the run's own clock, {measured} ms measured outside it" + ); + assert_eq!( + rate, + 1_000 * 1_000 / ms.max(1), + "the rate is the printed numbers'" + ); + + let (rows, longer, _, _) = measured_copy(10_000).await; + assert_eq!(rows, 10_000); + assert!( + longer > ms, + "ten times the work takes longer: {longer} ms against {ms} ms" + ); +} + +// --------------------------------------------------------------------------------------- +// AC8: a migrated event keeps what the traits would lose. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_migrated_event_keeps_first_seen_deduplication_and_its_deleted_state() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut live = event_record(1, "live"); + live.first_seen = Some(instant("2020-01-01T00:00:00Z")); + live.remote_address = Some("203.0.113.7".parse().expect("a fixture address")); + live.event_deduplication = Some(vec![json!("abc")]); + live.expires_at = Some(1_893_456_000); + + let mut deleted = event_record(2, "deleted"); + deleted.deleted_at = Some(at(500)); + + let mut expired = event_record(3, "expired"); + expired.expires_at = Some(1_577_836_800); + + let rows = vec![ + MigrationRow::Event(live.clone()), + MigrationRow::Event(deleted.clone()), + MigrationRow::Event(expired.clone()), + ]; + migrate::migrate_with( + &StubSource::new(rows.clone()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + + assert_eq!( + scan_all(&target, MigratedTable::Events).await, + rows_of(&rows, MigratedTable::Events) + ); + + // The store's filter read carries the rows' own state: `page` builds its page with + // `live: false`, so a soft-deleted or expired row comes back rather than disappearing, and + // `count_by_filters` is the read that applies the live predicate (ADR 0011 D4, pinned by + // `parity_lmdb::a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count`). + let filters = [Filter { + limit: Some(10), + ..Filter::default() + }]; + let read = collect_records( + EventRepository::find_by_filters(&target, &filters) + .await + .expect("the filter read opens"), + ) + .await + .expect("the filter read drains"); + assert_eq!(read.len(), 3, "every stored row reads back by filter"); + let live_read = read + .iter() + .find(|record| record.event.id == live.event.id) + .expect("the live row reads back"); + assert_eq!(live_read.first_seen, live.first_seen); + assert_eq!(live_read.remote_address, live.remote_address); + assert_eq!(live_read.event_deduplication, live.event_deduplication); + assert_eq!(live_read.expires_at, live.expires_at); + let deleted_read = read + .iter() + .find(|record| record.event.id == deleted.event.id) + .expect("the soft-deleted row reads back"); + assert_eq!(deleted_read.deleted_at, deleted.deleted_at); + let expired_read = read + .iter() + .find(|record| record.event.id == expired.event.id) + .expect("the expired row reads back"); + assert_eq!(expired_read.expires_at, expired.expires_at); + assert_eq!( + EventRepository::count_by_filters(&target, &filters) + .await + .expect("the count runs"), + 1, + "the live predicate holds for the count, so only the live row is counted" + ); + + // Both tail indexes are stated separately: a value-only copy passes every read above. + let deleted_at = deleted.deleted_at.expect("the fixture is deleted"); + assert_eq!( + database_keys(&target, "events_deleted"), + vec![time_key(deleted_at.timestamp(), deleted.event.id)], + "one entry, keyed by the deleted instant in whole seconds" + ); + + let mut expiring = database_keys(&target, "events_expiring"); + let mut expected = vec![ + time_key( + live.expires_at.expect("the live row expires"), + live.event.id, + ), + time_key( + expired.expires_at.expect("the expired row expires"), + expired.event.id, + ), + ]; + expiring.sort(); + expected.sort(); + assert_eq!(expiring, expected); +} + +// --------------------------------------------------------------------------------------- +// AC9: a refused row stops the run. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_refused_row_stops_the_run_and_keeps_the_resume_point() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + /// A tag whose key is past LMDB's 511-byte bound: `2 + 601 + 8 + 32` bytes. + fn refused(seed: u8) -> EventRecord { + let mut record = event_record(seed, "refused"); + record.event.tags = vec![vec!["t".to_owned(), "a".repeat(600)]]; + record + } + + let rows: Vec = (1..=2) + .map(|seed| MigrationRow::Event(event_record(seed, "fine"))) + .chain(std::iter::once(MigrationRow::Event(refused(3)))) + .collect(); + // The refused row is in the second batch of two. + let options = options(directory.path(), 2); + let error = migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut Vec::new()) + .await + .expect_err("the refused key fails the batch"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a migration failure is the variant `main` prints as `Migration failed:` and exits 1 for" + ); + assert!( + error.to_string().starts_with("events: "), + "the message names the table whose write failed, not a batch ordinal: {error}" + ); + + let written = manifest(directory.path()); + let events = &written["tables"]["events"]; + assert_eq!(events["complete"], false); + assert_eq!(events["rows"], 2); + assert_eq!( + events["after"], + hex::encode(event_record(2, "fine").event.id.as_bytes()) + ); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 2, + "the first batch is the store's content" + ); + + // The resume point is usable: a healthy source continues where the refused row stopped. + let resumed = StubSource::new(vec![MigrationRow::Event(event_record(3, "fine"))]); + let output = migrate::migrate_with(&resumed, &target, &options, &mut Vec::new()) + .await + .expect("the run resumes after the refusal"); + assert_eq!(output.exit_code, 0); + assert_eq!(resumed.calls()[0].0, MigratedTable::Events); + assert_eq!( + resumed.calls()[0].1, + Some(event_record(2, "fine").event.id.as_bytes().to_vec()) + ); +} + +/// The process boundary the two failure tests above stop at: `main` prints +/// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on +/// the way to the copy, which is the same arm. +#[test] +fn a_failed_run_reports_migration_failed_and_exits_1() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env("DB_URI", "postgres://127.0.0.1:1/nostream") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.starts_with("Migration failed: "), + "the failure is reported as one: {stderr}" + ); +} From fecfdbf667f5b8befac5f5d280924dceb2d32c8f Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:06 -0500 Subject: [PATCH 06/49] test(storage): pin the seam's canonical projection and key bytes --- .../nostream-storage/tests/migration_seam.rs | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 crates/nostream-storage/tests/migration_seam.rs diff --git a/crates/nostream-storage/tests/migration_seam.rs b/crates/nostream-storage/tests/migration_seam.rs new file mode 100644 index 00000000..f1417510 --- /dev/null +++ b/crates/nostream-storage/tests/migration_seam.rs @@ -0,0 +1,130 @@ +//! The migration seam's canonical projection and key bytes, with no driver compiled in. +//! +//! This target is also the proof that the seam builds without either backend: it runs under +//! `cargo test -p nostream-storage --no-default-features --test migration_seam`. +//! +//! These targets may unwrap and assert by panicking: a failed assertion is a failed test. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use chrono::{DateTime, TimeZone, Utc}; +use nostream_core::{Event, EventId, Kind, Pubkey, Signature}; +use nostream_storage::migration::MigrationRow; +use nostream_storage::repositories::{EventRecord, Report, ReportType}; + +fn instant(text: &str) -> DateTime { + DateTime::parse_from_rfc3339(text) + .expect("a fixture instant") + .with_timezone(&Utc) +} + +/// An event record with the given row id, content and `first_seen`. +fn record(row_id: Option<&str>, content: &str, first_seen: DateTime) -> EventRecord { + EventRecord { + row_id: row_id.map(str::to_owned), + event: Event { + id: EventId::from_byte_array([7; 32]), + pubkey: Pubkey::from_byte_array([1; 32]), + created_at: 1_577_836_800, + kind: Kind::TEXT_NOTE, + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([9; 64]), + }, + first_seen: Some(first_seen), + deleted_at: None, + expires_at: None, + event_deduplication: None, + remote_address: None, + } +} + +fn canonical(row: &MigrationRow) -> Vec { + row.canonical().expect("a row serialises") +} + +#[test] +fn the_canonical_projection_drops_the_row_id_and_truncates_instants() { + let epoch = instant("2020-01-01T00:00:00Z"); + + // `events.id` is the one column the store does not keep, so it is not in the bytes. + let with_row_id = canonical(&MigrationRow::Event(record( + Some("6f1e7c1a-0000-4000-8000-000000000000"), + "a", + epoch, + ))); + assert_eq!( + with_row_id, + canonical(&MigrationRow::Event(record(None, "a", epoch))) + ); + assert!(!with_row_id.is_empty()); + + // Every instant is truncated to the microsecond precision a `timestamp` column has. + let nanos = Utc + .timestamp_opt(1_577_836_800, 123_456_789) + .single() + .expect("a nanosecond instant"); + let micros = Utc + .timestamp_opt(1_577_836_800, 123_456_000) + .single() + .expect("the same instant at microseconds"); + assert_eq!( + canonical(&MigrationRow::Event(record(None, "a", nanos))), + canonical(&MigrationRow::Event(record(None, "a", micros))) + ); + + // A projection that returns one constant fails both of these. + assert_ne!( + canonical(&MigrationRow::Event(record(None, "a", epoch))), + canonical(&MigrationRow::Event(record(None, "b", epoch))), + "one wire field apart is different bytes" + ); + // `timestamp_opt`'s second argument is nanoseconds, so one microsecond is a thousand. + let one_microsecond_later = Utc + .timestamp_opt(1_577_836_800, 1_000) + .single() + .expect("one microsecond later"); + assert_ne!( + canonical(&MigrationRow::Event(record(None, "a", epoch))), + canonical(&MigrationRow::Event(record( + None, + "a", + one_microsecond_later + ))), + "a projection that truncates to whole seconds fails here" + ); + + // The whole literal, so an empty, truncated or re-ordered projection fails. + let report = MigrationRow::Report(Report { + id: 1, + event_id: EventId::from_byte_array([0; 32]), + reporter_pubkey: Pubkey::from_byte_array([0; 32]), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + created_at: epoch, + }); + let zeros = "0".repeat(64); + let expected = format!( + concat!( + "{{\"Report\":{{\"id\":1,\"event_id\":\"{}\",\"reporter_pubkey\":\"{}\",", + "\"reported_pubkey\":null,\"reported_event_id\":null,\"report_type\":\"spam\",", + "\"weight\":1.0,\"actionable\":true,\"created_at\":\"2020-01-01T00:00:00Z\"}}}}" + ), + zeros, zeros + ); + assert_eq!( + String::from_utf8(canonical(&report)).expect("the projection is utf-8"), + expected + ); + + // The key bytes are the store's: the raw id and the flip-signed `reports.id`. + let event = record(None, "a", epoch); + assert_eq!( + MigrationRow::Event(event.clone()).key(), + event.event.id.as_bytes().to_vec() + ); + assert_eq!(report.key(), vec![0x80, 0x00, 0x00, 0x01]); +} From 7bd1aefb2ce5274bf71756acfa2bfef6d6dbca77 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:06 -0500 Subject: [PATCH 07/49] test(storage): run the migration against the CI Postgres --- .github/workflows/checks.yml | 6 +- crates/nostream-storage/tests/migration_pg.rs | 346 ++++++++++++++++++ 2 files changed, 349 insertions(+), 3 deletions(-) create mode 100644 crates/nostream-storage/tests/migration_pg.rs diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fd73c120..eb47a54d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -273,12 +273,12 @@ jobs: -D clippy::todo -D clippy::unimplemented -D clippy::panic - name: Tests run: cargo test --workspace --all-features - # The two database-backed storage targets create and migrate their own scratch - # databases; the Cucumber half of the harness stays in scripts/storage-parity.sh. + # The database-backed storage targets create and migrate their own scratch databases; the + # Cucumber half of the harness stays in scripts/storage-parity.sh. - name: Storage parity against Postgres env: PARITY_ADMIN_URL: postgres://postgres:postgres@localhost:5432/postgres - run: cargo test -p nostream-storage --test migrations --test parity -- --ignored --test-threads=1 + run: cargo test -p nostream-storage --test migrations --test parity --test migration_pg -- --ignored --test-threads=1 - name: Storage builds without the Postgres driver run: cargo check -p nostream-storage --no-default-features --all-targets - name: Release build diff --git a/crates/nostream-storage/tests/migration_pg.rs b/crates/nostream-storage/tests/migration_pg.rs new file mode 100644 index 00000000..80da78ef --- /dev/null +++ b/crates/nostream-storage/tests/migration_pg.rs @@ -0,0 +1,346 @@ +//! The migration against a real Postgres source. +//! +//! `PARITY_ADMIN_URL` names an administrative database; this target creates, migrates and seeds +//! a scratch one, copies the nine tables into a temporary LMDB store through the seam, and +//! compares them row for row. It is `#[ignore]`d for the repository's own reason — a checkout +//! without a database still has to pass `cargo test --workspace` — and CI's `Rust Checks` job +//! runs it with `--ignored`. +//! +//! Two things a copy has to get right are asserted here and nowhere else: the reader sees every +//! row the source holds, compared against `count(*)` rather than a constant, and the canonical +//! bytes of the rows the source read are the canonical bytes of the rows the store returns, +//! which is what an equal checksum in the manifest means. A second pass over the same source +//! writes the same keys and values, which is what a replayed batch and a resumed run rely on. +//! +//! The CLI's manifest, `--verify` and the throughput lines live in `nostream-cli`, which this +//! crate cannot depend on — the dependency runs the other way — so the manifest-driven half is +//! `crates/nostream-cli/tests/migrate_cli.rs`. + +#![cfg(all(feature = "postgres", feature = "lmdb"))] +// This integration-test target may unwrap: a failed assertion here is a failed test, +// which is the point. The allowance names this file rather than exempting test targets +// workspace-wide, so a new test file still has to ask. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use chrono::{DateTime, NaiveDateTime, Utc}; +use sqlx::postgres::PgPool; +use sqlx::{Executor, Row}; + +use nostream_storage::LmdbBackend; +use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; +use nostream_storage::postgres::PostgresBackend; + +const DATABASE: &str = "nostream_migration_rust"; + +/// The paging the copy runs at here, small enough that a table of three is two pages. +const BATCH: usize = 2; + +fn admin_url() -> String { + std::env::var("PARITY_ADMIN_URL").expect( + "PARITY_ADMIN_URL must name an administrative database, e.g. \ + postgres://postgres:postgres@host:5432/postgres", + ) +} + +fn with_database(url: &str, database: &str) -> String { + let (prefix, query) = match url.split_once('?') { + Some((prefix, query)) => (prefix, Some(query)), + None => (url, None), + }; + let base = match prefix.rfind('/') { + Some(index) if index > "postgres://".len() => &prefix[..index], + _ => prefix, + }; + match query { + Some(query) => format!("{base}/{database}?{query}"), + None => format!("{base}/{database}"), + } +} + +/// A naive instant (a `timestamp` column), 2020-01-01T00:00:00Z plus `offset` seconds. +fn naive(offset: i64) -> NaiveDateTime { + timestamptz(offset).naive_utc() +} + +/// A UTC instant (a `timestamptz` column), 2020-01-01T00:00:00Z plus `offset` seconds. +fn timestamptz(offset: i64) -> DateTime { + DateTime::::from_timestamp(1_577_836_800 + offset, 0).expect("a whole second") +} + +fn canonical(row: &MigrationRow) -> Vec { + row.canonical().expect("a row serialises") +} + +/// Pages one table of the store to its end. +async fn scan_all(target: &LmdbBackend, table: MigratedTable) -> Vec { + let mut rows = Vec::new(); + let mut after: Option> = None; + loop { + let page = MigrationTarget::scan_batch(target, table, after.as_deref(), BATCH) + .await + .expect("the store scans"); + let count = page.len(); + if let Some(last) = page.last() { + after = Some(last.key()); + } + rows.extend(page); + if count < BATCH { + return rows; + } + } +} + +/// Copies one table of the source into the store, page by page, and answers the rows it read. +async fn copy_table( + source: &PostgresBackend, + target: &LmdbBackend, + table: MigratedTable, +) -> Vec { + let mut rows = Vec::new(); + let mut after: Option> = None; + loop { + let batch = MigrationSource::read_batch(source, table, after.as_deref(), BATCH) + .await + .expect("the source reads"); + let count = batch.len(); + if let Some(last) = batch.last() { + after = Some(last.key()); + } + let written = MigrationTarget::write_batch(target, &batch) + .await + .expect("the store writes"); + assert_eq!( + written, + count as u64, + "{} writes every row it reads", + table.name() + ); + rows.extend(batch); + if count < BATCH { + return rows; + } + } +} + +/// `count(*)` for one table, so the reader is compared with the source rather than a constant. +async fn source_count(pool: &PgPool, table: MigratedTable) -> i64 { + let sql = match table { + MigratedTable::DvmJobs => "SELECT count(*) FROM dvm_jobs", + MigratedTable::Events => "SELECT count(*) FROM events", + MigratedTable::InviteCodes => "SELECT count(*) FROM invite_codes", + MigratedTable::Invoices => "SELECT count(*) FROM invoices", + MigratedTable::Nip05Verifications => "SELECT count(*) FROM nip05_verifications", + MigratedTable::NotificationDeliveryLog => "SELECT count(*) FROM notification_delivery_log", + MigratedTable::NotificationOutbox => "SELECT count(*) FROM notification_outbox", + MigratedTable::Reports => "SELECT count(*) FROM reports", + MigratedTable::Users => "SELECT count(*) FROM users", + }; + sqlx::query(sqlx::AssertSqlSafe(sql)) + .fetch_one(pool) + .await + .expect("the source counts") + .try_get::(0) + .expect("a count") +} + +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] +async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { + let admin = PgPool::connect(&admin_url()) + .await + .expect("the administrative database is reachable"); + admin + .execute(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {DATABASE} WITH (FORCE)" + ))) + .await + .expect("the scratch database drops"); + admin + .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {DATABASE}"))) + .await + .expect("the scratch database is created"); + drop(admin); + + let source = PostgresBackend::connect(&with_database(&admin_url(), DATABASE)) + .await + .expect("the scratch database is reachable"); + source.migrate().await.expect("the schema migrates"); + seed(&source).await; + + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + for table in MigratedTable::ALL { + let read = copy_table(&source, &target, table).await; + assert_eq!( + read.len() as i64, + source_count(source.master(), table).await, + "{} reads every row the source holds", + table.name() + ); + + let scanned = scan_all(&target, table).await; + assert_eq!(scanned.len(), read.len(), "{} count", table.name()); + + let mut expected: Vec> = read.iter().map(canonical).collect(); + let mut actual: Vec> = scanned.iter().map(canonical).collect(); + expected.sort(); + actual.sort(); + assert_eq!(expected, actual, "{} canonical bytes", table.name()); + } + + // A second pass writes the same keys and values: the store's content does not move, which is + // what makes a replayed batch and a resumed run harmless. + for table in MigratedTable::ALL { + let again = copy_table(&source, &target, table).await; + assert_eq!( + scan_all(&target, table).await.len(), + again.len(), + "{} holds one row per source row after a second pass", + table.name() + ); + } +} + +/// Two rows in each seeded table, except events (three, one per liveness state) and the two +/// tables the manifest does not move. +async fn seed(backend: &PostgresBackend) { + let pool = backend.master(); + + // events: live, soft-deleted, expired; a tag, a deduplication value and a first_seen. + for (seed, deleted_at, expires_at) in [ + (1u8, None, None), + (2u8, Some(naive(500)), None), + (3u8, None, Some(1_577_836_800i32)), + ] { + sqlx::query( + "INSERT INTO events (event_id, event_pubkey, event_kind, event_created_at, \ + event_content, event_signature, event_tags, first_seen, deleted_at, expires_at, \ + event_deduplication, remote_address) \ + VALUES ($1, $2, 1, $3, 'hello', $4, $5, $6, $7, $8, $9, $10::inet)", + ) + .bind(vec![seed; 32]) + .bind(vec![1u8; 32]) + .bind(1_600_000_000i32 + i32::from(seed)) + .bind(vec![9u8; 64]) + .bind(serde_json::json!([["t", "x"]])) + .bind(naive(0)) + .bind(deleted_at) + .bind(expires_at) + .bind(serde_json::json!(["abc"])) + .bind(None::) + .execute(pool) + .await + .expect("the event inserts"); + } + + for seed in 1..=2u8 { + sqlx::query( + "INSERT INTO users (pubkey, is_admitted, is_vanished, balance, tos_accepted_at, \ + created_at, updated_at) VALUES ($1, true, false, $2, $3, $4, $4)", + ) + .bind(vec![30 + seed; 32]) + .bind(if seed == 1 { 42i64 } else { 0 }) + .bind((seed == 1).then(|| naive(60))) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the user inserts"); + + sqlx::query( + "INSERT INTO invoices (id, pubkey, bolt11, amount_requested, amount_paid, unit, \ + status, description, confirmed_at, expires_at, created_at, updated_at) \ + VALUES ($1::uuid, $2, $3, 1000, $4, 'msats', $5, 'an invoice', $6, $7, $8, $8)", + ) + .bind(format!("00000000-0000-4000-8000-00000000000{seed}")) + .bind(vec![4u8; 32]) + .bind(format!("lnbc1invoice{seed}")) + .bind((seed == 1).then_some(1_000i64)) + .bind(if seed == 1 { "completed" } else { "pending" }) + .bind((seed == 1).then(|| naive(50))) + .bind(naive(60)) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the invoice inserts"); + + sqlx::query( + "INSERT INTO reports (event_id, reporter_pubkey, reported_pubkey, reported_event_id, \ + report_type, weight, actionable, created_at) \ + VALUES ($1, $2, $3, NULL, 'spam', 1.5, true, $4)", + ) + .bind(vec![40 + seed; 32]) + .bind(vec![5u8; 32]) + .bind(vec![6u8; 32]) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the report inserts"); + + sqlx::query( + "INSERT INTO dvm_jobs (id, requester_pubkey, kind, worker_index, status, \ + result_event_id, error, picked_up_at, completed_at, created_at, updated_at) \ + VALUES ($1, $2, 5100, 1, 'completed', $3, NULL, $4, $5, $6, $6)", + ) + .bind(vec![seed; 32]) + .bind(vec![2u8; 32]) + .bind(vec![200 + seed; 32]) + .bind(timestamptz(10)) + .bind(timestamptz(20)) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the job inserts"); + } + + sqlx::query( + "INSERT INTO nip05_verifications (pubkey, nip05, domain, is_verified, last_verified_at, \ + last_checked_at, failure_count, created_at, updated_at) \ + VALUES ($1, 'user@example.com', 'example.com', true, $2, $3, 1, $3, $2)", + ) + .bind(vec![20u8; 32]) + .bind(timestamptz(80)) + .bind(timestamptz(90)) + .execute(pool) + .await + .expect("the verification inserts"); + + sqlx::query( + "INSERT INTO invite_codes (code, created_by, claimed_by, expires_at, remaining_uses, \ + created_at, updated_at) VALUES ('code-1', $1, NULL, $2, 2, $3, $3)", + ) + .bind(vec![3u8; 32]) + .bind(timestamptz(100)) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the invite code inserts"); + + sqlx::query( + "INSERT INTO notification_outbox (id, event_type, payload, status, attempt_count, \ + available_at, last_error, delivered_at, created_at, updated_at) \ + VALUES ($1::uuid, 'invoice.paid', $2, 'delivered', 1, $3, NULL, $4, $3, $4)", + ) + .bind("10000000-0000-4000-8000-000000000001") + .bind(serde_json::json!({ "amount": 1_000 })) + .bind(timestamptz(120)) + .bind(timestamptz(130)) + .execute(pool) + .await + .expect("the outbox row inserts"); + + sqlx::query( + "INSERT INTO notification_delivery_log (id, outbox_id, event_type, target_id, \ + target_type, status, attempt_number, error_snippet, created_at) \ + VALUES ($1::uuid, $2::uuid, 'invoice.paid', 'telegram:7175022', 'telegram', 'success', \ + 1, NULL, $3)", + ) + .bind("20000000-0000-4000-8000-000000000001") + .bind("10000000-0000-4000-8000-000000000001") + .bind(timestamptz(110)) + .execute(pool) + .await + .expect("the delivery-log row inserts"); +} From a1e2b8b9958dd558289aaa07defa0d8303c527fb Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:06 -0500 Subject: [PATCH 08/49] docs: record ADR 0012 for the Postgres-to-LMDB migration --- CLAUDE.md | 2 +- .../decisions/0012-pg-to-lmdb-migration.md | 208 ++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 docs/rewrite/decisions/0012-pg-to-lmdb-migration.md diff --git a/CLAUDE.md b/CLAUDE.md index 034dd0b0..f73437b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,7 +88,7 @@ resolver 3: | `nostream-relay` | connections, wire protocol, handlers, rate limiters, live dispatch, NIP-77 engine (the acceptor lives in `nostreamd`) | | `nostream-services` | payment processors and the invoice lifecycle, operator notifications, the web-of-trust graph, Tor, the scheduler and its workers, event import, NIP-66 monitors (#865) | | `nostream-admin` | admin API and auth | -| `nostream-cli` | operator command line (`nostream-cli config validate`, `nostream-cli import`) | +| `nostream-cli` | operator command line (`nostream-cli config validate`, `nostream-cli import`, `nostream-cli migrate pg-to-lmdb`) | | `nostreamd` | the daemon binary: runtime, telemetry, signals, the services composition root and the worker supervisor | - Every crate manifest repeats `version.workspace = true`, `edition.workspace = true`, diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md new file mode 100644 index 00000000..c781ea5c --- /dev/null +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -0,0 +1,208 @@ +# ADR 0012: The Postgres-to-LMDB migration (job 029) + +- **Status:** accepted (job 029, Phase 2) +- **Deciders:** the rewrite team, on the evidence in job 029's recon and spec +- **Date:** 2026-11-11 + +## Context + +The rewrite's store becomes LMDB, and the relay that is running holds its data in the +Postgres schema ADR 0003 defines. Cutover therefore needs one operation the repository did +not have: a copy of every table from the Postgres store into an LMDB store, resumable after +a crash, with a record of what it moved so the result can be checked before anything is +pointed at it. + +The nine repository traits cannot carry that copy. No trait enumerates a table — every read +is a filtered query — and every write is shaped like "a new row now": `create` stamps +`first_seen` with the clock, the invoice insert clears `amount_paid` and `confirmed_at`, +`UserPatch` carries neither `balance` nor `created_at`, and the outbox and delivery-log +inserts mint fresh ids. A trait-only copy would be both lossy and partial, which is the one +thing a cutover tool must not be. + +This document records the twelve decisions the tool rests on, each with the reason it was +taken. + +## The copy goes through a migration-only seam (D1) + +`nostream_storage::migration` is a new module with two traits — `MigrationSource` and +`MigrationTarget` — and the rows they move. It exists for `nostream-cli migrate pg-to-lmdb` +and for no other caller; its doc comment says so, and a second caller needs a record of its +own first. + +The alternative was to implement the copy in the CLI, and it loses on three counts. The +Postgres row decoders are in-crate, the event column list is private to the crate +(`postgres/sql.rs`), and eight of the nine LMDB per-table write paths already exist in +`lmdb/`. A CLI-side copy would restate nine column lists and nine index write paths and +drift from the ones the relay uses. The seam is also the only place where "write this row +exactly" can sit beside the write paths it has to agree with. + +The seam compiles with no driver: the two implementations are `#[cfg(feature)]` and the +vocabulary is not, so `cargo check -p nostream-storage --no-default-features --all-targets` +still passes and a build that links neither backend can still name a `MigrationRow`. + +The cost is a public surface in a crate that documents what it does not own. The crate's +`lib.rs` says which half it owns — the seam, not the tool — and the doc comment on the +module repeats it where an implementor meets it. + +## Nine tables, two stated limits (D2) + +`MigratedTable` has nine variants: `dvm_jobs`, `events`, `invite_codes`, `invoices`, +`nip05_verifications`, `notification_delivery_log`, `notification_outbox`, `reports`, +`users`. They are the tables the store has a repository and a database for. + +Two of the capture's tables are not moved, and both are stated limits rather than omissions: + +- `event_tags` is a trigger-maintained index in Postgres. LMDB's `tags` database is written + from the event row itself by the event write path, so copying the table would duplicate + what the event write already produces. +- `user_subscriptions` has a TypeScript repository and no Rust trait, and no LMDB database + (ADR 0011). It is that repository's slice to build, not this tool's to invent. + +`MigratedTable::NOT_MIGRATED` carries both names and the manifest prints them, so an +operator reading the result sees the two tables the store does not hold instead of having to +know. + +## Every column is preserved, through migration-only writers (D3) + +Each row goes to LMDB through a writer that puts the source's own key and value and every +index entry that table's existing write path writes, and stamps, clears and defaults +nothing. `events` is the one table whose write set had to be composed: the value, the +created-at index, the author index, the tag index and the expiring index are the set +`write_event` writes, the `replaceable` pointer is what `write_upserted` writes, and an +`events_deleted` entry is what `soft_delete` writes — so a migrated soft-deleted or expired +row, which never passed through `soft_delete`, gets both tail entries from the migration +writer itself. + +`users.tos_accepted_at` is carried, which no trait write can do: the seam's `User` omits the +column deliberately (`fromDBUser` drops it), so the seam carries the pair +`MigratedUser { user, tos_accepted_at }`. + +One field is lost, and it is lost on purpose. `events.id` is the Postgres v4 uuid, no LMDB +reader uses it, and the store does not keep it. It is dropped from the canonical projection +so a source row and the same row read back from the store stay comparable. + +## The canonical projection and its checksum (D4) + +`MigrationRow::canonical` is `serde_json` of the row — so the enum's variant tag is part of +the bytes — after exactly two normalisations: `EventRecord::row_id` is set to `None`, and +every `DateTime` is truncated to the microsecond precision a PostgreSQL `timestamp` +column has. Both are what makes a source row and its stored copy project identically. + +The checksum is the 256-bit sum of per-row `SHA-256(len32 ‖ key ‖ canonical)`, added +modulo 2^256. Three properties made that shape the choice: the sum does not depend on the +order rows are read in, it needs constant memory over a table of any size, and it composes +across a resume (`sum(A ∪ B) = sum(A) + sum(B)`), so an interrupted run continues the number +it had rather than starting again. + +It is a corruption and completeness check — any added, removed or altered row moves it — and +not a cryptographic commitment, and it is not a defence against a deliberate collision. +Nothing here needs one. + +## Resume is per table, and a replay is one batch (D5) + +The manifest holds, per table, a keyset cursor `after`, the accumulated `rows` and +`checksum`, and `complete`. A run resumes each incomplete table at its cursor and seeds the +count and the sum from the entry. + +The store commit and the manifest write cannot share a transaction through the public +surface, so the batch is committed first and the manifest is replaced atomically second — a +temporary file and a rename. A crash between the two therefore replays at most one batch, +which is harmless because every write in it is a put of the source's own key and value: the +same rows produce the same count and the same sum, so the merged entry is unchanged. + +## The command surface (D6) + +`nostream-cli migrate pg-to-lmdb` takes `--target `, `--batch-size ` and `--verify`, +and nothing else. `--target` is required with no default: a default of the working directory +would let a mistyped run write a store where none was meant, and verify needs it too because +the manifest lives under it. `--batch-size` defaults to 1 000, the import's default, and its +minimum is 1: a batch of zero would never advance a cursor. `--verify` copies nothing and +reads only the target. + +Exit codes are the CLI's own, from `CLI.md`: 0 on success, 2 for a usage error, 1 for every +other failure, which is written to stderr as `Migration failed: {message}`. That message +needs a `CliError` variant of its own — reusing `Import` would report `Import failed:` for a +migration — so `CliError::Migrate` exists and `main` prints it with its own prefix. + +The help text states the one precondition nothing can check (see below) and names the source: +Postgres from `DB_URI`, else the `DB_*` parts. + +## The manifest (D7) + +The manifest is `/migration-manifest.json`: `format: 1`, `not_migrated` with the two +table names, and one entry per table carrying `rows`, `checksum`, `after` and `complete`. + +It carries no timestamp, no rate and no path. That is what makes two runs over the same +source produce byte-identical manifests, and what lets an interrupted-then-resumed run be +compared with a clean one as bytes rather than as a parse. It also keeps a source URL out of +a file that may be copied around. + +The checksum string is sixty-four `0` characters for an empty table, so an entry is always +present and always well formed. + +## Throughput is the run's own measurement (D8) + +The copy measures the wall clock from just before its first read to just after its last +write and prints two lines: + +``` +Migrated {total} rows in {ms} ms +Events throughput: {rate} events/s +``` + +`total` is the rows this invocation wrote, and the rate is `events * 1000 / max(ms, 1)` over +the `events` table only. Both numbers are plain digits so a reader can recompute the rate +exactly, and the rate comes from the same integers that were printed. + +**No floor is asserted.** The board's 50 000 events/s target belongs to the benchmark rig, +which does not exist; a CI runner's rate is not that number, so this tool reports what it +measured and claims nothing about it. + +## The relay must be stopped (D9) + +Both stores are documented preconditions of a run, printed in the command's help: the relay +must be stopped on both. + +The reason is that nothing in this repository proves a second process may hold the LMDB +environment. LMDB's lock file would let one in, and heed opens with no environment flags, so +the locking protocol is active — but the daemon opens no backend of its own today, so +sharing is untested rather than known-safe. The tool takes no lock of its own: a lock that +nothing else honours is worse than a documented precondition, because an operator would read +it as protection. + +## A refused row stops the run (D10) + +A row the store refuses — a tag value at the 469-byte key bound, a NUL in a text key — fails +the batch and stops the run. The message names the table, not a batch ordinal, so an +operator knows which row stopped the copy, and the manifest keeps the previous batch's +cursor, count and sum, so the next run resumes at the refused row. + +Nothing is skipped and no table is marked complete. Skipping would leave the target missing +a row while the manifest's checksum still counted it, which turns a row an operator can look +at into a verify failure after a full copy. + +## The report sequence moves with the rows (D11) + +`reports.id` is carried across, and after each batch `meta`'s `reports.next_id` is raised to +the largest migrated id in that batch — never lowered, and left alone by a batch with no +report. A later `ReportRepository::create` then allocates past the migrated rows instead of +colliding with them. + +## The source is read-only (D12) + +The migration issues no `INSERT`, `UPDATE`, `DELETE` or DDL against Postgres, does not run +the embedded migrator, and runs no statement but the nine paged reads. Its schema contract +is the migrations' own, so a source that has not been migrated fails with the driver's +error rather than being repaired in place. + +A read that repaired its source would be a second writer on a live relay's database and +would make a failed run something an operator has to undo. + +## What this record does not decide + +- No throughput floor: the benchmark rig owns that number. +- No cutover, and no judgement that the migrated store is fit to serve. +- No migration of `user_subscriptions` or `event_tags`. +- No reverse migration, and no migration between two Postgres databases. +- No change to the nine traits, the LMDB encodings, the database list, the Postgres schema, + `LMDB_MAP_SIZE`, `LMDB_MAX_DBS` or the maximum key length. From aa8660d59dc58acdba2ba254578b3b14f46dfe5a Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:12:06 -0500 Subject: [PATCH 09/49] chore: add the migration tool's empty changeset --- .changeset/pg-to-lmdb-migration.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changeset/pg-to-lmdb-migration.md diff --git a/.changeset/pg-to-lmdb-migration.md b/.changeset/pg-to-lmdb-migration.md new file mode 100644 index 00000000..a845151c --- /dev/null +++ b/.changeset/pg-to-lmdb-migration.md @@ -0,0 +1,2 @@ +--- +--- From 71ef2a1c72e351323c97f0f2d9661c7f22358e8b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:21:38 -0500 Subject: [PATCH 10/49] test(cli): state AC8's liveness split where the assertion reads --- crates/nostream-cli/tests/migrate_cli.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index b04ec2dd..5dfeadde 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -935,10 +935,15 @@ async fn a_migrated_event_keeps_first_seen_deduplication_and_its_deleted_state() rows_of(&rows, MigratedTable::Events) ); - // The store's filter read carries the rows' own state: `page` builds its page with - // `live: false`, so a soft-deleted or expired row comes back rather than disappearing, and + // The criterion's split is the store's own: `lmdb/events.rs:775-807` builds a filter page with + // `live: false`, so a soft-deleted or expired row is readable by filter, and // `count_by_filters` is the read that applies the live predicate (ADR 0011 D4, pinned by - // `parity_lmdb::a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count`). + // `parity_lmdb::a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count`). The + // count is the half that proves B and C are stored *and* marked. + // + // A copy that read its **source** through `find_by_filters` would lose B and C instead: the + // Postgres composer adds `deleted_at IS NULL AND (expires_at IS NULL OR expires_at > $now)` + // (`postgres/sql.rs:265`), which is why the seam's reader has no filter at all. let filters = [Filter { limit: Some(10), ..Filter::default() From b93dc9efad3ea42215f95dd486f73052fa4a04ee Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:37:20 -0500 Subject: [PATCH 11/49] fix(storage): page invoices on their text id --- .../src/postgres/migration.rs | 102 +++++++++++++++++- crates/nostream-storage/tests/migration_pg.rs | 96 ++++++++++++++++- 2 files changed, 193 insertions(+), 5 deletions(-) diff --git a/crates/nostream-storage/src/postgres/migration.rs b/crates/nostream-storage/src/postgres/migration.rs index e19b5e61..e6ea85d2 100644 --- a/crates/nostream-storage/src/postgres/migration.rs +++ b/crates/nostream-storage/src/postgres/migration.rs @@ -47,11 +47,28 @@ impl MigrationSource for PostgresBackend { } /// How a table's unique key is bound as a keyset cursor. +/// +/// Each variant is the *column's* type, read from `migrations/`, not the shape the key looks +/// like: `invoices.id` began as a `uuid` and was changed to `text` +/// (`migrations/20230205004400_change_invoice_id_to_string.sql:1`), so a `::uuid` cast against +/// it is rejected by the server on the second page. The table below is the audit. +/// +/// | table | column | type | +/// |---|---|---| +/// | `dvm_jobs` | `id` | `bytea` | +/// | `events` | `event_id` | `bytea` | +/// | `invite_codes` | `code` | `varchar(64)` | +/// | `invoices` | `id` | `text` | +/// | `nip05_verifications` | `pubkey` | `bytea` | +/// | `notification_delivery_log` | `id` | `uuid` | +/// | `notification_outbox` | `id` | `uuid` | +/// | `reports` | `id` | `serial` (`i32`) | +/// | `users` | `pubkey` | `bytea` | #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Cursor { /// A `bytea` key: the row's key bytes are the column's value. Bytes, - /// A `uuid` key: the row's key bytes are its canonical text. + /// A `uuid` key: the row's key bytes are its canonical text, cast back to the column's type. Uuid, /// A `text`/`varchar` key: the row's key bytes are the column's value. Text, @@ -73,7 +90,8 @@ fn statement( MigratedTable::DvmJobs => (DVM_COLUMNS, "id", Cursor::Bytes), MigratedTable::Events => (EVENT_COLUMNS, "event_id", Cursor::Bytes), MigratedTable::InviteCodes => (INVITE_COLUMNS, "code", Cursor::Text), - MigratedTable::Invoices => (INVOICE_COLUMNS, "id", Cursor::Uuid), + // `text`, not `uuid`: `20230205004400_change_invoice_id_to_string.sql` altered it. + MigratedTable::Invoices => (INVOICE_COLUMNS, "id", Cursor::Text), MigratedTable::Nip05Verifications => (NIP05_COLUMNS, "pubkey", Cursor::Bytes), MigratedTable::NotificationDeliveryLog => (DELIVERY_LOG_COLUMNS, "id", Cursor::Uuid), MigratedTable::NotificationOutbox => (OUTBOX_COLUMNS, "id", Cursor::Uuid), @@ -184,3 +202,83 @@ fn user(row: &PgRow) -> Result { tos_accepted_at: tos_accepted_at.map(crate::records::utc_from_naive), })) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + /// The statement one table composes when paging from `after`. + fn sql(table: MigratedTable, after: Option<&[u8]>) -> String { + statement(table, after, 2) + .expect("the statement composes") + .0 + } + + #[test] + fn the_first_page_of_every_table_binds_no_cursor() { + for table in MigratedTable::ALL { + let text = sql(table, None); + assert!( + !text.contains("WHERE"), + "{} pages from the first row: {text}", + table.name() + ); + assert!( + text.ends_with("LIMIT $1"), + "{} binds its limit first: {text}", + table.name() + ); + } + } + + /// `invoices.id` was a `uuid` until `20230205004400_change_invoice_id_to_string.sql` made it + /// `text`; a `::uuid` cast on the cursor makes the second page a server-side type error. + #[test] + fn the_uuid_cursor_is_the_two_tables_the_schema_makes_uuid() { + let uuid = b"00000000-0000-4000-8000-000000000001"; + + let invoices = sql(MigratedTable::Invoices, Some(uuid)); + assert!(invoices.contains("id > $1"), "{invoices}"); + assert!( + !invoices.contains("::uuid"), + "invoices.id is text: {invoices}" + ); + + for table in [ + MigratedTable::NotificationDeliveryLog, + MigratedTable::NotificationOutbox, + ] { + let text = sql(table, Some(uuid)); + assert!( + text.contains("id > $1::uuid"), + "{} pages on a uuid column: {text}", + table.name() + ); + } + } + + #[test] + fn every_cursor_is_the_tables_own_key() { + assert!(sql(MigratedTable::Events, Some(&[7; 32])).contains("event_id > $1")); + assert!(sql(MigratedTable::Users, Some(&[7; 32])).contains("pubkey > $1")); + assert!(sql(MigratedTable::Nip05Verifications, Some(&[7; 32])).contains("pubkey > $1")); + assert!(sql(MigratedTable::DvmJobs, Some(&[0; 32])).contains("id > $1")); + assert!(sql(MigratedTable::InviteCodes, Some(b"code-1")).contains("code > $1")); + assert!(sql(MigratedTable::Reports, Some(&[0x80, 0x00, 0x00, 0x02])).contains("id > $1")); + } + + #[test] + fn a_reports_cursor_is_undone_before_it_is_bound() { + assert_eq!( + int32_key(MigratedTable::Reports, &[0x80, 0x00, 0x00, 0x02]).unwrap(), + 2 + ); + assert_eq!( + int32_key(MigratedTable::Reports, &[0x7f, 0xff, 0xff, 0xfd]).unwrap(), + -3 + ); + assert!(int32_key(MigratedTable::Reports, &[0x80]).is_err()); + } +} diff --git a/crates/nostream-storage/tests/migration_pg.rs b/crates/nostream-storage/tests/migration_pg.rs index 80da78ef..c240183b 100644 --- a/crates/nostream-storage/tests/migration_pg.rs +++ b/crates/nostream-storage/tests/migration_pg.rs @@ -143,9 +143,11 @@ async fn source_count(pool: &PgPool, table: MigratedTable) -> i64 { .expect("a count") } -#[tokio::test] -#[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] -async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { +/// A fresh scratch database with the schema applied. +/// +/// The targets that use it run with `--test-threads=1` (CI and `scripts/storage-parity.sh`), so +/// two of them never drop the database under each other. +async fn scratch() -> PostgresBackend { let admin = PgPool::connect(&admin_url()) .await .expect("the administrative database is reachable"); @@ -165,6 +167,13 @@ async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { .await .expect("the scratch database is reachable"); source.migrate().await.expect("the schema migrates"); + source +} + +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] +async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { + let source = scratch().await; seed(&source).await; let directory = tempfile::tempdir().expect("a temporary directory"); @@ -204,6 +213,87 @@ async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { } } +/// Three rows at a batch size of two makes the second page real, which is the statement a wrong +/// cursor type rejects: `invoices.id` is `text` since +/// `migrations/20230205004400_change_invoice_id_to_string.sql` (so `id > $1::uuid` is a +/// server-side type error), and the two notification tables' `id`s are `uuid`. Every page is read +/// through the seam's own reader, so the assertion is on the SQL that reader composes. +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] +async fn every_text_and_uuid_cursor_pages_over_its_own_column_type() { + let source = scratch().await; + let pool = source.master(); + + for seed in 1..=3u8 { + let outbox_id = format!("40000000-0000-4000-8000-00000000000{seed}"); + sqlx::query( + "INSERT INTO invoices (id, pubkey, bolt11, amount_requested, unit, status) \ + VALUES ($1, $2, $3, 1000, 'msats', 'completed')", + ) + .bind(format!("30000000-0000-4000-8000-00000000000{seed}")) + .bind(vec![4u8; 32]) + .bind(format!("lnbc1page{seed}")) + .execute(pool) + .await + .expect("the invoice inserts"); + + sqlx::query( + "INSERT INTO notification_outbox (id, event_type, payload) \ + VALUES ($1::uuid, 'invoice.paid', $2)", + ) + .bind(&outbox_id) + .bind(serde_json::json!({ "seed": seed })) + .execute(pool) + .await + .expect("the outbox row inserts"); + + sqlx::query( + "INSERT INTO notification_delivery_log (id, outbox_id, event_type, target_id, \ + target_type, status, attempt_number) \ + VALUES ($1::uuid, $2::uuid, 'invoice.paid', 'telegram:7175022', 'telegram', \ + 'success', 1)", + ) + .bind(format!("50000000-0000-4000-8000-00000000000{seed}")) + .bind(&outbox_id) + .execute(pool) + .await + .expect("the delivery-log row inserts"); + } + + for table in [ + MigratedTable::Invoices, + MigratedTable::NotificationDeliveryLog, + MigratedTable::NotificationOutbox, + ] { + let first = MigrationSource::read_batch(&source, table, None, BATCH) + .await + .expect("the first page reads"); + assert_eq!(first.len(), BATCH, "{} has a first page", table.name()); + let cursor = first.last().expect("a row").key(); + + let second = MigrationSource::read_batch(&source, table, Some(&cursor), BATCH) + .await + .expect("the cursor binds as the column's own type"); + assert_eq!( + second.len(), + 1, + "{} pages past its first batch", + table.name() + ); + assert!( + second[0].key() > cursor, + "{} pages in key order", + table.name() + ); + + let after = second.last().expect("a row").key(); + let third = MigrationSource::read_batch(&source, table, Some(&after), BATCH) + .await + .expect("the third page reads"); + assert!(third.is_empty(), "{} is exhausted", table.name()); + } +} + /// Two rows in each seeded table, except events (three, one per liveness state) and the two /// tables the manifest does not move. async fn seed(backend: &PostgresBackend) { From dae4d0ee70e32bba154a82a67d67a8dd63108cd2 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:38:33 -0500 Subject: [PATCH 12/49] feat(cli): refuse a target that already holds rows --- crates/nostream-cli/src/commands/migrate.rs | 38 ++++++++-- crates/nostream-cli/tests/migrate_cli.rs | 80 +++++++++++++++++++++ 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 4367a424..d09abee7 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -84,10 +84,23 @@ pub async fn migrate_with( options: &MigrateOptions, out: &mut dyn Write, ) -> Result { - let mut manifest = match read_manifest(&options.target)? { - Some(manifest) => manifest, - None => Manifest::fresh(), - }; + let existing = read_manifest(&options.target)?; + let mut manifest = existing.clone().unwrap_or_else(Manifest::fresh); + + // A store that already holds rows and carries no manifest is not this tool's. Copying over + // one leaves the displaced rows' index entries behind — a completed invoice keeps its + // `invoices_pending` entry — and verify scans the primary databases only, so it could never + // report them (R7). + if existing.is_none() + && let Some(table) = first_occupied_table(target).await? + { + return Err(CliError::Migrate(format!( + "{} already holds rows ({}): an empty target, or one that carries a manifest, is required", + options.target.display(), + table.name() + ))); + } + if manifest.complete() { return verify_with(target, options, out).await; } @@ -400,6 +413,23 @@ fn no_manifest(target: &Path, out: &mut dyn Write) -> Result Result, CliError> { + for table in MigratedTable::ALL { + let rows = target + .scan_batch(table, None, 1) + .await + .map_err(|error| table_failed(table, &error))?; + if !rows.is_empty() { + return Ok(Some(table)); + } + } + Ok(None) +} + /// Opens the store a run reads or writes. async fn open_target(root: &Path) -> Result { LmdbBackend::open(root) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 5dfeadde..9171b1ac 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1069,6 +1069,86 @@ async fn a_refused_row_stops_the_run_and_keeps_the_resume_point() { ); } +// --------------------------------------------------------------------------------------- +// AC16: a target that already holds rows and carries no manifest. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_target_with_rows_and_no_manifest_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + // Three rows in one table, through the seam's own write, and no manifest anywhere. + let seeded: Vec = (1..=3) + .map(|seed| MigrationRow::Event(event_record(seed, "seeded"))) + .collect(); + assert_eq!( + target.write_batch(&seeded).await.expect("the seed writes"), + 3 + ); + let manifest = directory.path().join("migration-manifest.json"); + assert!(!manifest.exists(), "the seed writes no manifest"); + + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("a target with rows and no manifest is refused"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a migration failure is the variant `main` prints as `Migration failed:` and exits 1 for" + ); + let message = error.to_string(); + assert!(message.contains("already holds rows"), "{message}"); + assert!( + message.contains(&directory.path().display().to_string()), + "the message names the target: {message}" + ); + assert_eq!(source.calls().len(), 0, "nothing was read"); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 3, + "the seeded rows are untouched" + ); + for table in MigratedTable::ALL { + if table == MigratedTable::Events { + continue; + } + assert_eq!( + scan_all(&target, table).await.len(), + 0, + "{} is untouched", + table.name() + ); + } + assert!(!manifest.exists(), "no manifest was written"); + + // The control: the same source into an empty directory proceeds. + let empty = tempfile::tempdir().expect("a temporary directory"); + let clean = LmdbBackend::open(empty.path()) + .await + .expect("the store opens"); + let output = migrate::migrate_with( + &StubSource::new(source_rows()), + &clean, + &options(empty.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("an empty target proceeds"); + assert_eq!(output.exit_code, 0); + assert!( + empty.path().join("migration-manifest.json").exists(), + "the control run wrote its manifest" + ); +} + /// The process boundary the two failure tests above stop at: `main` prints /// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on /// the way to the copy, which is the same arm. From 8c26b70f4973f0e157b3f939aea1f4066b73daa9 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:38:37 -0500 Subject: [PATCH 13/49] test(cli): measure the run against a target that sleeps --- crates/nostream-cli/tests/migrate_cli.rs | 93 ++++++++++++++---------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 9171b1ac..fdb6d869 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -11,7 +11,7 @@ use std::process::Command; use std::sync::Mutex; -use std::time::Instant; +use std::time::Duration; use chrono::{DateTime, TimeZone, Utc}; use heed::types::Bytes; @@ -820,32 +820,17 @@ async fn verify_rejects_a_missing_manifest_with_a_message_and_exit_1() { // AC7: the run's own measurement. // --------------------------------------------------------------------------------------- -/// Copies `count` events and answers the two summary numbers, the test's own measurement, and -/// the row total. -async fn measured_copy(count: usize) -> (u64, u64, u64, u128) { - let directory = tempfile::tempdir().expect("a temporary directory"); - let target = LmdbBackend::open(directory.path()) - .await - .expect("the store opens"); - let rows: Vec = (0..count) - .map(|index| { - let mut record = event_record(1, "hello"); - let mut id = [0u8; 32]; - id[..8].copy_from_slice(&(index as u64).to_be_bytes()); - record.event.id = EventId::from_byte_array(id); - MigrationRow::Event(record) - }) - .collect(); - let options = options(directory.path(), 1_000); - - let mut out = Vec::new(); - let started = Instant::now(); - migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut out) - .await - .expect("the copy succeeds"); - let measured = started.elapsed().as_millis(); +/// One event whose id is its index, so a thousand of them are a thousand keys in key order. +fn indexed_event(index: u64) -> MigrationRow { + let mut record = event_record(1, "hello"); + let mut id = [0u8; 32]; + id[..8].copy_from_slice(&index.to_be_bytes()); + record.event.id = EventId::from_byte_array(id); + MigrationRow::Event(record) +} - let text = output_text(out); +/// The two summary lines, parsed. +fn summary(text: &str) -> (u64, u64, u64) { let mut lines = text.lines(); let first = lines.next().expect("the rows line"); let second = lines.next().expect("the throughput line"); @@ -865,32 +850,62 @@ async fn measured_copy(count: usize) -> (u64, u64, u64, u128) { rows_text.parse().expect("a row count"), ms_text.parse().expect("a millisecond count"), rate_text.parse().expect("a rate"), - measured, ) } +/// A target that sleeps inside every batch it writes, so the run has work its own clock cannot +/// miss: five batches of 200 rows pay at least 100 ms. +struct SleepingTarget { + inner: LmdbBackend, +} + +impl MigrationTarget for SleepingTarget { + fn write_batch<'a>(&'a self, rows: &'a [MigrationRow]) -> StorageFuture<'a, u64> { + Box::pin(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + MigrationTarget::write_batch(&self.inner, rows).await + }) + } + + fn scan_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + MigrationTarget::scan_batch(&self.inner, table, after, limit) + } +} + #[tokio::test] async fn the_run_prints_a_rate_consistent_with_its_own_events_and_milliseconds() { - let (rows, ms, rate, measured) = measured_copy(1_000).await; + let directory = tempfile::tempdir().expect("a temporary directory"); + let store = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let target = SleepingTarget { inner: store }; + // 1 000 events at --batch-size 200 is five full batches plus the closing short read, and the + // target sleeps 20 ms in each, so the copy cannot finish in less than 100 ms. + let rows: Vec = (0..1_000).map(indexed_event).collect(); + let options = options(directory.path(), 200); + + let mut out = Vec::new(); + migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut out) + .await + .expect("the copy succeeds"); + let (rows, ms, rate) = summary(&output_text(out)); + assert_eq!(rows, 1_000); - assert!(ms > 0, "the run took a measurable time"); - let difference = i128::from(ms) - i128::try_from(measured).expect("a bounded duration"); assert!( - difference.abs() <= 20, - "the printed {ms} ms is the run's own clock, {measured} ms measured outside it" + ms >= 100, + "the run's clock includes the 100 ms the target slept: printed {ms} ms" ); assert_eq!( rate, 1_000 * 1_000 / ms.max(1), "the rate is the printed numbers'" ); - - let (rows, longer, _, _) = measured_copy(10_000).await; - assert_eq!(rows, 10_000); - assert!( - longer > ms, - "ten times the work takes longer: {longer} ms against {ms} ms" - ); + assert!(rate > 0); } // --------------------------------------------------------------------------------------- From a1c358362a141a92e91228a6ed82c9664d5e9e89 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:38:40 -0500 Subject: [PATCH 14/49] fix(cli): refuse a manifest of another format --- crates/nostream-cli/src/commands/migrate.rs | 13 ++++-- crates/nostream-cli/tests/migrate_cli.rs | 50 +++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index d09abee7..a9252f72 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -387,9 +387,16 @@ fn read_manifest(target: &Path) -> Result, CliError> { Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(CliError::Migrate(format!("{}: {error}", path.display()))), }; - serde_json::from_slice(&bytes) - .map(Some) - .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display()))) + let manifest: Manifest = serde_json::from_slice(&bytes) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + if manifest.format != MANIFEST_FORMAT { + return Err(CliError::Migrate(format!( + "{}: manifest format {} is not supported; this tool reads and writes {MANIFEST_FORMAT}", + path.display(), + manifest.format + ))); + } + Ok(Some(manifest)) } /// Replaces the manifest through a temporary file, so a reader never sees a half-written one. diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index fdb6d869..f6ab0676 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1164,6 +1164,56 @@ async fn a_target_with_rows_and_no_manifest_is_refused() { ); } +// --------------------------------------------------------------------------------------- +// A manifest of another format, a batch of zero, and a verify with no store. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_manifest_of_another_format_is_refused_before_anything_is_written() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + std::fs::write( + directory.path().join("migration-manifest.json"), + br#"{"format":2,"not_migrated":[],"tables":{}}"#, + ) + .expect("the fixture manifest is written"); + let usage = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("format 2 is refused"); + let message = error.to_string(); + assert!(message.contains("migration-manifest.json"), "{message}"); + assert!(message.contains("format 2"), "{message}"); + assert_eq!(source.calls().len(), 0, "nothing was read"); + assert_eq!( + target + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + usage, + "nothing was written" + ); + + let error = migrate::verify_with(&target, &options(directory.path(), 1_000), &mut Vec::new()) + .await + .expect_err("format 2 is refused by verify too"); + assert!(error.to_string().contains("format 2"), "{error}"); +} + /// The process boundary the two failure tests above stop at: `main` prints /// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on /// the way to the copy, which is the same arm. From 11afb3e380d87ff47d44e79ee5b6446113844565 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:38:51 -0500 Subject: [PATCH 15/49] fix(cli): refuse a batch size of zero in both entry points --- crates/nostream-cli/src/commands/migrate.rs | 11 +++++++++ crates/nostream-cli/tests/migrate_cli.rs | 25 +++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index a9252f72..907decb8 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -84,6 +84,7 @@ pub async fn migrate_with( options: &MigrateOptions, out: &mut dyn Write, ) -> Result { + refuse_zero_batch(options)?; let existing = read_manifest(&options.target)?; let mut manifest = existing.clone().unwrap_or_else(Manifest::fresh); @@ -177,6 +178,7 @@ pub async fn verify_with( options: &MigrateOptions, out: &mut dyn Write, ) -> Result { + refuse_zero_batch(options)?; let Some(manifest) = read_manifest(&options.target)? else { return no_manifest(&options.target, out); }; @@ -437,6 +439,15 @@ async fn first_occupied_table( Ok(None) } +/// A batch of zero never advances a cursor. The parser refuses it, and these two entry points are +/// public, so they refuse it themselves rather than loop. +fn refuse_zero_batch(options: &MigrateOptions) -> Result<(), CliError> { + if options.batch_size == 0 { + return Err(CliError::Migrate("--batch-size must be at least 1".to_owned())); + } + Ok(()) +} + /// Opens the store a run reads or writes. async fn open_target(root: &Path) -> Result { LmdbBackend::open(root) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index f6ab0676..43985a5f 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1214,6 +1214,31 @@ async fn a_manifest_of_another_format_is_refused_before_anything_is_written() { assert!(error.to_string().contains("format 2"), "{error}"); } +#[tokio::test] +async fn a_zero_batch_is_refused_instead_of_looping() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 0), + &mut Vec::new(), + ) + .await + .expect_err("a batch of zero is refused"); + assert!(error.to_string().contains("--batch-size"), "{error}"); + assert_eq!(source.calls().len(), 0, "nothing was read"); + + let error = migrate::verify_with(&target, &options(directory.path(), 0), &mut Vec::new()) + .await + .expect_err("a batch of zero is refused by verify too"); + assert!(error.to_string().contains("--batch-size"), "{error}"); +} + /// The process boundary the two failure tests above stop at: `main` prints /// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on /// the way to the copy, which is the same arm. From 08baaaf69cfa485d6adbab89f55a29915f158e29 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:38:55 -0500 Subject: [PATCH 16/49] fix(cli): verify without creating a missing store --- crates/nostream-cli/src/commands/migrate.rs | 32 +++++++++++++- crates/nostream-cli/tests/migrate_cli.rs | 46 +++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 907decb8..b999b822 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -15,6 +15,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; +use nostream_storage::backend::LMDB_DIR; use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; use nostream_storage::{LmdbBackend, PostgresBackend}; use sha2::{Digest, Sha256}; @@ -53,6 +54,11 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result Result Result { + writeln!(out, "No store at {}", store_path(options).display()).map_err(write_failed)?; + Ok(CommandOutput { + lines: Vec::new(), + exit_code: 1, + }) +} + +/// The store's directory under a target root: the environment `LmdbBackend::open` would create. +fn store_path(options: &MigrateOptions) -> PathBuf { + options.target.join(LMDB_DIR) +} + +/// Whether the target already holds an LMDB environment. +fn store_present(options: &MigrateOptions) -> bool { + store_path(options).exists() +} + /// The first table of a manifest-less target that already holds a row, if any: one row read per /// table, which is the probe R7 asks for. async fn first_occupied_table( @@ -443,7 +471,9 @@ async fn first_occupied_table( /// public, so they refuse it themselves rather than loop. fn refuse_zero_batch(options: &MigrateOptions) -> Result<(), CliError> { if options.batch_size == 0 { - return Err(CliError::Migrate("--batch-size must be at least 1".to_owned())); + return Err(CliError::Migrate( + "--batch-size must be at least 1".to_owned(), + )); } Ok(()) } diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 43985a5f..b12c1bb6 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -9,6 +9,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use std::collections::BTreeMap; use std::process::Command; use std::sync::Mutex; use std::time::Duration; @@ -1239,6 +1240,51 @@ async fn a_zero_batch_is_refused_instead_of_looping() { assert!(error.to_string().contains("--batch-size"), "{error}"); } +#[test] +fn verify_refuses_a_manifest_without_a_store_instead_of_creating_one() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let manifest = json!({ + "format": 1, + "not_migrated": MigratedTable::NOT_MIGRATED, + "tables": MigratedTable::ALL + .iter() + .map(|table| ( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": true}), + )) + .collect::>(), + }); + std::fs::write( + directory.path().join("migration-manifest.json"), + serde_json::to_vec(&manifest).expect("the fixture manifest serialises"), + ) + .expect("the fixture manifest is written"); + let store = directory.path().join("lmdb"); + + for verify in [true, false] { + let mut out = Vec::new(); + let output = migrate::run( + MigrateOptions { + verify, + ..options(directory.path(), 1_000) + }, + &mut out, + ) + .expect("a missing store is an answer, not a failure"); + assert_eq!(output.exit_code, 1); + let text = output_text(out); + assert!(text.contains("No store at"), "{text}"); + assert!( + text.contains(&store.display().to_string()), + "the line names the store: {text}" + ); + assert!( + !store.exists(), + "verify created no store (verify = {verify})" + ); + } +} + /// The process boundary the two failure tests above stop at: `main` prints /// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on /// the way to the copy, which is the same arm. From 73abd6dabe300dbae14ca9f29d0327b7439ca759 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:39:29 -0500 Subject: [PATCH 17/49] docs: run the migration target in the parity instructions --- CLAUDE.md | 8 +++++--- scripts/storage-parity.sh | 12 +++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f73437b5..de773bfa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,17 +36,19 @@ cargo test --workspace --all-features cargo build --workspace --all-features --release ``` -CI additionally runs the two database-backed storage tests ("Storage parity against Postgres") +CI additionally runs the three database-backed storage tests ("Storage parity against Postgres") against a `postgres:16-alpine` service, and they are `#[ignore]`d so a checkout without a database still passes `cargo test --workspace`: ``` -cargo test -p nostream-storage --test migrations --test parity -- --ignored --test-threads=1 +cargo test -p nostream-storage --test migrations --test parity --test migration_pg \ + -- --ignored --test-threads=1 ``` `PARITY_ADMIN_URL` names the administrative database (CI sets `postgres://postgres:postgres@localhost:5432/postgres`); the targets create, migrate and -drop their own scratch databases. `scripts/storage-parity.sh` runs both plus the +drop their own scratch databases — `migration_pg` seeds one and copies it into a temporary LMDB +store. `scripts/storage-parity.sh` runs all three plus the TypeScript Cucumber suite against the sqlx-migrated schema — the complete local oracle. The LMDB arm of that oracle needs no database: `cargo test -p nostream-storage --test parity_lmdb -- --nocapture` replays the same fixture on a temporary store and runs inside the fifth diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index 3e8291e4..494efe0e 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -1,6 +1,7 @@ #!/bin/sh -# The storage-parity oracle: the 37 sqlx migrations, the differential 51-method replay and the -# unmodified TypeScript relay's Cucumber suite, all against databases this script creates. +# The storage-parity oracle: the 37 sqlx migrations, the differential 51-method replay, the +# Postgres-to-LMDB migration and the unmodified TypeScript relay's Cucumber suite, all against +# databases this script creates. # # Usage: scripts/storage-parity.sh # @@ -285,6 +286,11 @@ echo "== behavioural parity" cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \ || fail "the parity target failed" +echo "== the Postgres-to-LMDB migration" +./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" \ + cargo test -p nostream-storage --test migration_pg -- --ignored --test-threads=1 \ + || fail "the migration target failed" + echo "== lmdb behavioural parity (no database)" ./scripts/rust.sh cargo test -p nostream-storage --test parity_lmdb -- --nocapture \ || fail "the lmdb parity target failed" @@ -317,4 +323,4 @@ NODE_OPTIONS=--no-experimental-strip-types \ "$CUCUMBER_BIN" --format summary \ || fail "the cucumber suite failed" -echo "== storage-parity: all four steps passed" +echo "== storage-parity: all steps passed" From 592add52b885595671f6f041afbeeced830ecc1b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:42:49 -0500 Subject: [PATCH 18/49] test(storage): page the invite-code cursor too --- crates/nostream-storage/tests/migration_pg.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/nostream-storage/tests/migration_pg.rs b/crates/nostream-storage/tests/migration_pg.rs index c240183b..c5e0af6a 100644 --- a/crates/nostream-storage/tests/migration_pg.rs +++ b/crates/nostream-storage/tests/migration_pg.rs @@ -216,8 +216,9 @@ async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { /// Three rows at a batch size of two makes the second page real, which is the statement a wrong /// cursor type rejects: `invoices.id` is `text` since /// `migrations/20230205004400_change_invoice_id_to_string.sql` (so `id > $1::uuid` is a -/// server-side type error), and the two notification tables' `id`s are `uuid`. Every page is read -/// through the seam's own reader, so the assertion is on the SQL that reader composes. +/// server-side type error), `invite_codes.code` is a `varchar(64)` that takes the same text bind, +/// and the two notification tables' `id`s are `uuid`. Every page is read through the seam's own +/// reader, so the assertion is on the SQL that reader composes. #[tokio::test] #[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] async fn every_text_and_uuid_cursor_pages_over_its_own_column_type() { @@ -258,9 +259,16 @@ async fn every_text_and_uuid_cursor_pages_over_its_own_column_type() { .execute(pool) .await .expect("the delivery-log row inserts"); + + sqlx::query("INSERT INTO invite_codes (code) VALUES ($1)") + .bind(format!("invite-{seed}")) + .execute(pool) + .await + .expect("the invite code inserts"); } for table in [ + MigratedTable::InviteCodes, MigratedTable::Invoices, MigratedTable::NotificationDeliveryLog, MigratedTable::NotificationOutbox, From 2be1e8b60504222e9db1735b307703bd3ca56d8c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:58:09 -0500 Subject: [PATCH 19/49] feat(cli): write the manifest before the first batch --- crates/nostream-cli/src/commands/migrate.rs | 6 ++ crates/nostream-cli/tests/migrate_cli.rs | 101 ++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index b999b822..dec1d5d3 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -111,6 +111,12 @@ pub async fn migrate_with( ))); } + // R6: written before the first `read_batch`, so a store this tool has begun to fill always + // carries a manifest and a crash after the first batch resumes rather than being refused. + if existing.is_none() { + write_manifest(&options.target, &manifest)?; + } + if manifest.complete() { return verify_with(target, options, out).await; } diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index b12c1bb6..54a2c899 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -817,6 +817,107 @@ async fn verify_rejects_a_missing_manifest_with_a_message_and_exit_1() { ); } +// --------------------------------------------------------------------------------------- +// AC17: a crash between the first batch's commit and its manifest write. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_crash_between_the_first_batch_and_the_manifest_write_resumes_to_the_clean_manifest() { + let table = MigratedTable::ALL[0]; + + // The uninterrupted run: the manifest and the counts a resume has to land on. + let clean = tempfile::tempdir().expect("a temporary directory"); + let clean_store = LmdbBackend::open(clean.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::new(source_rows()), + &clean_store, + &options(clean.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the clean copy succeeds"); + let clean_manifest = manifest_bytes(clean.path()); + let mut clean_counts = Vec::new(); + for table in MigratedTable::ALL { + clean_counts.push(scan_all(&clean_store, table).await.len()); + } + drop(clean_store); + + // The crash state, reproduced rather than killed: the fresh manifest R6 writes before its + // first batch, then that batch's rows, and nothing else. A source that fails on its first + // read leaves exactly that manifest behind, and its entries are asserted below. + let crashed = tempfile::tempdir().expect("a temporary directory"); + let store = LmdbBackend::open(crashed.path()) + .await + .expect("the store opens"); + let options = options(crashed.path(), 1_000); + let error = migrate::migrate_with( + &StubSource::failing_at(source_rows(), 0), + &store, + &options, + &mut Vec::new(), + ) + .await + .expect_err("the first read fails"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a source failure is the variant `main` reports as `Migration failed:`" + ); + + let fresh = manifest_bytes(crashed.path()); + let fresh: Value = serde_json::from_slice(&fresh).expect("the fresh manifest is json"); + assert_eq!(fresh["format"], 1); + assert_eq!( + fresh["not_migrated"], + json!(["event_tags", "user_subscriptions"]) + ); + let tables = fresh["tables"].as_object().expect("the tables object"); + assert_eq!(tables.len(), 9, "one entry per table"); + for entry in tables.values() { + assert_eq!(entry["rows"], 0); + assert_eq!(entry["checksum"], "0".repeat(64)); + assert_eq!(entry["after"], ""); + assert_eq!(entry["complete"], false); + } + + let source = StubSource::new(source_rows()); + let first = MigrationSource::read_batch(&source, table, None, 200) + .await + .expect("the first batch reads"); + assert!(!first.is_empty(), "the first table has rows"); + store + .write_batch(&first) + .await + .expect("the first batch is stored"); + + // The resume replays that batch from the fresh manifest's own cursor. + let resumed = StubSource::new(source_rows()); + let output = migrate::migrate_with(&resumed, &store, &options, &mut Vec::new()) + .await + .expect("the resume succeeds"); + assert_eq!(output.exit_code, 0); + assert_eq!( + resumed.calls().first(), + Some(&(table, None)), + "the resume starts at the fresh manifest's empty cursor" + ); + for (table, count) in MigratedTable::ALL.iter().zip(clean_counts) { + assert_eq!( + scan_all(&store, *table).await.len(), + count, + "{} count", + table.name() + ); + } + assert_eq!( + manifest_bytes(crashed.path()), + clean_manifest, + "the resume lands on the uninterrupted run's manifest" + ); +} + // --------------------------------------------------------------------------------------- // AC7: the run's own measurement. // --------------------------------------------------------------------------------------- From 2d6b2797d219929d14425b8a4d7cdd41a4628138 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:58:09 -0500 Subject: [PATCH 20/49] feat(storage): open an existing store read-only --- crates/nostream-storage/src/backend.rs | 36 ++++++++++- crates/nostream-storage/src/error.rs | 7 ++ crates/nostream-storage/src/lmdb/mod.rs | 85 +++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 980ec522..4383f70c 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use nostream_core::EventStoreBackend; #[cfg(feature = "lmdb")] -use heed::{Env, EnvOpenOptions}; +use heed::{Env, EnvFlags, EnvOpenOptions}; #[cfg(feature = "lmdb")] use crate::error::StorageError; @@ -94,6 +94,40 @@ impl LmdbEnv { Ok(Self { env, path }) } + /// Opens an existing LMDB environment at `path` read-only, and creates nothing. + /// + /// The directory is not made and the file is not created: a path with no environment is + /// heed's error rather than a new file. `READ_ONLY` takes no write lock, and this + /// constructor opens no transaction of its own. + /// + /// heed's process-global registry answers `Error::BadOpenOptions` for a second open of one + /// path with different options, so a caller that already holds this path writable must + /// drop that handle first. + pub fn open_existing( + path: impl Into, + map_size: usize, + ) -> Result { + let path = path.into(); + + // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on, and + // `READ_ONLY` makes the write half of them vacuous: no transaction is opened here at + // all. + #[allow(unsafe_code)] + let env = unsafe { + EnvOpenOptions::new() + .map_size(map_size) + .max_dbs(LMDB_MAX_DBS) + .max_readers(LMDB_MAX_READERS) + .flags(EnvFlags::READ_ONLY) + .open(&path) + } + .map_err(|source| StorageError::OpenLmdb { + path: path.clone(), + source, + })?; + Ok(Self { env, path }) + } + /// The environment handle, for the repository implementations. pub fn env(&self) -> &Env { &self.env diff --git a/crates/nostream-storage/src/error.rs b/crates/nostream-storage/src/error.rs index e0f1edbc..020683a8 100644 --- a/crates/nostream-storage/src/error.rs +++ b/crates/nostream-storage/src/error.rs @@ -53,6 +53,13 @@ pub enum StorageError { #[error("lmdb operation failed: {0}")] Lmdb(#[from] heed::Error), + /// A named database a caller required is absent from the environment. The migration + /// tool's verify opens a store it did not create, so a missing database is an error rather + /// than a new one. + #[cfg(feature = "lmdb")] + #[error("the store has no {name} database")] + MissingDatabase { name: &'static str }, + /// An LMDB environment could not be opened at the requested path. #[cfg(feature = "lmdb")] #[error("failed to open the lmdb environment at {path}: {source}")] diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index d976de36..a03df750 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -166,6 +166,54 @@ impl Databases { meta: env.create_database(txn, Some("meta"))?, }) } + /// Opens every database by name, refusing a store that lacks one. + /// + /// The migration tool's verify reads a store it did not create, so `create_database` is + /// wrong there: a missing database must be an error rather than a new one. + fn open(env: &Env, txn: &RoTxn<'_>) -> Result { + Ok(Self { + events: open_named(env, txn, "events")?, + events_by_created_at: open_named(env, txn, "events_by_created_at")?, + events_by_author: open_named(env, txn, "events_by_author")?, + events_deleted: open_named(env, txn, "events_deleted")?, + events_expiring: open_named(env, txn, "events_expiring")?, + replaceable: open_named(env, txn, "replaceable")?, + tags: open_named(env, txn, "tags")?, + users: open_named(env, txn, "users")?, + invoices: open_named(env, txn, "invoices")?, + invoices_pending: open_named(env, txn, "invoices_pending")?, + nip05: open_named(env, txn, "nip05_verifications")?, + nip05_by_last_checked_at: open_named(env, txn, "nip05_by_last_checked_at")?, + invite_codes: open_named(env, txn, "invite_codes")?, + invite_codes_by_created_at: open_named(env, txn, "invite_codes_by_created_at")?, + invite_codes_by_expires_at: open_named(env, txn, "invite_codes_by_expires_at")?, + dvm_jobs: open_named(env, txn, "dvm_jobs")?, + dvm_jobs_by_status_created_at: open_named(env, txn, "dvm_jobs_by_status_created_at")?, + reports: open_named(env, txn, "reports")?, + reports_by_event_id: open_named(env, txn, "reports_by_event_id")?, + reports_actionable_by_created_at: open_named(env, txn, "reports_actionable_by_created_at")?, + reports_actionable_event: open_named(env, txn, "reports_actionable_event")?, + reports_actionable_pubkey: open_named(env, txn, "reports_actionable_pubkey")?, + outbox: open_named(env, txn, "notification_outbox")?, + outbox_by_status_created_at: open_named(env, txn, "outbox_by_status_created_at")?, + outbox_by_status_updated_at: open_named(env, txn, "outbox_by_status_updated_at")?, + notification_delivery_log: open_named(env, txn, "notification_delivery_log")?, + delivery_log_by_created_at: open_named(env, txn, "delivery_log_by_created_at")?, + delivery_log_by_outbox: open_named(env, txn, "delivery_log_by_outbox")?, + meta: open_named(env, txn, "meta")?, + }) + } +} + +/// One named database of an existing environment, or [`StorageError::MissingDatabase`] when the +/// store does not hold it. +fn open_named( + env: &Env, + txn: &RoTxn<'_>, + name: &'static str, +) -> Result, StorageError> { + env.open_database(txn, Some(name))? + .ok_or(StorageError::MissingDatabase { name }) } /// How much of the fixed map one store has used (`Env::non_free_pages_size`). @@ -223,6 +271,43 @@ impl LmdbBackend { }) } + /// Opens the existing store under `/lmdb` for reading only, and creates nothing. + /// + /// The migration tool's verify path uses this instead of [`open`](Self::open): that one + /// `create_dir_all`s and `create_database`s, so verifying a store that is not there answers + /// with a fresh empty environment and every table short. Here a missing environment or a + /// missing database is an error. + /// + /// heed's process-global registry answers `BadOpenOptions` for a second open of one path + /// with different options, so a caller that already holds this root writable must drop that + /// handle first. + pub async fn open_existing(root: impl Into) -> Result { + let root = root.into(); + // The same rule `data_dir` answers, from the same constant. + let path = root.join(crate::backend::LMDB_DIR); + let opened = tokio::task::spawn_blocking(move || { + let environment = LmdbEnv::open_existing(&path, LMDB_MAP_SIZE)?; + let handle = environment.env().clone(); + let transaction = handle.read_txn()?; + let databases = Databases::open(&handle, &transaction)?; + Ok::<_, StorageError>((environment, databases)) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)??; + let (environment, databases) = opened; + + tracing::info!( + path = %environment.path().display(), + "the lmdb store is open for reading" + ); + + Ok(Self { + env: environment.env().clone(), + databases, + settings: None, + }) + } + /// How much of the fixed map the store has used, for an operator's tooling and the log /// line [`open`](Self::open) writes. /// From bb508d1afd532986648299179c24f84f6abd46bc Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:58:09 -0500 Subject: [PATCH 21/49] fix(cli): verify a store without creating one --- crates/nostream-cli/src/commands/migrate.rs | 35 +++-- crates/nostream-cli/tests/migrate_cli.rs | 139 +++++++++++++++----- 2 files changed, 127 insertions(+), 47 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index dec1d5d3..8393ff62 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -49,17 +49,17 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result PathBuf { options.target.join(LMDB_DIR) } -/// Whether the target already holds an LMDB environment. +/// Whether the target already holds an LMDB environment, asked the way R8 asks it: `/ +/// lmdb/data.mdb` is a file. A directory alone is not a store, and neither is a lock file. fn store_present(options: &MigrateOptions) -> bool { - store_path(options).exists() + store_path(options).join(DATA_FILE).is_file() } +/// The data file heed creates inside an environment's directory. +const DATA_FILE: &str = "data.mdb"; + /// The first table of a manifest-less target that already holds a row, if any: one row read per /// table, which is the probe R7 asks for. async fn first_occupied_table( @@ -484,13 +488,20 @@ fn refuse_zero_batch(options: &MigrateOptions) -> Result<(), CliError> { Ok(()) } -/// Opens the store a run reads or writes. +/// Opens the store a copy writes, creating it when it is not there. async fn open_target(root: &Path) -> Result { LmdbBackend::open(root) .await .map_err(|error| CliError::Migrate(error.to_string())) } +/// Opens the store a verify reads, read-only and creating nothing (R8). +async fn open_existing_target(root: &Path) -> Result { + LmdbBackend::open_existing(root) + .await + .map_err(|error| CliError::Migrate(error.to_string())) +} + /// A failure of one table's read or write, naming the table so an operator knows which row /// stopped the run. fn table_failed(table: MigratedTable, error: &dyn std::fmt::Display) -> CliError { diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 54a2c899..4d58c715 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -300,24 +300,6 @@ impl MigrationSource for StubSource { } } -/// A target a verify must never touch: every method panics. -struct UntouchedTarget; - -impl MigrationTarget for UntouchedTarget { - fn write_batch<'a>(&'a self, _rows: &'a [MigrationRow]) -> StorageFuture<'a, u64> { - panic!("verify must not write to the store") - } - - fn scan_batch<'a>( - &'a self, - _table: MigratedTable, - _after: Option<&'a [u8]>, - _limit: usize, - ) -> StorageFuture<'a, Vec> { - panic!("verify must answer a missing manifest before it scans") - } -} - // --------------------------------------------------------------------------------------- // Helpers. // --------------------------------------------------------------------------------------- @@ -782,20 +764,91 @@ async fn verify_accepts_the_runs_own_target_and_rejects_a_changed_row() { // AC6: no manifest. // --------------------------------------------------------------------------------------- -#[tokio::test] -async fn verify_rejects_a_missing_manifest_with_a_message_and_exit_1() { +/// The directory's entries with their kind, sorted by name: what a verify must leave as it found +/// it. +fn entries(path: &std::path::Path) -> Vec<(String, bool)> { + let mut entries: Vec<(String, bool)> = std::fs::read_dir(path) + .expect("the directory reads") + .map(|entry| { + let entry = entry.expect("an entry reads"); + ( + entry.file_name().to_string_lossy().into_owned(), + entry.file_type().expect("the entry's type").is_dir(), + ) + }) + .collect(); + entries.sort(); + entries +} + +// `run` builds a runtime of its own, so this test cannot be a `#[tokio::test]`: the async half +// is driven by a runtime this test owns, and the two `run` calls happen outside any `block_on`. +#[test] +fn verify_creates_nothing_and_rejects_a_target_with_no_store_or_no_manifest() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the test runtime builds"); let directory = tempfile::tempdir().expect("a temporary directory"); - let options = MigrateOptions { - verify: true, - ..options(directory.path(), 1_000) - }; + // Half one: a target with no store. The listing is the same afterwards. + let before = entries(directory.path()); let mut out = Vec::new(); - let output = migrate::verify_with(&UntouchedTarget, &options, &mut out) - .await - .expect("a missing manifest is an answer, not a failure"); + let output = migrate::run( + MigrateOptions { + verify: true, + ..options(directory.path(), 1_000) + }, + &mut out, + ) + .expect("a missing store is an answer, not a failure"); assert_eq!(output.exit_code, 1); + let text = output_text(out); + assert!(text.contains("No store at"), "got: {text}"); + assert!( + text.contains(&directory.path().join("lmdb").display().to_string()), + "the line names the store it looked for: {text}" + ); + assert_eq!(entries(directory.path()), before, "verify created nothing"); + + // Half two: a store a copy filled, and whose manifest has since been removed. + let copy = options(directory.path(), 1_000); + let (usage, counts) = runtime.block_on(async { + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + ©, + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + let usage = target.map_usage().await.expect("the usage reads").used_bytes; + let mut counts = Vec::new(); + for table in MigratedTable::ALL { + counts.push(scan_all(&target, table).await.len()); + } + // R8: heed answers `BadOpenOptions` for a second open of one path with different + // options, so the writable handle is dropped before the verify opens the store + // read-only. + drop(target); + (usage, counts) + }); + std::fs::remove_file(directory.path().join("migration-manifest.json")) + .expect("the manifest is removed"); + let mut out = Vec::new(); + let output = migrate::run( + MigrateOptions { + verify: true, + ..options(directory.path(), 1_000) + }, + &mut out, + ) + .expect("a missing manifest is an answer, not a failure"); + assert_eq!(output.exit_code, 1); let text = output_text(out); assert!(text.contains("No manifest at"), "got: {text}"); assert!( @@ -806,15 +859,31 @@ async fn verify_rejects_a_missing_manifest_with_a_message_and_exit_1() { .display() .to_string() ), - "the line names the path it looked for: {text}" - ); - assert_eq!( - std::fs::read_dir(directory.path()) - .expect("the directory reads") - .count(), - 0, - "verify created nothing" + "the line names the manifest it looked for: {text}" ); + + runtime.block_on(async { + let reopened = LmdbBackend::open(directory.path()) + .await + .expect("the store reopens"); + for (table, count) in MigratedTable::ALL.iter().zip(counts) { + assert_eq!( + scan_all(&reopened, *table).await.len(), + count, + "{} is unchanged", + table.name() + ); + } + assert_eq!( + reopened + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + usage, + "the store's bytes are unchanged" + ); + }); } // --------------------------------------------------------------------------------------- From b43b6ee4371fa1aef50e749837232717fc07dbad Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:58:10 -0500 Subject: [PATCH 22/49] docs: record the fresh manifest and the read-only verify --- .../rewrite/decisions/0012-pg-to-lmdb-migration.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index c781ea5c..6b85f7fb 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -110,6 +110,12 @@ temporary file and a rename. A crash between the two therefore replays at most o which is harmless because every write in it is a put of the source's own key and value: the same rows produce the same count and the same sum, so the merged entry is unchanged. +For the first batch that replay has nothing to fall back on, so the run writes the manifest +fresh before its first read: every table `rows: 0`, the zero checksum, `after: ""`, +`complete: false`. A store this tool has begun to fill therefore always carries a manifest, and +the refusal of a store that is not this tool's stays what it should be — a judgement about rows +that appeared without one, not about a crash. + ## The command surface (D6) `nostream-cli migrate pg-to-lmdb` takes `--target `, `--batch-size ` and `--verify`, @@ -119,6 +125,14 @@ the manifest lives under it. `--batch-size` defaults to 1 000, the import's defa minimum is 1: a batch of zero would never advance a cursor. `--verify` copies nothing and reads only the target. +Verify creates nothing, in the store or beside it. It requires `/lmdb/data.mdb` to be a +file before it opens anything, and opens through `LmdbBackend::open_existing`: a read-only +environment, no `create_dir_all`, every database opened by name rather than created. The +ordinary opener would answer a verify of a store that is not there with a fresh empty +environment and nine short tables — a check that reports corruption it caused. heed's +process-global registry answers `BadOpenOptions` for a second open of one path with different +options, so a caller already holding the root writable drops that handle first. + Exit codes are the CLI's own, from `CLI.md`: 0 on success, 2 for a usage error, 1 for every other failure, which is written to stderr as `Migration failed: {message}`. That message needs a `CliError` variant of its own — reusing `Import` would report `Import failed:` for a From 54673c49f4eb0af6a4c732a0abb551aeb32377a1 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 00:58:44 -0500 Subject: [PATCH 23/49] style: rustfmt the read-only opener and the resume test --- crates/nostream-cli/tests/migrate_cli.rs | 6 +++++- crates/nostream-storage/src/backend.rs | 5 +---- crates/nostream-storage/src/lmdb/mod.rs | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 4d58c715..27ed5237 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -825,7 +825,11 @@ fn verify_creates_nothing_and_rejects_a_target_with_no_store_or_no_manifest() { ) .await .expect("the copy succeeds"); - let usage = target.map_usage().await.expect("the usage reads").used_bytes; + let usage = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; let mut counts = Vec::new(); for table in MigratedTable::ALL { counts.push(scan_all(&target, table).await.len()); diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 4383f70c..0e828d3c 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -103,10 +103,7 @@ impl LmdbEnv { /// heed's process-global registry answers `Error::BadOpenOptions` for a second open of one /// path with different options, so a caller that already holds this path writable must /// drop that handle first. - pub fn open_existing( - path: impl Into, - map_size: usize, - ) -> Result { + pub fn open_existing(path: impl Into, map_size: usize) -> Result { let path = path.into(); // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on, and diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index a03df750..28b40934 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -191,7 +191,11 @@ impl Databases { dvm_jobs_by_status_created_at: open_named(env, txn, "dvm_jobs_by_status_created_at")?, reports: open_named(env, txn, "reports")?, reports_by_event_id: open_named(env, txn, "reports_by_event_id")?, - reports_actionable_by_created_at: open_named(env, txn, "reports_actionable_by_created_at")?, + reports_actionable_by_created_at: open_named( + env, + txn, + "reports_actionable_by_created_at", + )?, reports_actionable_event: open_named(env, txn, "reports_actionable_event")?, reports_actionable_pubkey: open_named(env, txn, "reports_actionable_pubkey")?, outbox: open_named(env, txn, "notification_outbox")?, From 51d6bf949dcc4e4cf5ff791782ab7a91e1eff3b1 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:01:53 -0500 Subject: [PATCH 24/49] refactor(storage): name the store's data file beside its directory --- crates/nostream-cli/src/commands/migrate.rs | 7 ++----- crates/nostream-storage/src/backend.rs | 4 ++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 8393ff62..d040c7fa 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -15,7 +15,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; -use nostream_storage::backend::LMDB_DIR; +use nostream_storage::backend::{LMDB_DATA_FILE, LMDB_DIR}; use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; use nostream_storage::{LmdbBackend, PostgresBackend}; use sha2::{Digest, Sha256}; @@ -454,12 +454,9 @@ fn store_path(options: &MigrateOptions) -> PathBuf { /// Whether the target already holds an LMDB environment, asked the way R8 asks it: `/ /// lmdb/data.mdb` is a file. A directory alone is not a store, and neither is a lock file. fn store_present(options: &MigrateOptions) -> bool { - store_path(options).join(DATA_FILE).is_file() + store_path(options).join(LMDB_DATA_FILE).is_file() } -/// The data file heed creates inside an environment's directory. -const DATA_FILE: &str = "data.mdb"; - /// The first table of a manifest-less target that already holds a row, if any: one row read per /// table, which is the probe R7 asks for. async fn first_occupied_table( diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 0e828d3c..1af1256c 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -18,6 +18,10 @@ use crate::lmdb::{LMDB_MAX_DBS, LMDB_MAX_READERS}; /// database it replaces. pub const LMDB_DIR: &str = "lmdb"; +/// The file heed keeps an environment's data in, under [`LMDB_DIR`]. The store never changes +/// it: another name would be a different environment to every existing reader of the path. +pub const LMDB_DATA_FILE: &str = "data.mdb"; + /// Resolves the data directory under `root` for one backend. A backend that keeps its data /// somewhere other than a directory of its own answers `None`. pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { From 6e639722969bb0975169fe560bdd0ef4b17320e5 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:02:13 -0500 Subject: [PATCH 25/49] fix(cli): refuse a manifest that does not describe the nine tables --- crates/nostream-cli/src/commands/migrate.rs | 37 ++++++++++- crates/nostream-cli/tests/migrate_cli.rs | 74 +++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index d040c7fa..14c5204e 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -9,7 +9,7 @@ //! crash between them replays one batch: every write is a put of the source's own key and //! value, so the replay is idempotent and the merged entry is unchanged. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; @@ -396,6 +396,15 @@ fn manifest_path(target: &Path) -> PathBuf { target.join(MANIFEST_FILE) } +/// A comma-separated list, or `none`, for a refusal that names what it found. +fn named_list(names: &[&str]) -> String { + if names.is_empty() { + "none".to_owned() + } else { + names.join(", ") + } +} + /// The manifest at the target, or `None` when there is none. fn read_manifest(target: &Path) -> Result, CliError> { let path = manifest_path(target); @@ -413,6 +422,32 @@ fn read_manifest(target: &Path) -> Result, CliError> { manifest.format ))); } + + // A manifest is this tool's only when it describes all nine tables and the two it leaves + // alone. `Manifest::get` answers an empty entry for a table the file does not name, so a + // partial file would restart every table at `rows: 0` over a store of unknown contents and + // skip R7's refusal entirely. + let named: BTreeSet<&str> = manifest.tables.keys().map(String::as_str).collect(); + let expected: BTreeSet<&str> = MigratedTable::ALL.iter().map(|table| table.name()).collect(); + if named != expected { + let missing: Vec<&str> = expected.difference(&named).copied().collect(); + let unknown: Vec<&str> = named.difference(&expected).copied().collect(); + return Err(CliError::Migrate(format!( + "{}: the manifest does not name this tool's nine tables (missing: {}; unknown: {})", + path.display(), + named_list(&missing), + named_list(&unknown) + ))); + } + if manifest.not_migrated != MigratedTable::NOT_MIGRATED { + let named: Vec<&str> = manifest.not_migrated.iter().map(String::as_str).collect(); + return Err(CliError::Migrate(format!( + "{}: the manifest's not_migrated is not the two tables this tool leaves alone (it \ + names: {})", + path.display(), + named_list(&named) + ))); + } Ok(Some(manifest)) } diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 27ed5237..0d5eaeb8 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1389,6 +1389,80 @@ async fn a_manifest_of_another_format_is_refused_before_anything_is_written() { assert!(error.to_string().contains("format 2"), "{error}"); } +#[tokio::test] +async fn a_manifest_that_does_not_name_the_nine_tables_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + // A store with three rows already in it: a partial manifest must not let a copy restart + // every table at `rows: 0` over them, which is what R7's refusal exists to prevent. + let seeded: Vec = (1..=3) + .map(|seed| MigrationRow::Event(event_record(seed, "seeded"))) + .collect(); + assert_eq!( + target.write_batch(&seeded).await.expect("the seed writes"), + 3 + ); + let manifest = directory.path().join("migration-manifest.json"); + + std::fs::write(&manifest, br#"{"format":1,"not_migrated":[],"tables":{}}"#) + .expect("the fixture manifest is written"); + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("a manifest that names no tables is refused"); + let message = error.to_string(); + assert!(message.contains("nine tables"), "got: {message}"); + for table in MigratedTable::ALL { + assert!( + message.contains(table.name()), + "the refusal names {}: {message}", + table.name() + ); + } + assert_eq!(source.calls().len(), 0, "nothing was read"); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 3, + "the seeded rows are untouched" + ); + + // The nine tables, and a `not_migrated` that is not this tool's. + let mut tables = serde_json::Map::new(); + for table in MigratedTable::ALL { + tables.insert( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}), + ); + } + let fixture = json!({"format": 1, "not_migrated": ["event_tags"], "tables": tables}); + std::fs::write( + &manifest, + serde_json::to_vec(&fixture).expect("the fixture serialises"), + ) + .expect("the fixture manifest is written"); + let error = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("the wrong not_migrated list is refused"); + assert!(error.to_string().contains("not_migrated"), "{error}"); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 3, + "the seeded rows are untouched" + ); +} + #[tokio::test] async fn a_zero_batch_is_refused_instead_of_looping() { let directory = tempfile::tempdir().expect("a temporary directory"); From ba41d9771ded83ef22fa64582120d1cb21bba463 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:02:14 -0500 Subject: [PATCH 26/49] test(cli): assert the seam's replaceable, outbox and status index writes --- crates/nostream-cli/tests/migrate_cli.rs | 153 ++++++++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 0d5eaeb8..6d4ec614 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -333,7 +333,7 @@ fn time_key(seconds: i64, id: EventId) -> Vec { } /// Every key in one named database, through the public environment handle. -fn database_keys(backend: &LmdbBackend, name: &str) -> Vec> { +fn database_entries(backend: &LmdbBackend, name: &str) -> Vec<(Vec, Vec)> { let txn = backend.env().read_txn().expect("a read transaction"); let database = backend .env() @@ -343,7 +343,17 @@ fn database_keys(backend: &LmdbBackend, name: &str) -> Vec> { database .iter(&txn) .expect("the database iterates") - .map(|entry| entry.expect("a stored entry").0.to_vec()) + .map(|entry| { + let (key, value) = entry.expect("a stored entry"); + (key.to_vec(), value.to_vec()) + }) + .collect() +} + +fn database_keys(backend: &LmdbBackend, name: &str) -> Vec> { + database_entries(backend, name) + .into_iter() + .map(|(key, _)| key) .collect() } @@ -1533,6 +1543,145 @@ fn verify_refuses_a_manifest_without_a_store_instead_of_creating_one() { } } +// --------------------------------------------------------------------------------------- +// The seam's index writes, asserted where the store keeps them. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_migrated_replaceable_event_writes_the_pointer_its_kind_index_is_keyed_by() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut record = event_record(7, "the first value"); + record.event.kind = Kind(10000); + // A stored, empty deduplication list: the author and kind alone, with no `d` element. + record.event_deduplication = Some(Vec::new()); + target + .write_batch(&[MigrationRow::Event(record.clone())]) + .await + .expect("the event writes"); + + let entries = database_entries(&target, "replaceable"); + assert_eq!(entries.len(), 1, "one pointer for the author and kind"); + let (key, value) = &entries[0]; + assert_eq!( + value.as_slice(), + record.event.id.as_bytes(), + "the pointer carries the event id" + ); + assert_eq!( + key.len(), + 36, + "pubkey ++ kind, with no deduplication element" + ); + assert_eq!(&key[..32], record.event.pubkey.as_bytes()); + + // A second event of the same author and kind replaces the pointer at the same key, so the + // key is the author and kind rather than the id. + let mut second = record.clone(); + second.event.id = event_id(9); + target + .write_batch(&[MigrationRow::Event(second.clone())]) + .await + .expect("the replacement writes"); + let entries = database_entries(&target, "replaceable"); + assert_eq!(entries.len(), 1, "one pointer per author and kind"); + assert_eq!(entries[0].0, *key, "the same key"); + assert_eq!(entries[0].1.as_slice(), second.event.id.as_bytes()); +} + +#[tokio::test] +async fn a_migrated_delivery_without_an_outbox_id_stores_no_outbox_pointer() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut orphan = delivery_log(1); + orphan.outbox_id = None; + let linked = delivery_log(2); + target + .write_batch(&[ + MigrationRow::DeliveryLogEntry(orphan.clone()), + MigrationRow::DeliveryLogEntry(linked.clone()), + ]) + .await + .expect("the rows write"); + + let rows = scan_all(&target, MigratedTable::NotificationDeliveryLog).await; + let stored = rows + .iter() + .find_map(|row| match row { + MigrationRow::DeliveryLogEntry(entry) if entry.id == orphan.id => Some(entry), + _ => None, + }) + .expect("the row without an outbox id is stored"); + assert_eq!(stored.outbox_id, None, "the NULL outbox id survives"); + assert_eq!(stored.event_type, orphan.event_type); + + let pointers = database_entries(&target, "delivery_log_by_outbox"); + assert_eq!(pointers.len(), 1, "only the row with an outbox id has one"); + assert!( + pointers[0] + .0 + .windows(linked.id.len()) + .any(|window| window == linked.id.as_bytes()), + "the pointer names the linked row" + ); +} + +#[tokio::test] +async fn a_migrated_job_writes_the_status_index_its_finder_reads() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + // Same `created_at`, one status apart, so the keys can differ only in the status. + let mut submitted = dvm_job(3); + submitted.status = DvmJobStatus::Submitted; + submitted.created_at = at(5); + let mut completed = dvm_job(4); + completed.created_at = at(5); + for job in [&submitted, &completed] { + target + .write_batch(&[MigrationRow::DvmJob(job.clone())]) + .await + .expect("the job writes"); + } + + let entries = database_entries(&target, "dvm_jobs_by_status_created_at"); + assert_eq!(entries.len(), 2, "one pointer per job"); + let pointer_for = |job: &DvmJob| { + entries + .iter() + .find(|(key, _)| key.ends_with(job.id.as_bytes())) + .map(|(key, value)| (key.clone(), value.clone())) + .expect("a pointer for the job") + }; + for job in [&submitted, &completed] { + let (_, value) = pointer_for(job); + assert!(value.is_empty(), "a status pointer carries no value"); + } + + // Both jobs have the same `created_at`, so the bytes before the id are the status: they + // differ, and the finder's status prefix is what the writer put there. + let submitted_prefix = { + let (key, _) = pointer_for(&submitted); + key[..key.len() - 32].to_vec() + }; + let completed_prefix = { + let (key, _) = pointer_for(&completed); + key[..key.len() - 32].to_vec() + }; + assert_ne!( + submitted_prefix, completed_prefix, + "the status leads the key, after the timestamp part" + ); +} + /// The process boundary the two failure tests above stop at: `main` prints /// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on /// the way to the copy, which is the same arm. From aed4ecdbacab884f2e43b9fefaf26ff6334c05f8 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:02:19 -0500 Subject: [PATCH 27/49] style: rustfmt the manifest check --- crates/nostream-cli/src/commands/migrate.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 14c5204e..611b0438 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -428,7 +428,10 @@ fn read_manifest(target: &Path) -> Result, CliError> { // partial file would restart every table at `rows: 0` over a store of unknown contents and // skip R7's refusal entirely. let named: BTreeSet<&str> = manifest.tables.keys().map(String::as_str).collect(); - let expected: BTreeSet<&str> = MigratedTable::ALL.iter().map(|table| table.name()).collect(); + let expected: BTreeSet<&str> = MigratedTable::ALL + .iter() + .map(|table| table.name()) + .collect(); if named != expected { let missing: Vec<&str> = expected.difference(&named).copied().collect(); let unknown: Vec<&str> = named.difference(&expected).copied().collect(); From 5abbba6ddf36c69f942f6b4191cb38d08ba1dc4b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:20:07 -0500 Subject: [PATCH 28/49] feat(cli): refuse a manifest that records progress over a missing store --- crates/nostream-cli/src/commands/migrate.rs | 34 +++- crates/nostream-cli/tests/migrate_cli.rs | 162 ++++++++++++++++++-- 2 files changed, 179 insertions(+), 17 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 611b0438..e617ee68 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -63,9 +63,14 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result bool { + self.tables + .values() + .any(|entry| entry.rows != 0 || !entry.after.is_empty() || entry.complete) + } + /// This table's entry, as the empty one when the manifest carries none. fn get(&self, table: MigratedTable) -> TableEntry { self.tables.get(table.name()).cloned().unwrap_or_default() diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 6d4ec614..5d900997 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -316,6 +316,27 @@ fn manifest_bytes(directory: &std::path::Path) -> Vec { std::fs::read(directory.join("migration-manifest.json")).expect("the manifest is written") } +/// The manifest of the shape R6 writes: nine entries, `rows: 0`, the zero checksum, `after: ""`, +/// and `complete` as the caller asks. +fn write_manifest_fixture(directory: &std::path::Path, complete: bool) { + let manifest = json!({ + "format": 1, + "not_migrated": MigratedTable::NOT_MIGRATED, + "tables": MigratedTable::ALL + .iter() + .map(|table| ( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": complete}), + )) + .collect::>(), + }); + std::fs::write( + directory.join("migration-manifest.json"), + serde_json::to_vec(&manifest).expect("the fixture manifest serialises"), + ) + .expect("the fixture manifest is written"); +} + fn manifest(directory: &std::path::Path) -> Value { serde_json::from_slice(&manifest_bytes(directory)).expect("the manifest is JSON") } @@ -1501,22 +1522,7 @@ async fn a_zero_batch_is_refused_instead_of_looping() { #[test] fn verify_refuses_a_manifest_without_a_store_instead_of_creating_one() { let directory = tempfile::tempdir().expect("a temporary directory"); - let manifest = json!({ - "format": 1, - "not_migrated": MigratedTable::NOT_MIGRATED, - "tables": MigratedTable::ALL - .iter() - .map(|table| ( - table.name().to_owned(), - json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": true}), - )) - .collect::>(), - }); - std::fs::write( - directory.path().join("migration-manifest.json"), - serde_json::to_vec(&manifest).expect("the fixture manifest serialises"), - ) - .expect("the fixture manifest is written"); + write_manifest_fixture(directory.path(), true); let store = directory.path().join("lmdb"); for verify in [true, false] { @@ -1682,6 +1688,130 @@ async fn a_migrated_job_writes_the_status_index_its_finder_reads() { ); } +// --------------------------------------------------------------------------------------- +// AC18: a manifest that records progress over a missing store. +// --------------------------------------------------------------------------------------- + +// The first half needs the binary's stderr and `run` builds a runtime of its own, so this is a +// plain test with a runtime it owns. +#[test] +fn a_manifest_that_records_progress_over_a_missing_store_is_refused() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the test runtime builds"); + + // Half one: a copy at --batch-size 2 that stops after its first batch, then its store gone. + let partial = tempfile::tempdir().expect("a temporary directory"); + let partial_options = options(partial.path(), 2); + let error = runtime.block_on(async { + let target = LmdbBackend::open(partial.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::failing_at(source_rows(), 1), + &target, + &partial_options, + &mut Vec::new(), + ) + .await + }); + assert!( + matches!(error, Err(nostream_cli::CliError::Migrate(_))), + "the stub fails the second read_batch" + ); + let manifest: Value = serde_json::from_slice(&manifest_bytes(partial.path())) + .expect("the partial run's manifest parses"); + let dvm = &manifest["tables"]["dvm_jobs"]; + assert_eq!(dvm["rows"], 2, "the first batch is recorded"); + assert!( + !dvm["after"].as_str().expect("a hex cursor").is_empty(), + "and so is its cursor" + ); + let store = partial.path().join("lmdb"); + std::fs::remove_dir_all(&store).expect("the store is removed"); + assert!(!store.exists()); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(partial.path()) + .args(["--batch-size", "2"]) + .env_clear() + .output() + .expect("the binary runs"); + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("Migration failed:"), "got: {stderr}"); + assert!( + stderr.contains(&store.display().to_string()), + "the line names /lmdb: {stderr}" + ); + assert!(!store.exists(), "the refused run created nothing"); + + // Half two: the manifest R6 writes, with no progress in it, over an empty directory. + let fresh = tempfile::tempdir().expect("a temporary directory"); + write_manifest_fixture(fresh.path(), false); + let fresh_store = fresh.path().join("lmdb"); + + // `run` takes the copy path rather than the guard: a source it cannot parse fails it, not a + // store, and it creates nothing. The URL is unparsable so no dial is attempted. + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(fresh.path()) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + assert_eq!( + output.status.code(), + Some(1), + "there is no database to read" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("No store"), + "a zero-progress manifest is not refused: {stderr}" + ); + assert!( + !stderr.contains(&fresh_store.display().to_string()), + "the failure is not about the store: {stderr}" + ); + assert!(!fresh_store.exists()); + + // And the copy that run would have driven completes over the same target. + let copy = options(fresh.path(), 2); + let (counts, complete) = runtime.block_on(async { + let target = LmdbBackend::open(fresh.path()) + .await + .expect("the store opens"); + let output = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + ©, + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + assert_eq!(output.exit_code, 0); + let mut counts = Vec::new(); + for table in MigratedTable::ALL { + counts.push(scan_all(&target, table).await.len()); + } + let manifest: Value = serde_json::from_slice(&manifest_bytes(fresh.path())) + .expect("the finished manifest parses"); + let complete = manifest["tables"] + .as_object() + .expect("the tables object") + .values() + .all(|entry| entry["complete"] == true); + (counts, complete) + }); + for count in counts { + assert_eq!(count, 3, "nine tables at three rows"); + } + assert!(complete, "every table is complete"); +} + /// The process boundary the two failure tests above stop at: `main` prints /// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on /// the way to the copy, which is the same arm. From 8d9086766eeac07e7c8193c45178aeb77f55e0a0 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:20:07 -0500 Subject: [PATCH 29/49] docs: record the refusal a progressed manifest needs --- docs/rewrite/decisions/0012-pg-to-lmdb-migration.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index 6b85f7fb..cc25c4da 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -116,6 +116,16 @@ fresh before its first read: every table `rows: 0`, the zero checksum, `after: " the refusal of a store that is not this tool's stays what it should be — a judgement about rows that appeared without one, not about a crash. +Two refusals guard the target, and they are the same judgement from two sides. A store that holds +rows and carries no manifest is not this tool's: copying over it would leave the displaced rows' +index entries behind, and verify reads the primary databases only, so the run refuses before it +reads anything. A manifest that records progress — a row counted, a cursor moved, a table +finished — is only meaningful beside the store that progress went into, so the run opens that +store read-only first and refuses a missing one instead of creating it; a copy over an empty +store would skip the tables already counted and exit 0 against a store missing every row the +manifest claims. A manifest with no progress cannot have written a row, so a fresh store beside +it is honest and the run starts one. + ## The command surface (D6) `nostream-cli migrate pg-to-lmdb` takes `--target `, `--batch-size ` and `--verify`, From 2bd48d931e921197d013934e839dacf63fdf0fa6 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:36:22 -0500 Subject: [PATCH 30/49] fix(migrate): make the manifest durable and the checksum add carry Two static findings, both from round 4. `write_manifest` renamed its temporary file without syncing the contents or the directory, so a power loss could leave `migration-manifest.json` as a name pointing at truncated JSON. `read_manifest` refuses that, and the refusal of a target that holds rows without a manifest then blocks the store it describes - the operator's only way forward is a discard. The file is now synced before the rename and the directory after it, and ADR 0012 D5 says so instead of claiming only "a temporary file and a rename". `RowSum::add` added its four words independently, dropping the carry between them, while the ADR documents the checksum as the sum modulo 2^256. Copy and verify agreed with each other so nothing was wrong in practice, but a checker written from the document computed a different number. The carry now crosses the word boundary - the loop runs from index 3 to 0 because limb 0 is the most significant - and three unit tests pin it: a wrap carrying upward, a full wrap to zero, and the ordering. The first two fail against the independent add. `a_failed_run_reports_migration_failed_and_exits_1` also took 60 s: with nothing on port 1 the connection is refused at once, but the pool still retried to the acquire timeout. `DB_ACQUIRE_CONNECTION_TIMEOUT=1000` takes the suite from 60.04 s to 2.29 s. --- crates/nostream-cli/src/commands/migrate.rs | 98 ++++++++++++++++++- crates/nostream-cli/tests/migrate_cli.rs | 3 + .../decisions/0012-pg-to-lmdb-migration.md | 6 +- 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index e617ee68..d7e453f6 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -390,9 +390,7 @@ impl RowSum { hasher.update(&canonical); let digest = hasher.finalize(); let (words, _) = digest.as_chunks::<8>(); - for (limb, word) in self.0.iter_mut().zip(words) { - *limb = limb.wrapping_add(u64::from_be_bytes(*word)); - } + sum_words(&mut self.0, words); Ok(()) } @@ -423,6 +421,24 @@ impl RowSum { } } +/// Adds four big-endian words into a 256-bit limb sum, modulo 2^256. +/// +/// The manifest documents the checksum as the sum of the row digests modulo 2^256 +/// (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`). Adding the four words +/// independently is a different number whenever a limb wraps, and an independent checker +/// written from the documentation would compute a checksum this tool calls wrong, so the +/// carry has to cross the word boundary. Limb 3 is the least significant — `to_hex` writes +/// limb 0 first — so the carry runs from index 3 up to 0, and the carry out of the top limb +/// is dropped, which is what makes the result the sum modulo 2^256. +fn sum_words(limbs: &mut [u64; 4], words: &[[u8; 8]]) { + let mut carry = 0u128; + for (index, word) in words.iter().enumerate().rev() { + let sum = u128::from(limbs[index]) + u128::from(u64::from_be_bytes(*word)) + carry; + limbs[index] = sum as u64; + carry = sum >> 64; + } +} + /// The manifest's path under a target root. fn manifest_path(target: &Path) -> PathBuf { target.join(MANIFEST_FILE) @@ -492,10 +508,35 @@ fn write_manifest(target: &Path, manifest: &Manifest) -> Result<(), CliError> { let temporary = path.with_extension("json.tmp"); let bytes = serde_json::to_vec_pretty(manifest) .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; - fs::write(&temporary, &bytes) + + // The rename is the commit point, so it needs both halves of the durability it implies: + // the bytes on disk before the name moves, and the directory entry durable after it. + // Without the file sync, a power loss can leave the new name pointing at a truncated + // manifest; `read_manifest` then refuses it, and the nonempty-target rule refuses to + // migrate over the store it describes, so the operator's only way forward is a discard. + let mut file = fs::File::create(&temporary) .map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?; + file.write_all(&bytes) + .and_then(|()| file.sync_all()) + .map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?; + drop(file); + fs::rename(&temporary, &path) - .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display()))) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + sync_directory_of(&path) +} + +/// Makes a rename inside a directory durable. +/// +/// A rename changes the directory, not the file, and an unsynced directory can lose that +/// entry on power loss even though the file's own content was synced. +fn sync_directory_of(path: &Path) -> Result<(), CliError> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let directory = fs::File::open(parent) + .map_err(|error| CliError::Migrate(format!("{}: {error}", parent.display())))?; + directory + .sync_all() + .map_err(|error| CliError::Migrate(format!("{}: {error}", parent.display()))) } /// The line and exit code a verify answers when the target has no manifest. @@ -591,3 +632,50 @@ fn decode_key(text: &str) -> Result>, CliError> { fn write_failed(error: std::io::Error) -> CliError { CliError::Migrate(format!("unable to write the migration's output: {error}")) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The documented checksum is the sum modulo 2^256, so a wrap in the least significant + /// limb carries into the one above it. Adding the four words independently — which is + /// what this replaced — gives a different number whenever a limb wraps, and a checker + /// written from the documentation would then disagree with the tool. + #[test] + fn a_wrapped_limb_carries_into_the_next_one() { + let mut limbs = [0u64, 0, 0, u64::MAX]; + let words = [[0u8; 8], [0u8; 8], [0u8; 8], 1u64.to_be_bytes()]; + + sum_words(&mut limbs, &words); + + assert_eq!( + limbs, + [0, 0, 1, 0], + "the carry left the least significant limb" + ); + } + + /// The carry out of the most significant limb is dropped, which is the `modulo 2^256` in + /// the documented definition: four saturated limbs plus one sum to zero, not to a carry. + #[test] + fn the_carry_out_of_the_top_limb_is_dropped() { + let mut limbs = [u64::MAX, u64::MAX, u64::MAX, u64::MAX]; + let words = [[0u8; 8], [0u8; 8], [0u8; 8], 1u64.to_be_bytes()]; + + sum_words(&mut limbs, &words); + + assert_eq!(limbs, [0, 0, 0, 0], "the sum wraps at 2^256"); + } + + /// Limb 0 is the most significant, because `to_hex` writes it first; a word added into + /// it must land there and not at the other end. + #[test] + fn the_first_word_is_the_most_significant_limb() { + let mut limbs = [0u64; 4]; + let words = [1u64.to_be_bytes(), [0u8; 8], [0u8; 8], [0u8; 8]]; + + sum_words(&mut limbs, &words); + + assert_eq!(limbs, [1, 0, 0, 0]); + } +} diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 5d900997..7900d730 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1822,6 +1822,9 @@ fn a_failed_run_reports_migration_failed_and_exits_1() { .args(["migrate", "pg-to-lmdb", "--target"]) .arg(directory.path()) .env("DB_URI", "postgres://127.0.0.1:1/nostream") + // Nothing listens on port 1, so the failure is immediate; without this the pool still + // retries to the 60 s acquire timeout, and this one test would be a minute of every run. + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") .output() .expect("the binary runs"); diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index cc25c4da..809c298c 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -106,7 +106,11 @@ count and the sum from the entry. The store commit and the manifest write cannot share a transaction through the public surface, so the batch is committed first and the manifest is replaced atomically second — a -temporary file and a rename. A crash between the two therefore replays at most one batch, +temporary file, synced before the rename, and the directory synced after it. Both syncs are +load-bearing: a rename is durable only when the directory entry is on disk, and a manifest +that survives as a name pointing at a truncated file is worse than a missing one, because +`read_manifest` refuses it and the target refusal below then blocks the store it describes. +A crash between the two therefore replays at most one batch, which is harmless because every write in it is a put of the source's own key and value: the same rows produce the same count and the same sum, so the merged entry is unchanged. From 6307de2e5ca3a02126738a334f76311d5b9190ed Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 01:45:02 -0500 Subject: [PATCH 31/49] fix(lmdb): commit the read transaction that opens the databases `open_existing` opened the nine databases inside a read transaction and dropped it. heed publishes a database handle only once its opening transaction has committed (`src/txn.rs`, quoted by its own documentation for exactly this case), so the handles stayed unusable and the next transaction on them answered `Invalid argument`. In practice that is every `--verify` and every automatic verification of a completed migration, because both open the store through this path in a fresh process. No test caught it because every in-process test opens the store writable first, where heed's process-global registry already holds valid handles. The new test closes that: it fills a store in the test process and then runs the binary, which has never opened it, and requires `Verified 9 tables, N rows`. Without the commit it fails with `Migration failed: dvm_jobs: lmdb operation failed: Invalid argument (os error 22)`; with it, 20 passed. ADR 0012 now states the requirement in the read-only opener's contract rather than leaving it to be rediscovered. --- crates/nostream-cli/tests/migrate_cli.rs | 54 +++++++++++++++++++ crates/nostream-storage/src/lmdb/mod.rs | 5 ++ .../decisions/0012-pg-to-lmdb-migration.md | 9 +++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 7900d730..064012d3 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1835,3 +1835,57 @@ fn a_failed_run_reports_migration_failed_and_exits_1() { "the failure is reported as one: {stderr}" ); } + +// --------------------------------------------------------------------------------------- +// A process that never opened the store verifies it. +// --------------------------------------------------------------------------------------- + +// Every other verify test runs in the process that opened the store writable, where heed's +// process-global registry already holds valid database handles, so a read-only opener that +// never commits its read transaction looks fine. The binary is the only way to get a process +// that has never opened the store, which is the shape a real `--verify` runs in. +#[test] +fn the_binary_verifies_a_store_this_process_wrote() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the test runtime builds"); + + let exit_code = runtime.block_on(async { + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let output = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + output.exit_code + }); + assert_eq!(exit_code, 0, "the copy completes"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .arg("--verify") + .env("DB_URI", "not-a-url") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(0), + "a fresh process verifies what the store holds: stdout {stdout} stderr {stderr}" + ); + assert!( + stdout.starts_with("Verified 9 tables, "), + "the verify reports its work: {stdout}" + ); +} diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 28b40934..8d15b0e0 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -294,6 +294,11 @@ impl LmdbBackend { let handle = environment.env().clone(); let transaction = handle.read_txn()?; let databases = Databases::open(&handle, &transaction)?; + // heed only publishes the database handles a transaction opened once that + // transaction commits; a dropped read transaction leaves them unusable, and a + // fresh process - which has no earlier committed open to inherit them from - + // answers `Invalid argument` on its first read. + transaction.commit()?; Ok::<_, StorageError>((environment, databases)) }) .await diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index 809c298c..e98d40d8 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -141,8 +141,13 @@ reads only the target. Verify creates nothing, in the store or beside it. It requires `/lmdb/data.mdb` to be a file before it opens anything, and opens through `LmdbBackend::open_existing`: a read-only -environment, no `create_dir_all`, every database opened by name rather than created. The -ordinary opener would answer a verify of a store that is not there with a fresh empty +environment, no `create_dir_all`, every database opened by name rather than created. That +opener commits the read transaction it opens those handles in, because heed publishes a +database handle only once its opening transaction has committed: a dropped read transaction +leaves the handles unusable, and the next transaction on them answers `Invalid argument` — +which a fresh process, having no earlier committed open to inherit them from, hits on its +first read. Every in-process test misses it, because the store was opened writable first. +The ordinary opener would answer a verify of a store that is not there with a fresh empty environment and nine short tables — a check that reports corruption it caused. heed's process-global registry answers `BadOpenOptions` for a second open of one path with different options, so a caller already holding the root writable drops that handle first. From f64fe2ab8b1c30cf177b950ae98fbee8388d6742 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:10:20 -0500 Subject: [PATCH 32/49] fix(migrate): ask about the store by its file, and pin the checksum The resume check opened the store read-only to decide whether it was there. heed keeps its environment registered for the life of the process, so the writable open that follows answered `BadOpenOptions` - the probe refusing a resume exactly when the store it wanted existed. It is the store's file that is asked about now, which is what `data.mdb` means and what verify already used, so nothing is opened and nothing is registered. Nothing in the suite could see it: the one resume test that runs the command fails on the source before the store is opened. `the_resume_check_leaves_the _store_openable` drives the check and the writable open in one process, in the order the run performs them; with the probe in place it fails `an environment is already opened with different options`, and with the file check it passes. Three more from the same round: - the documented checksum now has a golden value, computed outside the crate from one row's bytes on D4's framing (`len32 | key | canonical`, summed modulo 2^256). Removing the length prefix from `add` turns it red; the unit tests only ever pinned the carry. - "verify creates nothing" was false: a read-only LMDB open recreates `lock.mdb` beside `data.mdb` (measured), and refuses a directory it cannot write it in. The comment and ADR 0012 now say what it does. - the nine-name refusal had no fixture with the right count and a wrong name, so a count check would have passed it. One is added. `open_existing` also stops carrying its own `#[allow(unsafe_code)]`: both openers go through `LmdbEnv::open_with`, so the crate keeps the single audited boundary its CLAUDE.md names. Its doc comment repeated the false premise about dropping the handle freeing the path, as did the test comment that dropped one for that reason. `fe2823d`'s `phoenix@ricardocabral.go` stays as it is: it is published, no ruleset requires signatures, and rewriting a branch a lens has read costs more than the typo. --- crates/nostream-cli/src/commands/migrate.rs | 81 ++++++++++++++----- crates/nostream-cli/tests/migrate_cli.rs | 73 ++++++++++++++++- crates/nostream-storage/src/backend.rs | 50 +++++------- crates/nostream-storage/src/lmdb/mod.rs | 6 +- .../decisions/0012-pg-to-lmdb-migration.md | 13 ++- 5 files changed, 165 insertions(+), 58 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index d7e453f6..84fd49c9 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -50,9 +50,11 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result Result<(), CliError> { Ok(()) } +/// The store requirement R7 puts on a resume. +/// +/// A manifest that records progress is only meaningful beside the store the progress went into: +/// the writable open that follows this check creates a missing store, and the copy would then +/// skip the tables already counted and the rows already summed, exiting 0 against a store +/// missing every row the manifest claims. +/// +/// The question is asked by the store's file and not by opening it. A read-only open registers +/// the environment with heed process-wide, and the writable open of the same path in the same +/// process then answers `BadOpenOptions` — so a probe here would make a resume fail exactly +/// when its store is present. `tests::the_resume_check_leaves_the_store_openable` drives both +/// steps for that reason. +fn check_resume_store( + existing: Option<&Manifest>, + options: &MigrateOptions, +) -> Result<(), CliError> { + if existing.is_some_and(|manifest| manifest.records_progress()) && !store_present(options) { + return Err(CliError::Migrate(format!( + "No store at {}", + store_path(options).display() + ))); + } + Ok(()) +} + /// Opens the store a copy writes, creating it when it is not there. async fn open_target(root: &Path) -> Result { LmdbBackend::open(root) @@ -637,6 +651,33 @@ fn write_failed(error: std::io::Error) -> CliError { mod tests { use super::*; + /// The resume path's store check and the writable open that follows it run in one process, + /// so the check must not open the store: heed keeps an environment registered for the life + /// of the process, and a read-only open there makes the writable one answer + /// `BadOpenOptions` — a resume failing exactly when its store is present. Nothing but both + /// steps together can see it. + #[tokio::test] + #[allow(clippy::expect_used, clippy::unwrap_used)] + async fn the_resume_check_leaves_the_store_openable() { + let directory = tempfile::tempdir().expect("a temporary directory"); + LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let mut manifest = Manifest::fresh(); + manifest.entry(MigratedTable::Events).rows = 1; + let options = MigrateOptions { + target: directory.path().to_path_buf(), + batch_size: 1_000, + verify: false, + }; + + check_resume_store(Some(&manifest), &options) + .expect("the store the manifest needs is there"); + open_target(&options.target) + .await + .expect("the writable open after the check succeeds"); + } + /// The documented checksum is the sum modulo 2^256, so a wrap in the least significant /// limb carries into the one above it. Adding the four words independently — which is /// what this replaced — gives a different number whenever a limb wraps, and a checker diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 064012d3..8a9a7ca1 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -865,10 +865,6 @@ fn verify_creates_nothing_and_rejects_a_target_with_no_store_or_no_manifest() { for table in MigratedTable::ALL { counts.push(scan_all(&target, table).await.len()); } - // R8: heed answers `BadOpenOptions` for a second open of one path with different - // options, so the writable handle is dropped before the verify opens the store - // read-only. - drop(target); (usage, counts) }); std::fs::remove_file(directory.path().join("migration-manifest.json")) @@ -1492,6 +1488,41 @@ async fn a_manifest_that_does_not_name_the_nine_tables_is_refused() { 3, "the seeded rows are untouched" ); + + // Nine names, one of them swapped for a name that is not a table. A check that counted the + // entries would accept this, so the refusal has to be about which names they are. + let mut tables = serde_json::Map::new(); + for table in MigratedTable::ALL.iter().skip(1) { + tables.insert( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}), + ); + } + tables.insert( + "not_a_table".to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}), + ); + let fixture = + json!({"format": 1, "not_migrated": MigratedTable::NOT_MIGRATED, "tables": tables}); + std::fs::write( + &manifest, + serde_json::to_vec(&fixture).expect("the fixture serialises"), + ) + .expect("the fixture manifest is written"); + let error = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("one swapped name is refused even though the count is right"); + let message = error.to_string(); + assert!(message.contains("not_a_table"), "{message}"); + assert!( + message.contains(MigratedTable::ALL[0].name()), + "the refusal names the table that is missing: {message}" + ); } #[tokio::test] @@ -1889,3 +1920,37 @@ fn the_binary_verifies_a_store_this_process_wrote() { "the verify reports its work: {stdout}" ); } + +// --------------------------------------------------------------------------------------- +// The checksum is the number the document says it is. +// --------------------------------------------------------------------------------------- + +// D4 states the row sum as the sum of `SHA-256(len32 ‖ key ‖ canonical)` per row, modulo 2^256, +// and the value exists so an independent checker can be written from that sentence. This +// literal was computed outside this crate from a single `events` row's bytes on exactly that +// framing, so a missing length prefix or a different order fails here instead of passing every +// criterion: the unit tests pin `sum_words`, not `add`. +#[tokio::test] +async fn the_row_checksum_matches_an_independent_computation() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let source = StubSource::new(vec![MigrationRow::Event(event_record(1, "golden"))]); + migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + + let written = manifest(directory.path()); + let events = &written["tables"][MigratedTable::Events.name()]; + assert_eq!(events["rows"], 1, "one row was copied"); + assert_eq!( + events["checksum"], "d81f052af7bfeb879aca5d630db8730d7168d6ca5cb2f5fc59f1a1c749150fbe", + "the checksum an independent checker computes for this row" + ); +} diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 1af1256c..255e2111 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -77,24 +77,7 @@ impl LmdbEnv { path: path.clone(), source, })?; - - // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on: - // no env flags (LMDB's locking runs), transactions never escape the closure they are - // opened in, and heed's per-process dedup turns a second open into the same Env or - // BadOpenOptions. The network-filesystem precondition rests on the operator passing a - // local path; see the list above. - #[allow(unsafe_code)] - let env = unsafe { - EnvOpenOptions::new() - .map_size(map_size) - .max_dbs(LMDB_MAX_DBS) - .max_readers(LMDB_MAX_READERS) - .open(&path) - } - .map_err(|source| StorageError::OpenLmdb { - path: path.clone(), - source, - })?; + let env = Self::open_with(&path, map_size, EnvFlags::empty())?; Ok(Self { env, path }) } @@ -104,29 +87,40 @@ impl LmdbEnv { /// heed's error rather than a new file. `READ_ONLY` takes no write lock, and this /// constructor opens no transaction of its own. /// - /// heed's process-global registry answers `Error::BadOpenOptions` for a second open of one - /// path with different options, so a caller that already holds this path writable must - /// drop that handle first. + /// heed answers `Error::BadOpenOptions` for a second open of one path with different + /// options, and its registry holds the environment process-wide, so dropping a handle does + /// not free the path: a caller that already opened this path writable cannot then open it + /// read-only, or the other way round, and a caller that needs the writable one afterwards + /// must not open it here first. pub fn open_existing(path: impl Into, map_size: usize) -> Result { let path = path.into(); + let env = Self::open_with(&path, map_size, EnvFlags::READ_ONLY)?; + Ok(Self { env, path }) + } - // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on, and - // `READ_ONLY` makes the write half of them vacuous: no transaction is opened here at - // all. + /// The one place this crate opens an environment, so the `unsafe` precondition list above + /// is answered once and `open` and `open_existing` cannot drift apart. + fn open_with(path: &Path, map_size: usize, flags: EnvFlags) -> Result { + // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on: + // no env flags beyond `READ_ONLY` when the caller passes it (LMDB's locking runs, and + // the write half of the list is vacuous), transactions never escape the closure they + // are opened in, and heed's per-process dedup turns a second open into the same Env or + // BadOpenOptions. The network-filesystem precondition rests on the operator passing a + // local path; see the list above. #[allow(unsafe_code)] let env = unsafe { EnvOpenOptions::new() .map_size(map_size) .max_dbs(LMDB_MAX_DBS) .max_readers(LMDB_MAX_READERS) - .flags(EnvFlags::READ_ONLY) - .open(&path) + .flags(flags) + .open(path) } .map_err(|source| StorageError::OpenLmdb { - path: path.clone(), + path: path.to_path_buf(), source, })?; - Ok(Self { env, path }) + Ok(env) } /// The environment handle, for the repository implementations. diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 8d15b0e0..84e5cc9b 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -283,8 +283,10 @@ impl LmdbBackend { /// missing database is an error. /// /// heed's process-global registry answers `BadOpenOptions` for a second open of one path - /// with different options, so a caller that already holds this root writable must drop that - /// handle first. + /// with different options, and it keeps the environment registered for the life of the + /// process, so dropping a handle does not free the path: a process that holds this root + /// writable cannot also open it read-only, and one that needs the writable root afterwards + /// must not open it here first. pub async fn open_existing(root: impl Into) -> Result { let root = root.into(); // The same rule `data_dir` answers, from the same constant. diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index e98d40d8..de5be582 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -139,18 +139,23 @@ the manifest lives under it. `--batch-size` defaults to 1 000, the import's defa minimum is 1: a batch of zero would never advance a cursor. `--verify` copies nothing and reads only the target. -Verify creates nothing, in the store or beside it. It requires `/lmdb/data.mdb` to be a -file before it opens anything, and opens through `LmdbBackend::open_existing`: a read-only +Verify creates no store and no manifest: it requires `/lmdb/data.mdb` to be a file +before it opens anything, and opens through `LmdbBackend::open_existing`: a read-only environment, no `create_dir_all`, every database opened by name rather than created. That opener commits the read transaction it opens those handles in, because heed publishes a database handle only once its opening transaction has committed: a dropped read transaction leaves the handles unusable, and the next transaction on them answers `Invalid argument` — which a fresh process, having no earlier committed open to inherit them from, hits on its first read. Every in-process test misses it, because the store was opened writable first. -The ordinary opener would answer a verify of a store that is not there with a fresh empty +LMDB's read-only open is not entirely a read: it recreates `lock.mdb` beside an existing +`data.mdb` (measured), and a target directory it cannot create that file in is refused. The +ordinary opener would answer a verify of a store that is not there with a fresh empty environment and nine short tables — a check that reports corruption it caused. heed's process-global registry answers `BadOpenOptions` for a second open of one path with different -options, so a caller already holding the root writable drops that handle first. +options, and it holds the environment process-wide, so dropping a handle does not free the +path. That is why the resume check asks whether `data.mdb` is there instead of opening the +store read-only first: the writable open that follows would otherwise refuse the one case the +check exists to allow. Exit codes are the CLI's own, from `CLI.md`: 0 on success, 2 for a usage error, 1 for every other failure, which is written to stderr as `Migration failed: {message}`. That message From 255a09a9f50d6f08930d1f31291f09e293398e12 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:31:16 -0500 Subject: [PATCH 33/49] fix(migrate): refuse a store that lost the rows the manifest counts Asking about the store by its file tells a missing store from a present one, not a store this run wrote from one truncated or written by something else that still has a `data.mdb`: the copy would add the remaining rows to a table that never held the ones the manifest counts and report success over a store missing them. A table the manifest counts rows for must now hold at least one, which is one read per counted table. Verify would have caught it, but only if someone ran it. Two documents still described the probe round 6 removed: ADR 0012's D5 said the run "opens that store read-only first", and `backend.rs`'s `NO_LOCK` bullet claimed no env flags are passed and the crate never touches the lock file. `open_existing` passes `READ_ONLY`, and a read-only open recreates `lock.mdb`. The guarantee is unchanged; its evidence is now accurate. The six criterion-bearing branches the code lens could not see fail now can: each was inverted and each turned tests red - the complete-manifest branches in `run` and `migrate_with` (2 and 12 failures), the occupancy guard (5), the manifest write (1), the skip of finished tables (11). The sixth, the report sequence in `lmdb/metadata.rs`, turned nothing red, which was a real gap: a migrated report with id 7 left the next `create` returning 1. It now has a test that fails `left: 1, right: 8` when the comparison is inverted. --- crates/nostream-cli/src/commands/migrate.rs | 26 ++++++++++ crates/nostream-cli/tests/migrate_cli.rs | 45 +++++++++++++++++ crates/nostream-storage/src/backend.rs | 8 ++-- crates/nostream-storage/src/lmdb/mod.rs | 48 +++++++++++++++++++ .../decisions/0012-pg-to-lmdb-migration.md | 11 +++-- 5 files changed, 131 insertions(+), 7 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 84fd49c9..feeb1882 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -122,6 +122,32 @@ pub async fn migrate_with( ))); } + // The same premise from the other side: a manifest that counts rows must find them. Asking + // about the store by its file tells a missing store from a present one, not a store this run + // wrote from one truncated, or written by something else, that still has a `data.mdb`. The + // copy would then add the remaining rows to a table that never held the ones the manifest + // counts, and report success over a store missing them; verify would catch it, but only if + // someone runs it. One row tells an empty table, and only counted tables are read. + if let Some(recorded) = &existing { + for table in MigratedTable::ALL { + let counted = recorded.get(table).rows; + if counted == 0 { + continue; + } + let held = target + .scan_batch(table, None, 1) + .await + .map_err(|error| table_failed(table, &error))?; + if held.is_empty() { + return Err(CliError::Migrate(format!( + "{} counts {counted} rows and the store holds none: the store beside the \ + manifest is not the one the run wrote to", + table.name() + ))); + } + } + } + // R6: written before the first `read_batch`, so a store this tool has begun to fill always // carries a manifest and a crash after the first batch resumes rather than being refused. if existing.is_none() { diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 8a9a7ca1..426fa01d 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1954,3 +1954,48 @@ async fn the_row_checksum_matches_an_independent_computation() { "the checksum an independent checker computes for this row" ); } + +// --------------------------------------------------------------------------------------- +// A manifest that counts rows must find them in the store beside it. +// --------------------------------------------------------------------------------------- + +// Asking about the store by its file tells a missing store from a present one, not a store the +// run wrote from one that lost its tables: a `data.mdb` can be there with none of the nine +// tables in it, and the copy would then add the remaining rows to a table that never held the +// ones the manifest counts, reporting success over a store missing them. +#[tokio::test] +async fn a_manifest_that_counts_rows_over_a_store_without_them_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut tables = serde_json::Map::new(); + for table in MigratedTable::ALL { + let entry = if table.name() == MigratedTable::Events.name() { + json!({"rows": 5, "checksum": "ab".repeat(32), "after": "", "complete": false}) + } else { + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}) + }; + tables.insert(table.name().to_owned(), entry); + } + let fixture = + json!({"format": 1, "not_migrated": MigratedTable::NOT_MIGRATED, "tables": tables}); + std::fs::write( + directory.path().join("migration-manifest.json"), + serde_json::to_vec(&fixture).expect("the fixture serialises"), + ) + .expect("the fixture manifest is written"); + + let error = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("a store that never held the counted rows is refused"); + let message = error.to_string(); + assert!(message.contains("counts 5 rows"), "{message}"); + assert!(message.contains(MigratedTable::Events.name()), "{message}"); +} diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 255e2111..8762a8b9 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -52,9 +52,11 @@ pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { /// location before this environment is wired into settings. /// - **`NO_LOCK` requires self-managed concurrency; a broken lock file means /// synchronization issues and possible UB** — split responsibility. The code guarantees -/// its half: no env flags are passed (`EnvOpenOptions::new()` only), so `NO_LOCK` is -/// never in play and this crate never reads, writes or removes the lock file. The -/// operator must supply the rest: the data directory's lock file must not be broken. +/// its half: the only flag it passes is `READ_ONLY`, for the verify opener, so `NO_LOCK` +/// is never in play and LMDB's own locking runs. The lock file belongs to LMDB: this crate +/// never reads, writes or removes it, and a read-only open recreates a missing `lock.mdb` +/// beside `data.mdb`. The operator must supply the rest: the data directory's lock file +/// must not be broken. /// - **One `map_size` per path for the life of the process** — upheld by heed + propagated: /// heed's per-process `OPENED_ENV` registry returns the same `Env` for a second open with /// equal options and `Error::BadOpenOptions` when they differ, and `open` surfaces that diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 84e5cc9b..126cd40a 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -1088,6 +1088,54 @@ mod tests { } } + /// A report the migration carried across must not be allocated again: the batch raises the + /// `reports.next_id` sequence past the largest id it wrote, so the next `create` continues + /// from there instead of colliding with a migrated row. + #[tokio::test] + async fn a_migrated_report_raises_the_sequence_the_next_create_uses() { + use crate::migration::{MigrationRow, MigrationTarget}; + use crate::repositories::{NewReport, Report, ReportRepository, ReportType}; + + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let migrated = Report { + id: 7, + event_id: nostream_core::EventId::from_byte_array([21; 32]), + reporter_pubkey: nostream_core::Pubkey::from_byte_array([22; 32]), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + created_at: chrono::Utc::now(), + }; + MigrationTarget::write_batch(&backend, &[MigrationRow::Report(migrated)]) + .await + .expect("the batch writes"); + + let created = ReportRepository::create( + &backend, + &NewReport { + event_id: nostream_core::EventId::from_byte_array([23; 32]), + reporter_pubkey: nostream_core::Pubkey::from_byte_array([24; 32]), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }, + ) + .await + .expect("the report is created"); + + assert_eq!( + created.id, 8, + "the sequence continues past the migrated report" + ); + } + #[tokio::test] async fn the_store_reports_how_much_of_its_map_it_uses() { // The map never grows (LMDB_MAP_SIZE), so the approach to it is what an operator has to diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index de5be582..193f69bf 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -124,10 +124,13 @@ Two refusals guard the target, and they are the same judgement from two sides. A rows and carries no manifest is not this tool's: copying over it would leave the displaced rows' index entries behind, and verify reads the primary databases only, so the run refuses before it reads anything. A manifest that records progress — a row counted, a cursor moved, a table -finished — is only meaningful beside the store that progress went into, so the run opens that -store read-only first and refuses a missing one instead of creating it; a copy over an empty -store would skip the tables already counted and exit 0 against a store missing every row the -manifest claims. A manifest with no progress cannot have written a row, so a fresh store beside +finished — is only meaningful beside the store that progress went into, so the run requires +`data.mdb` to be a file there and refuses a missing one instead of creating it; a copy over an +empty store would skip the tables already counted and exit 0 against a store missing every row +the manifest claims. The file test cannot tell a store the run wrote from a truncated one that +still has a `data.mdb`, so a second refusal covers that shape: a table the manifest counts rows +for must hold at least one row, or the store beside the manifest is not the one the run wrote +to. A manifest with no progress cannot have written a row, so a fresh store beside it is honest and the run starts one. ## The command surface (D6) From f34d762e79abeb6ff635a956def73cf5373cf340 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:35:23 -0500 Subject: [PATCH 34/49] fix(migrate): keep the manifest private and refuse a redirectable target The manifest was created 0644 under a 022 umask and records the last key written per table. For `invite_codes` that key is the invite code itself - a bearer credential NIP-43 accepts - and the store's own files are heed's 0600, so the manifest was the world-readable copy of a secret it was the only copy of. It is now created exclusively at 0600, which closes a second hole with the same flag: a planted symlink at `migration-manifest.json.tmp` used to be written through and renamed into place. Anything at the temporary path is removed first and the file is then created with `create_new`, so a second plant fails instead of being followed. Nothing inspected the target either. A group- or world-writable target lets another local user pre-place `/lmdb/data.mdb` and read the whole dataset out of a file they own; a symlinked target sends the copy somewhere the operator did not name. Both are refused before anything is opened, and the summary now prints the store the copy landed in. Each of the three is proved red by the fix's absence: the manifest test fails on mode 0644 and on the followed link, and both target tests fail on their message when the refusal is removed. `.gitignore` gains root-anchored `/lmdb/` and `/migration-manifest.json`: `--target .` writes the dataset into a public repository's working tree. ADR 0012 also stops saying heed opens with no environment flags - the verify opener passes `READ_ONLY` - and records the manifest's mode and the target refusal. --- .gitignore | 4 + crates/nostream-cli/src/commands/migrate.rs | 66 +++++++++- crates/nostream-cli/tests/migrate_cli.rs | 114 +++++++++++++++++- .../decisions/0012-pg-to-lmdb-migration.md | 13 +- 4 files changed, 191 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index c2156dc1..41f1dd99 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,7 @@ coverage-report/ # Rust /target/ + +# A migration target is the dataset: `--target .` writes it here, and this repository is public. +/lmdb/ +/migration-manifest.json diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index feeb1882..02a600cf 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -11,7 +11,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; +use std::io; use std::io::Write; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -48,6 +50,8 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result, CliError> { } /// Replaces the manifest through a temporary file, so a reader never sees a half-written one. +/// +/// The file is created exclusively and written 0600, and both halves are load-bearing. The +/// manifest records the last key written per table, and for `invite_codes` that key is the +/// invite code itself - a bearer credential NIP-43 accepts - so the manifest must not be the +/// world-readable copy of it; the store's own files are heed's 0600. Creating exclusively +/// refuses an entry already at the temporary path, which is how a planted symlink would +/// otherwise redirect the write and be renamed into place. fn write_manifest(target: &Path, manifest: &Manifest) -> Result<(), CliError> { let path = manifest_path(target); let temporary = path.with_extension("json.tmp"); let bytes = serde_json::to_vec_pretty(manifest) .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + match fs::remove_file(&temporary) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(CliError::Migrate(format!( + "{}: {error}", + temporary.display() + ))); + } + } + // The rename is the commit point, so it needs both halves of the durability it implies: // the bytes on disk before the name moves, and the directory entry durable after it. // Without the file sync, a power loss can leave the new name pointing at a truncated // manifest; `read_manifest` then refuses it, and the nonempty-target rule refuses to // migrate over the store it describes, so the operator's only way forward is a discard. - let mut file = fs::File::create(&temporary) + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) .map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?; file.write_all(&bytes) .and_then(|()| file.sync_all()) @@ -554,6 +583,41 @@ fn sync_directory_of(path: &Path) -> Result<(), CliError> { .map_err(|error| CliError::Migrate(format!("{}: {error}", parent.display()))) } +/// Refuses a target another local user could write to or redirect. +/// +/// The copy writes the whole dataset into `/lmdb`, so a group- or world-writable target +/// lets another local user pre-place the store and read what lands in it, and a symlinked target +/// sends the copy somewhere the operator did not name. Both are refused before anything is +/// opened. A target that does not exist yet is left alone: `open_target` creates it under the +/// operator's own umask. +fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> { + for path in [&options.target, &store_path(options)] { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(CliError::Migrate(format!("{}: {error}", path.display()))); + } + }; + if metadata.file_type().is_symlink() { + return Err(CliError::Migrate(format!( + "{} is a symlink: the target must be the directory the copy lands in", + path.display() + ))); + } + let mode = metadata.permissions().mode(); + if mode & 0o022 != 0 { + return Err(CliError::Migrate(format!( + "{} is writable by group or other ({:04o}): the target must be a directory only \ + its owner can write to", + path.display(), + mode & 0o7777 + ))); + } + } + Ok(()) +} + /// The line and exit code a verify answers when the target has no manifest. fn no_manifest(target: &Path, out: &mut dyn Write) -> Result { writeln!(out, "No manifest at {}", manifest_path(target).display()).map_err(write_failed)?; diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 426fa01d..67d3d7e2 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1032,11 +1032,12 @@ fn indexed_event(index: u64) -> MigrationRow { } /// The two summary lines, parsed. -fn summary(text: &str) -> (u64, u64, u64) { +fn summary(text: &str) -> (u64, u64, u64, String) { let mut lines = text.lines(); let first = lines.next().expect("the rows line"); let second = lines.next().expect("the throughput line"); - assert_eq!(lines.next(), None, "exactly two summary lines"); + let third = lines.next().expect("the store line"); + assert_eq!(lines.next(), None, "exactly three summary lines"); let (rows_text, rest) = first .strip_prefix("Migrated ") @@ -1048,10 +1049,16 @@ fn summary(text: &str) -> (u64, u64, u64) { .and_then(|rest| rest.strip_suffix(" events/s")) .expect("the throughput line's shape"); + let store = third + .strip_prefix("Store: ") + .expect("the store line's shape") + .to_owned(); + ( rows_text.parse().expect("a row count"), ms_text.parse().expect("a millisecond count"), rate_text.parse().expect("a rate"), + store, ) } @@ -1095,7 +1102,12 @@ async fn the_run_prints_a_rate_consistent_with_its_own_events_and_milliseconds() migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut out) .await .expect("the copy succeeds"); - let (rows, ms, rate) = summary(&output_text(out)); + let (rows, ms, rate, store) = summary(&output_text(out)); + assert_eq!( + store, + directory.path().join("lmdb").display().to_string(), + "the summary names where the copy landed" + ); assert_eq!(rows, 1_000); assert!( @@ -1999,3 +2011,99 @@ async fn a_manifest_that_counts_rows_over_a_store_without_them_is_refused() { assert!(message.contains("counts 5 rows"), "{message}"); assert!(message.contains(MigratedTable::Events.name()), "{message}"); } + +// --------------------------------------------------------------------------------------- +// The target and the manifest are not readable by anyone else. +// --------------------------------------------------------------------------------------- + +// The manifest records the last key written per table, and for `invite_codes` that key is the +// invite code itself - a bearer credential NIP-43 accepts. The store's files are heed's 0600, so +// the manifest was the world-readable copy of it. The planted link is the second half: a +// temporary path that already exists must not be written through and renamed into place. +#[tokio::test] +async fn the_manifest_is_private_and_its_temporary_path_is_not_followed() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let victim = directory.path().join("victim"); + std::fs::write(&victim, b"not the manifest").expect("the victim file is written"); + let temporary = directory.path().join("migration-manifest.json.tmp"); + std::os::unix::fs::symlink(&victim, &temporary).expect("the link is planted"); + + migrate::migrate_with( + &StubSource::new(vec![MigrationRow::Event(event_record(1, "golden"))]), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + + assert_eq!( + std::fs::read(&victim).expect("the victim still reads"), + b"not the manifest", + "the planted link's target is untouched" + ); + let manifest = std::fs::symlink_metadata(directory.path().join("migration-manifest.json")) + .expect("the manifest exists"); + assert!( + !manifest.file_type().is_symlink(), + "the manifest is a file, not the link" + ); + assert_eq!( + manifest.permissions().mode() & 0o777, + 0o600, + "the manifest is readable by its owner alone" + ); +} + +// A group- or world-writable target lets another local user pre-place the store and read what +// lands in it, so the copy refuses before opening anything. +#[test] +fn a_group_writable_target_is_refused() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().expect("a temporary directory"); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o777)) + .expect("the target is made world-writable"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("writable by group or other"), + "the refusal says why: {stderr}" + ); +} + +// A symlinked target sends the copy somewhere the operator did not name. +#[test] +fn a_symlinked_target_is_refused() { + let root = tempfile::tempdir().expect("a temporary directory"); + let real = root.path().join("real"); + std::fs::create_dir(&real).expect("the real directory is made"); + let link = root.path().join("link"); + std::os::unix::fs::symlink(&real, &link).expect("the link is planted"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(&link) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("is a symlink"), "{stderr}"); +} diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index 193f69bf..5daed433 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -110,6 +110,10 @@ temporary file, synced before the rename, and the directory synced after it. Bot load-bearing: a rename is durable only when the directory entry is on disk, and a manifest that survives as a name pointing at a truncated file is worse than a missing one, because `read_manifest` refuses it and the target refusal below then blocks the store it describes. +The manifest is written 0600 and through an exclusively created temporary file: it records the +last key written per table, and for `invite_codes` that key is the invite code itself — a bearer +credential NIP-43 accepts — so a world-readable manifest would hand one out, and a planted link +at the temporary path must not be written through and renamed into place. A crash between the two therefore replays at most one batch, which is harmless because every write in it is a put of the source's own key and value: the same rows produce the same count and the same sum, so the merged entry is unchanged. @@ -135,6 +139,11 @@ it is honest and the run starts one. ## The command surface (D6) +A target that is a symlink, or writable by group or other, is refused before anything is +opened: the copy writes the dataset into `/lmdb`, so a writable target lets another +local user pre-place the store and read what lands in it, and a link sends the copy somewhere +the operator did not name. The target the run writes into is printed with the summary. + `nostream-cli migrate pg-to-lmdb` takes `--target `, `--batch-size ` and `--verify`, and nothing else. `--target` is required with no default: a default of the working directory would let a mistyped run write a store where none was meant, and verify needs it too because @@ -205,8 +214,8 @@ Both stores are documented preconditions of a run, printed in the command's help must be stopped on both. The reason is that nothing in this repository proves a second process may hold the LMDB -environment. LMDB's lock file would let one in, and heed opens with no environment flags, so -the locking protocol is active — but the daemon opens no backend of its own today, so +environment. LMDB's lock file would let one in, and heed opens with no flags beyond the +verify opener's `READ_ONLY`, so the locking protocol is active — but the daemon opens no backend of its own today, so sharing is untested rather than known-safe. The tool takes no lock of its own: a lock that nothing else honours is worse than a documented precondition, because an operator would read it as protection. From 8d80d97dedf8f4a5a4dbf7446a3dd1410e01ba9b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:41:17 -0500 Subject: [PATCH 35/49] fix(migrate): name the source, the store and the way back The run named none of the three things an operator needs when it goes wrong. The source: `PostgresBackend::from_env` falls back to `postgres@localhost:5432/postgres` when nothing is set, so an unconfigured run dialled whatever a local server held, and with nothing listening it waited out the pool's 60 s acquire timeout (measured: exactly 60.00 s, "pool timed out while waiting for an open connection"). The setting is now read and named first - `DB_URI`, else `DB_HOST`/`DB_NAME`, with the password dropped - and one row is read before the copy so an unreachable host is reported as the database it could not reach. The store: the summary's `Store:` line is canonicalised, so a target reached through a link shows the directory the copy actually landed in, and a `--target` that is a file is named as one instead of failing on `Not a directory (os error 20)`. The way back: the help says a re-run resumes and that deleting the manifest starts a new copy, and the refusal of a store the manifest counts rows for repeats it. `micros` and the `i32` key flip now live in one feature-free module, `crate::keys`, with `lmdb::encoding` re-exporting them and `postgres::migration` using the same inverse. They were restated three times, each with a comment explaining it had to compile without the other driver; a divergence there is a false checksum mismatch after a full copy has already run. Each guard is proved by its absence: without the source naming the test waits 60 s and fails on the pool message; without the canonicalise the link test reports the argument; without the password drop the credential is printed; without the help text the test has nothing to find. --- crates/nostream-cli/src/commands/migrate.rs | 96 +++++++++-- crates/nostream-cli/src/main.rs | 8 +- crates/nostream-cli/tests/migrate_cli.rs | 150 +++++++++++++++++- crates/nostream-storage/src/backend.rs | 18 +++ crates/nostream-storage/src/keys.rs | 30 ++++ crates/nostream-storage/src/lib.rs | 1 + crates/nostream-storage/src/lmdb/encoding.rs | 18 +-- crates/nostream-storage/src/lmdb/mod.rs | 4 +- crates/nostream-storage/src/migration.rs | 15 +- .../src/postgres/migration.rs | 7 +- .../decisions/0012-pg-to-lmdb-migration.md | 10 ++ 11 files changed, 312 insertions(+), 45 deletions(-) create mode 100644 crates/nostream-storage/src/keys.rs diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 02a600cf..5ced5d56 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -17,7 +17,6 @@ use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; -use nostream_storage::backend::{LMDB_DATA_FILE, LMDB_DIR}; use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; use nostream_storage::{LmdbBackend, PostgresBackend}; use sha2::{Digest, Sha256}; @@ -88,9 +87,15 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result<(), CliError> { .map_err(|error| CliError::Migrate(format!("{}: {error}", parent.display()))) } +/// The source the run reads, as the setting that named it: `db.internal:5432/nostream (from +/// DB_URI)`. An unconfigured source is refused here, because `from_env` would otherwise fall +/// back to a local server's `postgres` database and copy whatever it holds. +fn describe_source() -> Result { + if let Some(uri) = std::env::var("DB_URI") + .ok() + .filter(|uri| !uri.trim().is_empty()) + { + return Ok(format!("{} (from DB_URI)", without_password(&uri))); + } + match std::env::var("DB_HOST") { + Ok(host) => Ok(format!( + "{host}:{port}/{database} (from DB_HOST, DB_NAME)", + port = std::env::var("DB_PORT").unwrap_or_else(|_| "5432".to_owned()), + database = std::env::var("DB_NAME").unwrap_or_else(|_| "postgres".to_owned()), + )), + Err(_) => Err(CliError::Migrate( + "no source is configured: set DB_URI (for example \ + postgres://user:pass@host:5432/nostream), or DB_HOST and DB_NAME. Nothing was read." + .to_owned(), + )), + } +} + +/// A connection URI with the password dropped, so an error can name the source without printing +/// a credential into a terminal's scrollback. +fn without_password(uri: &str) -> String { + let Some((scheme, rest)) = uri.split_once("://") else { + return uri.to_owned(); + }; + let Some((userinfo, after)) = rest.rsplit_once('@') else { + return uri.to_owned(); + }; + let user = userinfo.split(':').next().unwrap_or(userinfo); + format!("{scheme}://{user}@{after}") +} + +/// Reads one row before the copy, so an unreachable source is reported as the source it is - +/// named by its setting, host and database - rather than as a pool acquire timeout. +async fn probe_source(source: &PostgresBackend, described: &str) -> Result<(), CliError> { + let budget = std::env::var("DB_ACQUIRE_CONNECTION_TIMEOUT") + .ok() + .and_then(|value| value.parse::().ok()) + .map(Duration::from_millis) + .unwrap_or(Duration::from_secs(15)); + match tokio::time::timeout(budget, source.read_batch(MigratedTable::ALL[0], None, 1)).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(error)) => Err(CliError::Migrate(format!( + "could not read the source at {described}: {error}" + ))), + Err(_) => Err(CliError::Migrate(format!( + "could not read the source at {described}: no connection within {} s", + budget.as_secs() + ))), + } +} + /// Refuses a target another local user could write to or redirect. /// /// The copy writes the whole dataset into `/lmdb`, so a group- or world-writable target @@ -605,6 +674,13 @@ fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> { path.display() ))); } + if !metadata.is_dir() { + return Err(CliError::Migrate(format!( + "{} is not a directory: --target takes the store root, and the environment is \ + created under it", + path.display() + ))); + } let mode = metadata.permissions().mode(); if mode & 0o022 != 0 { return Err(CliError::Migrate(format!( @@ -638,13 +714,13 @@ fn no_store(options: &MigrateOptions, out: &mut dyn Write) -> Result PathBuf { - options.target.join(LMDB_DIR) + nostream_storage::backend::store_dir(&options.target) } /// Whether the target already holds an LMDB environment, asked the way R8 asks it: `/ /// lmdb/data.mdb` is a file. A directory alone is not a store, and neither is a lock file. fn store_present(options: &MigrateOptions) -> bool { - store_path(options).join(LMDB_DATA_FILE).is_file() + nostream_storage::backend::store_exists(&options.target) } /// The first table of a manifest-less target that already holds a row, if any: one row read per @@ -693,8 +769,10 @@ fn check_resume_store( ) -> Result<(), CliError> { if existing.is_some_and(|manifest| manifest.records_progress()) && !store_present(options) { return Err(CliError::Migrate(format!( - "No store at {}", - store_path(options).display() + "No store at {}: the manifest records progress, so a re-run resumes once the store is \ + back; delete {} to start a new copy", + store_path(options).display(), + manifest_path(&options.target).display() ))); } Ok(()) diff --git a/crates/nostream-cli/src/main.rs b/crates/nostream-cli/src/main.rs index 07f2cec6..d657a189 100644 --- a/crates/nostream-cli/src/main.rs +++ b/crates/nostream-cli/src/main.rs @@ -94,13 +94,15 @@ enum ConfigCommand { #[derive(Debug, Subcommand)] enum MigrateCommand { - /// Copy every table from Postgres into an LMDB store. + /// Copy every table from Postgres into an LMDB store, with the relay stopped. /// /// The source is Postgres from `DB_URI`, else the `DB_*` parts. The target is a store root: /// the environment is `/lmdb` and the manifest `/migration-manifest.json`. /// - /// Nothing may write to either store while a run is in progress, so - /// the relay must be stopped on both before the first batch is read. + /// A re-run resumes where the manifest says the last one stopped; deleting + /// `/migration-manifest.json` starts a new copy. Nothing may write to either store + /// while a run is in progress, so the relay must be stopped on both before the first batch + /// is read. PgToLmdb { /// The store root (the environment lives under `/lmdb`). #[arg(long, value_name = "DIR")] diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 67d3d7e2..b400e750 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1105,8 +1105,11 @@ async fn the_run_prints_a_rate_consistent_with_its_own_events_and_milliseconds() let (rows, ms, rate, store) = summary(&output_text(out)); assert_eq!( store, - directory.path().join("lmdb").display().to_string(), - "the summary names where the copy landed" + std::fs::canonicalize(directory.path().join("lmdb")) + .expect("the store exists") + .display() + .to_string(), + "the summary names the canonical store the copy landed in" ); assert_eq!(rows, 1_000); @@ -1790,6 +1793,10 @@ fn a_manifest_that_records_progress_over_a_missing_store_is_refused() { "the line names /lmdb: {stderr}" ); assert!(!store.exists(), "the refused run created nothing"); + assert!( + stderr.contains("start a new copy"), + "the refusal names the remedy: {stderr}" + ); // Half two: the manifest R6 writes, with no progress in it, over an empty directory. let fresh = tempfile::tempdir().expect("a temporary directory"); @@ -2107,3 +2114,142 @@ fn a_symlinked_target_is_refused() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("is a symlink"), "{stderr}"); } + +// --------------------------------------------------------------------------------------- +// AC19: the source is named, and an unconfigured one fails at once. +// --------------------------------------------------------------------------------------- + +// `from_env` falls back to `postgres@localhost:5432/postgres` when nothing is set, so a run with +// no `DB_URI` dials whatever local server is listening and copies it, and when nothing is +// listening it waits out the pool's acquire timeout. +#[test] +fn an_unconfigured_source_is_refused_at_once() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let started = std::time::Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("DB_URI"), "the refusal names it: {stderr}"); + assert!(stderr.contains("DB_HOST"), "and the parts: {stderr}"); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the refusal does not wait out the pool: {:?}", + started.elapsed() + ); +} + +// An unreachable host is reported as the database it could not read, with the password dropped. +#[test] +fn an_unreachable_source_is_named_without_its_password() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://alice:secret@127.0.0.1:1/nostream") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("127.0.0.1:1/nostream"), + "the failure names the host and database it resolved to: {stderr}" + ); + assert!(stderr.contains("DB_URI"), "and the setting: {stderr}"); + assert!( + !stderr.contains("secret"), + "the password is not printed: {stderr}" + ); +} + +// AC20: the remedy is in the command's own help, not only in the behaviour. +#[test] +fn the_help_names_the_resume_and_the_restart() { + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--help"]) + .output() + .expect("the binary runs"); + + assert!(output.status.success(), "{}", output.status); + let help = String::from_utf8_lossy(&output.stdout); + assert!( + help.contains("resumes"), + "the help says a re-run resumes: {help}" + ); + assert!( + help.contains("starts a new copy"), + "and how to start over: {help}" + ); + assert!( + help.contains("relay must be stopped"), + "and that the relay must be stopped: {help}" + ); +} + +// AC7: the summary names the store the copy landed in, so a path that went through a link is +// visible as the real one rather than echoed back as the argument. +#[tokio::test] +async fn the_summary_names_the_canonical_store_not_the_argument() { + let root = tempfile::tempdir().expect("a temporary directory"); + let real = root.path().join("real"); + std::fs::create_dir(&real).expect("the real directory is made"); + let link = root.path().join("link"); + std::os::unix::fs::symlink(&real, &link).expect("the link is planted"); + let argument = link.join("store"); + std::fs::create_dir(&argument).expect("the target is made through the link"); + + let target = LmdbBackend::open(&argument).await.expect("the store opens"); + let mut out = Vec::new(); + migrate::migrate_with( + &StubSource::new(vec![MigrationRow::Event(event_record(1, "golden"))]), + &target, + &options(&argument, 1_000), + &mut out, + ) + .await + .expect("the copy succeeds"); + + let (_, _, _, store) = summary(&output_text(out)); + assert_ne!( + store, + argument.join("lmdb").display().to_string(), + "the argument went through a link, so it is not what is reported" + ); + assert_eq!( + store, + std::fs::canonicalize(real.join("store").join("lmdb")) + .expect("the store exists") + .display() + .to_string(), + "the summary names the real store" + ); +} + +// U4: a `--target` that is a file is named as one, rather than reported as the operating +// system's `Not a directory (os error 20)` from a path the operator never wrote. +#[test] +fn a_target_that_is_a_file_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let file = directory.path().join("not-a-dir"); + std::fs::write(&file, b"x").expect("the file is written"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(&file) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("is not a directory"), "{stderr}"); +} diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 8762a8b9..a02c0952 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -24,6 +24,24 @@ pub const LMDB_DATA_FILE: &str = "data.mdb"; /// Resolves the data directory under `root` for one backend. A backend that keeps its data /// somewhere other than a directory of its own answers `None`. +/// The store directory inside `root`: `/lmdb`. +/// +/// One home for the shape, so the openers, the CLI's target check and its messages cannot +/// disagree about where the store is. +pub fn store_dir(root: &Path) -> PathBuf { + root.join(LMDB_DIR) +} + +/// The `data.mdb` of the store in `root`, whether or not it is there. +pub fn store_file(root: &Path) -> PathBuf { + store_dir(root).join(LMDB_DATA_FILE) +} + +/// Whether `root` holds a store this crate opened: `data.mdb` is a file. +pub fn store_exists(root: &Path) -> bool { + store_file(root).is_file() +} + pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { match backend { EventStoreBackend::Lmdb => Some(root.join(LMDB_DIR)), diff --git a/crates/nostream-storage/src/keys.rs b/crates/nostream-storage/src/keys.rs new file mode 100644 index 00000000..6a3f7e38 --- /dev/null +++ b/crates/nostream-storage/src/keys.rs @@ -0,0 +1,30 @@ +//! The key codecs both backends and the migration seam share. +//! +//! A key's byte order *is* the sort order LMDB and PostgreSQL both rely on, so an arithmetic +//! slip here is not a compile error and not a visible wrong value: it is a cursor that seeks to +//! the wrong place, or a checksum that disagrees after a full copy. These lived in +//! `lmdb::encoding` and were restated in `migration` and `postgres::migration`, each with a +//! comment saying it had to compile without the other driver. One feature-free home removes the +//! restatement and the drift it invited; `lmdb::encoding` re-exports these so its callers are +//! unchanged. + +use chrono::{DateTime, Utc}; + +/// An `i32` ordered by its bytes, with the sign bit flipped so a negative value sorts before +/// zero rather than after every positive one. +pub(crate) fn i32_key(value: i32) -> [u8; 4] { + ((value as u32) ^ (1 << 31)).to_be_bytes() +} + +/// The inverse of [`i32_key`], and `None` when the bytes are not four long. The caller names the +/// table in its own error, so the message an operator sees stays the one that backend wrote. +pub(crate) fn i32_from_key(bytes: &[u8]) -> Option { + let raw = u32::from_be_bytes(bytes.try_into().ok()?); + Some((raw ^ (1 << 31)) as i32) +} + +/// The instant a row carries, truncated to the microsecond precision every PostgreSQL +/// `timestamp` column has, so a value written by either backend reads back the same. +pub(crate) fn micros(value: DateTime) -> DateTime { + DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) +} diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index 6b61e3b2..d87c3ca8 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -34,6 +34,7 @@ pub mod backend; pub mod error; pub mod event_store; +mod keys; #[cfg(feature = "lmdb")] pub mod lmdb; pub mod migration; diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index d60eafe0..16ccfd17 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -53,11 +53,7 @@ pub(crate) fn invalid(table: &'static str, message: String) -> StorageError { } } -/// The instant a row carries, truncated to the microsecond precision every PostgreSQL -/// `timestamp` column has, so a value written by either backend reads back the same. -pub(crate) fn micros(value: DateTime) -> DateTime { - DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) -} +pub(crate) use crate::keys::{i32_key, micros}; /// An `i64` ordered by its bytes: the sign bit is flipped so a negative value sorts before /// zero rather than after every positive one. @@ -71,15 +67,11 @@ pub(crate) fn i64_from_key(bytes: &[u8]) -> Result { Ok((raw ^ (1 << 63)) as i64) } -/// An `i32` ordered by its bytes, sign-flipped for the same reason as [`i64_key`]. -pub(crate) fn i32_key(value: i32) -> [u8; 4] { - ((value as u32) ^ (1 << 31)).to_be_bytes() -} - -/// The inverse of [`i32_key`]. +/// The inverse of [`i32_key`]. The arithmetic lives in `crate::keys` with the forward one; the +/// error is this module's, so the message an operator sees is unchanged. pub(crate) fn i32_from_key(bytes: &[u8]) -> Result { - let raw = u32::from_be_bytes(fixed::<4>("events", bytes)?); - Ok((raw ^ (1 << 31)) as i32) + crate::keys::i32_from_key(bytes) + .ok_or_else(|| invalid("events", format!("4 bytes expected, {} found", bytes.len()))) } fn fixed(table: &'static str, bytes: &[u8]) -> Result<[u8; N], StorageError> { diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 126cd40a..fd134da0 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -245,8 +245,8 @@ impl LmdbBackend { /// write. pub async fn open(root: impl Into) -> Result { let root = root.into(); - // The same rule `data_dir` answers, from the same constant. - let path = root.join(crate::backend::LMDB_DIR); + // The same shape `data_dir` answers, from the one home for it. + let path = crate::backend::store_dir(&root); let opened = tokio::task::spawn_blocking(move || { let environment = LmdbEnv::open(&path, LMDB_MAP_SIZE)?; let handle = environment.env().clone(); diff --git a/crates/nostream-storage/src/migration.rs b/crates/nostream-storage/src/migration.rs index e14809a2..483bd0e6 100644 --- a/crates/nostream-storage/src/migration.rs +++ b/crates/nostream-storage/src/migration.rs @@ -252,18 +252,9 @@ pub trait MigrationTarget: Send + Sync { ) -> StorageFuture<'a, Vec>; } -/// The instant at the microsecond precision every PostgreSQL `timestamp` column has. -/// -/// The store's value codec truncates the same way (`lmdb::encoding::micros`); this module -/// restates it because it has to compile with no driver. -fn micros(value: DateTime) -> DateTime { - DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) -} +use crate::keys::micros; -/// The `reports.id` key: an `i32` ordered by its bytes, the sign bit flipped so a negative value -/// sorts before zero. -/// -/// The store's `lmdb::encoding::i32_key`, restated for the same reason as [`micros`]. +/// The `reports.id` key, in the one home the key codecs share (`crate::keys`). fn report_key(id: i32) -> [u8; 4] { - ((id as u32) ^ (1 << 31)).to_be_bytes() + crate::keys::i32_key(id) } diff --git a/crates/nostream-storage/src/postgres/migration.rs b/crates/nostream-storage/src/postgres/migration.rs index e6ea85d2..63c2df04 100644 --- a/crates/nostream-storage/src/postgres/migration.rs +++ b/crates/nostream-storage/src/postgres/migration.rs @@ -145,14 +145,13 @@ fn text(table: MigratedTable, after: &[u8]) -> Result { }) } -/// One cursor as the `i32` `reports.id` holds, undoing the key's sign flip. +/// One cursor as the `i32` `reports.id` holds, through the shared key codec. fn int32_key(table: MigratedTable, after: &[u8]) -> Result { - let bytes: [u8; 4] = after.try_into().map_err(|_| StorageError::InvalidRow { + crate::keys::i32_from_key(after).ok_or_else(|| StorageError::InvalidRow { table: table.name(), column: "key", value: format!("{} bytes are not a reports key", after.len()), - })?; - Ok((u32::from_be_bytes(bytes) ^ (1 << 31)) as i32) + }) } /// One page of rows, through the decoder the backend already has for that table. diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index 5daed433..243de0b9 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -139,6 +139,12 @@ it is honest and the run starts one. ## The command surface (D6) +The source is named before it is dialled. With nothing configured, `PostgresBackend::from_env` +falls back to `postgres@localhost:5432/postgres`, so the run refuses at once and names `DB_URI` +and the `DB_*` parts rather than copying whatever a local server happens to hold; a source that +cannot be reached is reported as the database it could not read — host and database, password +dropped — instead of as the pool's acquire timeout a minute later. + A target that is a symlink, or writable by group or other, is refused before anything is opened: the copy writes the dataset into `/lmdb`, so a writable target lets another local user pre-place the store and read what lands in it, and a link sends the copy somewhere @@ -169,6 +175,10 @@ path. That is why the resume check asks whether `data.mdb` is there instead of o store read-only first: the writable open that follows would otherwise refuse the one case the check exists to allow. +A re-run resumes wherever the manifest says the last one stopped; deleting +`/migration-manifest.json` starts a new copy, and the command's help and its refusal of +a store the manifest counts rows for both say so. + Exit codes are the CLI's own, from `CLI.md`: 0 on success, 2 for a usage error, 1 for every other failure, which is written to stderr as `Migration failed: {message}`. That message needs a `CliError` variant of its own — reusing `Import` would report `Import failed:` for a From f5b720d10b67af10ac1b146876298ba2a5e55e9a Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:46:26 -0500 Subject: [PATCH 36/49] fix(migrate): drop the query string too when naming the source `without_password` removed the userinfo's password and left everything else, so a URI that carries the credential as a query parameter - `?password=secret` - was printed whole into the error that reaches stderr and any captured log. The query string is now dropped as well, not matched against credential-shaped keys: a percent-encoded key ends a name-based check, and the description exists to identify the source, which the host and database already do. Proved by absence: with the query left in, the test prints `postgres://127.0.0.1:1/nostream?password=secret&sslmode=require` verbatim. --- crates/nostream-cli/src/commands/migrate.rs | 23 +++++++++++----- crates/nostream-cli/tests/migrate_cli.rs | 29 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 5ced5d56..0ae246db 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -619,17 +619,26 @@ fn describe_source() -> Result { } } -/// A connection URI with the password dropped, so an error can name the source without printing -/// a credential into a terminal's scrollback. +/// A connection URI with its credentials dropped, so an error can name the source without +/// printing a password into a terminal's scrollback or a captured log. +/// +/// Two places a URI can carry one are removed: the password in the userinfo, and the whole query +/// string, which may hold `?password=`. Matching credential-shaped keys instead would miss a +/// percent-encoded name, and a description exists to identify the source, which host and +/// database already do. fn without_password(uri: &str) -> String { let Some((scheme, rest)) = uri.split_once("://") else { return uri.to_owned(); }; - let Some((userinfo, after)) = rest.rsplit_once('@') else { - return uri.to_owned(); - }; - let user = userinfo.split(':').next().unwrap_or(userinfo); - format!("{scheme}://{user}@{after}") + let (rest, _query) = rest.split_once('?').unwrap_or((rest, "")); + let rest = rest.split('#').next().unwrap_or(rest); + match rest.rsplit_once('@') { + Some((userinfo, after)) => { + let user = userinfo.split(':').next().unwrap_or(userinfo); + format!("{scheme}://{user}@{after}") + } + None => format!("{scheme}://{rest}"), + } } /// Reads one row before the copy, so an unreachable source is reported as the source it is - diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index b400e750..a7a90bd5 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2170,6 +2170,35 @@ fn an_unreachable_source_is_named_without_its_password() { ); } +// A URI can carry the password in a query parameter instead of the userinfo, and the description +// is printed into an error that reaches stderr and any captured log. +#[test] +fn a_query_string_password_is_not_printed() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env( + "DB_URI", + "postgres://127.0.0.1:1/nostream?password=secret&sslmode=require", + ) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("127.0.0.1:1/nostream"), + "the failure still names the source: {stderr}" + ); + assert!( + !stderr.contains("secret"), + "the query-string password is not printed: {stderr}" + ); +} + // AC20: the remedy is in the command's own help, not only in the behaviour. #[test] fn the_help_names_the_resume_and_the_restart() { From be82e5eebc7cfa58500be62c2c724918e871b2cf Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:47:39 -0500 Subject: [PATCH 37/49] fix(migrate): name only the host and database in a source error The description kept the user name after the password went. AC19 asks for the host and database, a role is neither, and an identifier carried into a message that does not need it is a smaller copy of the problem the redaction was written to fix. The userinfo, the query string and the fragment all go; a permission error still names the role, because the server's own message says it. --- crates/nostream-cli/src/commands/migrate.rs | 27 +++++++++------------ crates/nostream-cli/tests/migrate_cli.rs | 4 +++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 0ae246db..04eff051 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -603,7 +603,7 @@ fn describe_source() -> Result { .ok() .filter(|uri| !uri.trim().is_empty()) { - return Ok(format!("{} (from DB_URI)", without_password(&uri))); + return Ok(format!("{} (from DB_URI)", without_credentials(&uri))); } match std::env::var("DB_HOST") { Ok(host) => Ok(format!( @@ -620,25 +620,22 @@ fn describe_source() -> Result { } /// A connection URI with its credentials dropped, so an error can name the source without -/// printing a password into a terminal's scrollback or a captured log. +/// printing a password - or a role - into a terminal's scrollback or a captured log. /// -/// Two places a URI can carry one are removed: the password in the userinfo, and the whole query -/// string, which may hold `?password=`. Matching credential-shaped keys instead would miss a -/// percent-encoded name, and a description exists to identify the source, which host and -/// database already do. -fn without_password(uri: &str) -> String { +/// Three things go: the userinfo, the query string and the fragment. The userinfo because the +/// message exists to name the host and database, and a role is neither; the query because it can +/// carry `?password=`, and matching credential-shaped keys would miss a percent-encoded name. +/// When a permission error is the problem, the server's own message names the role. +fn without_credentials(uri: &str) -> String { let Some((scheme, rest)) = uri.split_once("://") else { return uri.to_owned(); }; - let (rest, _query) = rest.split_once('?').unwrap_or((rest, "")); + let rest = rest.split('?').next().unwrap_or(rest); let rest = rest.split('#').next().unwrap_or(rest); - match rest.rsplit_once('@') { - Some((userinfo, after)) => { - let user = userinfo.split(':').next().unwrap_or(userinfo); - format!("{scheme}://{user}@{after}") - } - None => format!("{scheme}://{rest}"), - } + let rest = rest + .rsplit_once('@') + .map_or(rest, |(_userinfo, after)| after); + format!("{scheme}://{rest}") } /// Reads one row before the copy, so an unreachable source is reported as the source it is - diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index a7a90bd5..2ddd4381 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2168,6 +2168,10 @@ fn an_unreachable_source_is_named_without_its_password() { !stderr.contains("secret"), "the password is not printed: {stderr}" ); + assert!( + !stderr.contains("alice"), + "the role is not printed either, only the host and database: {stderr}" + ); } // A URI can carry the password in a query parameter instead of the userinfo, and the description From 201883dc50d0c44708c5c3735041ed79eaad73bf Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 02:54:17 -0500 Subject: [PATCH 38/49] fix(migrate): read the master only, and own the store directory Three findings, none of them from a test that existed before it. A replica that is offline must not stop a cutover. `from_env` dials the replica when `READ_REPLICA_ENABLED=true` and fails when it cannot reach it, which is right for the relay and wrong for a one-shot copy that reads the master and nothing else. `master_only_from_env` reads the master settings and none of the replica ones; the test proves it by pairing a zero replica count - which makes `from_env` refuse before it dials - with the master-only opener, which then fails on the connection instead. The help promised a restart that the tool refuses: "delete `migration-manifest.json` to start a new copy" leaves a store that holds rows and no manifest, which is exactly the shape `migrate_with` declines to copy over. It now says a fresh `--target` is what starting over takes, or both gone together. The store directory is created 0700 rather than at the umask's mercy. On a host with umask 002 `create_dir_all` leaves the dataset group-writable, which is a wider audience than heed's 0600 files - and is refused by the migration's own target check on the next run, so the tool would decline its own store. All three are proved by absence: the master-only test fails with `READ_REPLICAS must be at least 1` when the opener reads the replica settings, the help test has nothing to find when the sentence is the old one, and the directory test reads 0755 when the builder takes the umask. --- crates/nostream-cli/src/commands/migrate.rs | 10 +++-- crates/nostream-cli/src/main.rs | 9 ++-- crates/nostream-cli/tests/migrate_cli.rs | 4 +- crates/nostream-storage/src/backend.rs | 17 ++++++-- crates/nostream-storage/src/lmdb/mod.rs | 18 ++++++++ crates/nostream-storage/src/postgres/mod.rs | 46 +++++++++++++++++++++ 6 files changed, 91 insertions(+), 13 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 04eff051..1ea8eeab 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -92,9 +92,13 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result/lmdb` and the manifest `/migration-manifest.json`. /// - /// A re-run resumes where the manifest says the last one stopped; deleting - /// `/migration-manifest.json` starts a new copy. Nothing may write to either store - /// while a run is in progress, so the relay must be stopped on both before the first batch - /// is read. + /// A re-run resumes where the manifest says the last one stopped. Deleting + /// `/migration-manifest.json` does not restart a half-filled store: a store that holds + /// rows and no manifest is refused, so starting over takes a fresh `--target`, or the + /// manifest and the store removed together. Nothing may write to either store while a run + /// is in progress, so the relay must be stopped on both before the first batch is read. PgToLmdb { /// The store root (the environment lives under `/lmdb`). #[arg(long, value_name = "DIR")] diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 2ddd4381..ac8d09f8 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2218,8 +2218,8 @@ fn the_help_names_the_resume_and_the_restart() { "the help says a re-run resumes: {help}" ); assert!( - help.contains("starts a new copy"), - "and how to start over: {help}" + help.contains("fresh `--target`"), + "and what starting over actually takes: {help}" ); assert!( help.contains("relay must be stopped"), diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index a02c0952..6621f25d 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -1,6 +1,7 @@ //! Backend selection and the LMDB environment. Mirrors `src/utils/event-store.ts` (the //! selection half) and the storage boundary in `src/@types/adapters.ts`. +use std::os::unix::fs::DirBuilderExt; use std::path::{Path, PathBuf}; use nostream_core::EventStoreBackend; @@ -93,10 +94,18 @@ impl LmdbEnv { /// for the backend's named databases and its reader slots. pub fn open(path: impl Into, map_size: usize) -> Result { let path = path.into(); - std::fs::create_dir_all(&path).map_err(|source| StorageError::CreateDirectory { - path: path.clone(), - source, - })?; + // 0700, not the umask's answer: the directory holds a relay's whole dataset, and on a + // host whose umask is 002 `create_dir_all` leaves it group-writable, which is both a + // wider audience than the store's own 0600 files and something the migration's target + // check refuses on the next run. + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&path) + .map_err(|source| StorageError::CreateDirectory { + path: path.clone(), + source, + })?; let env = Self::open_with(&path, map_size, EnvFlags::empty())?; Ok(Self { env, path }) } diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index fd134da0..1ff3fbb1 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -1136,6 +1136,24 @@ mod tests { ); } + /// The store directory holds the whole dataset, so it is the owner's alone: heed writes the + /// files 0600, and a directory the umask left group-writable would both widen the audience + /// and be refused by the migration's own target check on the next run. + #[tokio::test] + async fn the_store_directory_is_the_owners_alone() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path().join("store"); + LmdbBackend::open(&root).await.expect("the store opens"); + let mode = std::fs::metadata(crate::backend::store_dir(&root)) + .expect("the store directory exists") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "group and other have no access"); + } + #[tokio::test] async fn the_store_reports_how_much_of_its_map_it_uses() { // The map never grows (LMDB_MAP_SIZE), so the approach to it is what an operator has to diff --git a/crates/nostream-storage/src/postgres/mod.rs b/crates/nostream-storage/src/postgres/mod.rs index d81506fe..bf6e164d 100644 --- a/crates/nostream-storage/src/postgres/mod.rs +++ b/crates/nostream-storage/src/postgres/mod.rs @@ -205,6 +205,25 @@ impl PostgresBackend { Self::with_pools(utc(master), bounds, replica).await } + /// Connects the master pool only, reading none of the replica settings. + /// + /// The migration copies from the master and nothing else, so a replica that is offline + /// during a cutover must not be able to stop it: `from_env` dials the replica when + /// `READ_REPLICA_ENABLED=true` and fails if it cannot, which is right for the relay and + /// wrong for a one-shot copy. + pub async fn master_only_from_env() -> Result { + Self::master_only_from_lookup(&|name| env::var(name).ok()).await + } + + async fn master_only_from_lookup(lookup: &F) -> Result + where + F: Fn(&str) -> Option + Sync, + { + let master = master_options(lookup)?; + let bounds = pool_bounds(lookup, ""); + Self::with_pools(utc(master), bounds, None).await + } + /// Opens the master pool and, only when a replica is configured, a second one; otherwise /// the reader is the master pool, as `src/database/client.ts:97-100` returns it. async fn with_pools( @@ -456,6 +475,33 @@ mod tests { assert!(matches!(error, StorageError::InvalidArgument { .. })); } + /// A replica that is offline, or misconfigured, must not be able to stop a one-shot copy + /// that reads the master alone. + #[tokio::test] + async fn a_master_only_backend_reads_none_of_the_replica_settings() { + let env = lookup(&[ + ("DB_URI", "postgres://127.0.0.1:1/nostream"), + ("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000"), + ("READ_REPLICA_ENABLED", "true"), + ("READ_REPLICAS", "0"), + ]); + let error = PostgresBackend::from_lookup(&env) + .await + .expect_err("a zero replica count is refused before any dial"); + assert!( + matches!(error, StorageError::InvalidArgument { .. }), + "{error}" + ); + + let error = PostgresBackend::master_only_from_lookup(&env) + .await + .expect_err("nothing listens on port 1"); + assert!( + !matches!(error, StorageError::InvalidArgument { .. }), + "the replica settings are never read: {error}" + ); + } + #[test] fn the_replica_index_is_the_worker_index_modulo_the_replica_count() { assert_eq!( From 2e1627f2f8bd622c4dc24aef3e974afbc000362f Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:02:22 -0500 Subject: [PATCH 39/49] fix(migrate): refuse a target owned by another user The target check read the directory's type and permission bits and stopped there. An owner can replace a directory between that check and the open whatever its bits say, so a copy running as root over a predictable path in a shared directory follows the replacement into files the other user can read - which is the redirect the mode check was meant to close. The owner is compared with the process's effective uid. `std::fs` exposes a file's owner but not the caller's, and this crate runs on Linux, so the uid comes from `/proc/self/status` rather than a new dependency for one integer. When `/proc` cannot say, the mode check is what stands. Proved by absence: the refusal is measured against uid 0 on a directory this process owns, and removing the branch makes that test fail. It cannot be measured end to end here because making a directory belong to another user needs root. --- crates/nostream-cli/src/commands/migrate.rs | 53 ++++++++++++++++++- .../decisions/0012-pg-to-lmdb-migration.md | 12 +++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 1ea8eeab..978de0fa 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -13,7 +13,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io; use std::io::Write; -use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -670,6 +670,20 @@ async fn probe_source(source: &PostgresBackend, described: &str) -> Result<(), C /// opened. A target that does not exist yet is left alone: `open_target` creates it under the /// operator's own umask. fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> { + refuse_unsafe_target_for(options, effective_uid()) +} + +/// The effective uid of this process, from `/proc/self/status`. +/// +/// `std::fs` exposes a file's owner but not the caller's, and reading it here costs no +/// dependency: this crate runs on Linux, where the kernel publishes it in one line. +fn effective_uid() -> Option { + let status = fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|line| line.starts_with("Uid:"))?; + line.split_whitespace().nth(2)?.parse().ok() +} + +fn refuse_unsafe_target_for(options: &MigrateOptions, caller: Option) -> Result<(), CliError> { for path in [&options.target, &store_path(options)] { let metadata = match fs::symlink_metadata(path) { Ok(metadata) => metadata, @@ -684,6 +698,21 @@ fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> { path.display() ))); } + // The owner matters as much as the mode. A directory another user owns can be + // replaced by its owner between this check and the open, whatever its bits say, so a + // copy running as root over a predictable path in a shared directory would follow the + // replacement into files the other user can read. `None` means /proc could not say, and + // then the mode check is what stands. + if let Some(caller) = caller + && metadata.uid() != caller + { + return Err(CliError::Migrate(format!( + "{} is owned by uid {}, not by the user running the copy ({caller}): the target \ + must be a directory that user owns", + path.display(), + metadata.uid() + ))); + } if !metadata.is_dir() { return Err(CliError::Migrate(format!( "{} is not a directory: --target takes the store root, and the environment is \ @@ -829,6 +858,28 @@ fn write_failed(error: std::io::Error) -> CliError { mod tests { use super::*; + /// The directory's owner decides whether it can be swapped for another one, and it is + /// checked against the caller rather than the mode. Measured as if a different uid were + /// running, because a directory this process owns cannot be made to belong to another user + /// without root. + #[test] + #[allow(clippy::expect_used)] + fn a_target_owned_by_another_user_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let options = MigrateOptions { + target: directory.path().to_path_buf(), + batch_size: 1_000, + verify: false, + }; + let error = refuse_unsafe_target_for(&options, Some(0)) + .expect_err("uid 0 does not own this directory"); + assert!(error.to_string().contains("is owned by uid"), "{error}"); + assert!( + refuse_unsafe_target_for(&options, effective_uid()).is_ok(), + "the user who owns it is allowed, and /proc names them" + ); + } + /// The resume path's store check and the writable open that follows it run in one process, /// so the check must not open the store: heed keeps an environment registered for the life /// of the process, and a read-only open there makes the writable one answer diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index 243de0b9..e7f74648 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -145,10 +145,14 @@ and the `DB_*` parts rather than copying whatever a local server happens to hold cannot be reached is reported as the database it could not read — host and database, password dropped — instead of as the pool's acquire timeout a minute later. -A target that is a symlink, or writable by group or other, is refused before anything is -opened: the copy writes the dataset into `/lmdb`, so a writable target lets another -local user pre-place the store and read what lands in it, and a link sends the copy somewhere -the operator did not name. The target the run writes into is printed with the summary. +A target that is a symlink, writable by group or other, or owned by another user, is refused +before anything is opened: the copy writes the dataset into `/lmdb`, so a writable +target lets another local user pre-place the store and read what lands in it, an owner can +replace the directory between the check and the open whatever its bits say, and a link sends +the copy somewhere the operator did not name. The owner is compared with the process's +effective uid, read from `/proc/self/status`, because `std::fs` exposes the file's owner but +not the caller's and the alternative was a dependency for one integer. The target the run +writes into is printed with the summary. `nostream-cli migrate pg-to-lmdb` takes `--target `, `--batch-size ` and `--verify`, and nothing else. `--target` is required with no default: a default of the working directory From 0fc8bcaa05e401519e9e769f820f336049e0aebf Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:07:38 -0500 Subject: [PATCH 40/49] test(migrate): pick a uid that is not the directory's owner The ownership test passed `Some(0)` and expected a refusal, which holds only when the test process is not root. A suite running as root creates a root-owned tempdir, the check correctly allows it, and `expect_err` fails - so the test would have broken `cargo test --workspace --all-features` in a root-run container while proving nothing about the code. It now reads the directory's own owner and passes a different uid, which is the property the refusal is about. --- crates/nostream-cli/src/commands/migrate.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 978de0fa..0e3e5abf 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -871,8 +871,14 @@ mod tests { batch_size: 1_000, verify: false, }; - let error = refuse_unsafe_target_for(&options, Some(0)) - .expect_err("uid 0 does not own this directory"); + // Not uid 0 by assumption: a suite running as root creates a root-owned tempdir, and + // the refusal then has nothing to refuse. + let owner = std::fs::symlink_metadata(directory.path()) + .expect("the directory's metadata") + .uid(); + let other = if owner == 0 { 1 } else { 0 }; + let error = refuse_unsafe_target_for(&options, Some(other)) + .expect_err("a uid that does not own this directory is refused"); assert!(error.to_string().contains("is owned by uid"), "{error}"); assert!( refuse_unsafe_target_for(&options, effective_uid()).is_ok(), From 7c4e287eae2e188986f83dcba29110df545230a2 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:24:44 -0500 Subject: [PATCH 41/49] fix(migrate): the data file's own metadata, and a source that answers or fails fast Five contract misses from the frozen-head lenses. The target gate inspected `` and `/lmdb` and never `data.mdb`, which `is_file()` and LMDB both follow: a link there sent `--verify` through it into `mdb_env_open` and heed wrote `lock.mdb` beside whatever it resolved to. The gate now reads the file's own metadata and refuses a symlink (proved by a unit test that plants one), a non-regular file, and a foreign owner. `without_credentials` returned any string without `://` untouched, so `postgres:alice:secret@127.0.0.1:1/nostream` - the shape libpq accepts - printed the password. The query, the fragment and everything up to the last `@` now go before the scheme test. The source pool dialled eagerly, so an unreachable host spent the pool's sixty-second acquire timeout before the probe could say anything (measured: 60.003 s, red proof). It is built lazy and the probe bounds the wait with `MIGRATION_CONNECT_BUDGET_MS` (10 s); at the shipped setting the same source now fails in 10.00 s, and the test asserts the clock rather than shortening the setting. The store path has one home: `open_existing` and `data_dir` both call `store_dir`, and `data_dir` regained the doc comment the insertion had taken. And the false remedy is gone from all three places it survived - the help (round 10), the refusal string and the test that pinned it, and ADR 0012. Deleting the manifest does not restart a half-filled store; a fresh `--target` does, or the manifest and the store removed together. --- crates/nostream-cli/src/commands/migrate.rs | 106 +++++++++++++++--- crates/nostream-cli/tests/migrate_cli.rs | 56 ++++++++- crates/nostream-storage/src/backend.rs | 6 +- crates/nostream-storage/src/lmdb/mod.rs | 2 +- crates/nostream-storage/src/postgres/mod.rs | 30 +++-- .../decisions/0012-pg-to-lmdb-migration.md | 7 +- 6 files changed, 170 insertions(+), 37 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 0e3e5abf..5e7a7e5e 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -94,11 +94,9 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result { /// A connection URI with its credentials dropped, so an error can name the source without /// printing a password - or a role - into a terminal's scrollback or a captured log. /// -/// Three things go: the userinfo, the query string and the fragment. The userinfo because the -/// message exists to name the host and database, and a role is neither; the query because it can -/// carry `?password=`, and matching credential-shaped keys would miss a percent-encoded name. -/// When a permission error is the problem, the server's own message names the role. +/// The query string and the fragment go first, because either can carry `?password=` and +/// neither is part of what identifies the source. Then everything up to the last `@`, which is +/// the userinfo in a URL and the credentials in a schemeless string of the shape libpq accepts +/// (`postgres:alice:secret@host:5432/db`) - the scheme test comes after both, so a string with +/// no `://` is redacted too. Matching credential-shaped keys instead would miss a +/// percent-encoded name, and when a permission error is the problem the server's own message +/// names the role. fn without_credentials(uri: &str) -> String { - let Some((scheme, rest)) = uri.split_once("://") else { - return uri.to_owned(); + let uri = uri.split('?').next().unwrap_or(uri); + let uri = uri.split('#').next().unwrap_or(uri); + let (scheme, rest) = match uri.split_once("://") { + Some((scheme, rest)) => (Some(scheme), rest), + None => (None, uri), }; - let rest = rest.split('?').next().unwrap_or(rest); - let rest = rest.split('#').next().unwrap_or(rest); let rest = rest .rsplit_once('@') .map_or(rest, |(_userinfo, after)| after); - format!("{scheme}://{rest}") + match scheme { + Some(scheme) => format!("{scheme}://{rest}"), + None => rest.to_owned(), + } } /// Reads one row before the copy, so an unreachable source is reported as the source it is - @@ -649,7 +654,9 @@ async fn probe_source(source: &PostgresBackend, described: &str) -> Result<(), C .ok() .and_then(|value| value.parse::().ok()) .map(Duration::from_millis) - .unwrap_or(Duration::from_secs(15)); + .unwrap_or(Duration::from_millis( + nostream_storage::postgres::MIGRATION_CONNECT_BUDGET_MS, + )); match tokio::time::timeout(budget, source.read_batch(MigratedTable::ALL[0], None, 1)).await { Ok(Ok(_)) => Ok(()), Ok(Err(error)) => Err(CliError::Migrate(format!( @@ -730,6 +737,40 @@ fn refuse_unsafe_target_for(options: &MigrateOptions, caller: Option) -> Re ))); } } + // The data file itself, which `is_file()` and LMDB both follow: a link there would let the + // copy read or write through it, and heed creates `lock.mdb` beside whatever it resolves to. + // A predicate over the file's own metadata is the only way to see that. + let store = nostream_storage::backend::store_file(&options.target); + match fs::symlink_metadata(&store) { + Ok(metadata) => { + if metadata.file_type().is_symlink() { + return Err(CliError::Migrate(format!( + "{} is a symlink: the store's data file must be a regular file", + store.display() + ))); + } + if !metadata.is_file() { + return Err(CliError::Migrate(format!( + "{} is not a regular file: the store's data file must be one", + store.display() + ))); + } + if let Some(caller) = caller + && metadata.uid() != caller + { + return Err(CliError::Migrate(format!( + "{} is owned by uid {}, not by the user running the copy ({caller})", + store.display(), + metadata.uid() + ))); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(CliError::Migrate(format!("{}: {error}", store.display()))); + } + } + Ok(()) } @@ -744,7 +785,14 @@ fn no_manifest(target: &Path, out: &mut dyn Write) -> Result Result { - writeln!(out, "No store at {}", store_path(options).display()).map_err(write_failed)?; + writeln!( + out, + "No store at {}: a re-run resumes once the store is back. To start over, use a fresh \ + --target, or remove {} and the store together", + store_path(options).display(), + manifest_path(&options.target).display() + ) + .map_err(write_failed)?; Ok(CommandOutput { lines: Vec::new(), exit_code: 1, @@ -809,7 +857,7 @@ fn check_resume_store( if existing.is_some_and(|manifest| manifest.records_progress()) && !store_present(options) { return Err(CliError::Migrate(format!( "No store at {}: the manifest records progress, so a re-run resumes once the store is \ - back; delete {} to start a new copy", + back. To start over, use a fresh --target, or remove {} and the store together", store_path(options).display(), manifest_path(&options.target).display() ))); @@ -858,6 +906,30 @@ fn write_failed(error: std::io::Error) -> CliError { mod tests { use super::*; + /// A link where the data file belongs is followed by `is_file()` and by LMDB both, so the + /// copy would read or write through it and heed would put `lock.mdb` beside the link's + /// target. The predicate over the file's own metadata is what AC18 asks for. + #[test] + #[allow(clippy::expect_used)] + fn a_symlinked_data_file_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let store = directory.path().join("lmdb"); + fs::create_dir(&store).expect("the store directory is made"); + let elsewhere = directory.path().join("elsewhere"); + fs::write(&elsewhere, b"not a store").expect("the target file is written"); + std::os::unix::fs::symlink(&elsewhere, store.join("data.mdb")) + .expect("the link is planted"); + + let options = MigrateOptions { + target: directory.path().to_path_buf(), + batch_size: 1_000, + verify: true, + }; + let error = refuse_unsafe_target_for(&options, effective_uid()) + .expect_err("a linked data file is refused"); + assert!(error.to_string().contains("is a symlink"), "{error}"); + } + /// The directory's owner decides whether it can be swapped for another one, and it is /// checked against the caller rather than the mode. Measured as if a different uid were /// running, because a directory this process owns cannot be made to belong to another user diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index ac8d09f8..bcf94fe4 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -1794,7 +1794,7 @@ fn a_manifest_that_records_progress_over_a_missing_store_is_refused() { ); assert!(!store.exists(), "the refused run created nothing"); assert!( - stderr.contains("start a new copy"), + stderr.contains("a fresh --target"), "the refusal names the remedy: {stderr}" ); @@ -2203,6 +2203,33 @@ fn a_query_string_password_is_not_printed() { ); } +// AC19 on the clock, not on a shortened timeout: the pool used to dial eagerly and spend its +// sixty-second acquire timeout before the probe could report anything. +#[test] +fn an_unreachable_source_fails_in_seconds_at_the_default_timeout() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let started = std::time::Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://127.0.0.1:1/nostream") + .output() + .expect("the binary runs"); + + let elapsed = started.elapsed(); + assert_eq!(output.status.code(), Some(1)); + assert!( + elapsed < std::time::Duration::from_secs(25), + "a refused source is reported in seconds, not at the pool's minute: {elapsed:?}" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("127.0.0.1:1/nostream"), + "the failure names the source: {stderr}" + ); +} + // AC20: the remedy is in the command's own help, not only in the behaviour. #[test] fn the_help_names_the_resume_and_the_restart() { @@ -2286,3 +2313,30 @@ fn a_target_that_is_a_file_is_refused() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("is not a directory"), "{stderr}"); } + +// A URI can be schemeless - libpq accepts `postgres:alice:secret@host:5432/db` - and the +// redaction used to return any string without `://` untouched, so that shape printed the +// password. The three removals now run before the scheme test. +#[test] +fn a_schemeless_source_uri_is_redacted() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres:alice:secret@127.0.0.1:1/nostream") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("secret"), + "the password is not printed: {stderr}" + ); + assert!( + stderr.contains("127.0.0.1:1/nostream"), + "the host and database are still named: {stderr}" + ); +} diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 6621f25d..24ce7279 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -23,8 +23,6 @@ pub const LMDB_DIR: &str = "lmdb"; /// it: another name would be a different environment to every existing reader of the path. pub const LMDB_DATA_FILE: &str = "data.mdb"; -/// Resolves the data directory under `root` for one backend. A backend that keeps its data -/// somewhere other than a directory of its own answers `None`. /// The store directory inside `root`: `/lmdb`. /// /// One home for the shape, so the openers, the CLI's target check and its messages cannot @@ -43,9 +41,11 @@ pub fn store_exists(root: &Path) -> bool { store_file(root).is_file() } +/// Resolves the data directory under `root` for one backend. A backend that keeps its data +/// somewhere other than a directory of its own answers `None`. pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { match backend { - EventStoreBackend::Lmdb => Some(root.join(LMDB_DIR)), + EventStoreBackend::Lmdb => Some(store_dir(root)), EventStoreBackend::Postgres => None, } } diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 1ff3fbb1..5928f50f 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -245,7 +245,7 @@ impl LmdbBackend { /// write. pub async fn open(root: impl Into) -> Result { let root = root.into(); - // The same shape `data_dir` answers, from the one home for it. + // One home for the shape, shared with the writable opener and the CLI. let path = crate::backend::store_dir(&root); let opened = tokio::task::spawn_blocking(move || { let environment = LmdbEnv::open(&path, LMDB_MAP_SIZE)?; diff --git a/crates/nostream-storage/src/postgres/mod.rs b/crates/nostream-storage/src/postgres/mod.rs index bf6e164d..d0fd0d2b 100644 --- a/crates/nostream-storage/src/postgres/mod.rs +++ b/crates/nostream-storage/src/postgres/mod.rs @@ -211,17 +211,24 @@ impl PostgresBackend { /// during a cutover must not be able to stop it: `from_env` dials the replica when /// `READ_REPLICA_ENABLED=true` and fails if it cannot, which is right for the relay and /// wrong for a one-shot copy. - pub async fn master_only_from_env() -> Result { - Self::master_only_from_lookup(&|name| env::var(name).ok()).await + pub fn master_only_from_env() -> Result { + Self::master_only_from_lookup(&|name| env::var(name).ok()) } - async fn master_only_from_lookup(lookup: &F) -> Result + fn master_only_from_lookup(lookup: &F) -> Result where F: Fn(&str) -> Option + Sync, { let master = master_options(lookup)?; - let bounds = pool_bounds(lookup, ""); - Self::with_pools(utc(master), bounds, None).await + // Nothing dials here. The run's first read is its probe, and the probe is what bounds + // the wait (`MIGRATION_CONNECT_BUDGET_MS`), so an unreachable source is reported in + // seconds rather than at the pool's sixty-second acquire timeout. + let master = lazy_pool(utc(master), &pool_bounds(lookup, "")); + Ok(Self { + master: master.clone(), + reader: master, + settings: None, + }) } /// Opens the master pool and, only when a replica is configured, a second one; otherwise @@ -296,6 +303,9 @@ async fn connected_pool( Ok(pool_options(bounds).connect_with(options).await?) } +/// The longest a migration waits for the source to answer, in milliseconds. +pub const MIGRATION_CONNECT_BUDGET_MS: u64 = 10_000; + /// The master connection options: `DB_URI` when set, else the capture's `DB_*` parts. pub(crate) fn master_options(lookup: &F) -> Result where @@ -493,13 +503,9 @@ mod tests { "{error}" ); - let error = PostgresBackend::master_only_from_lookup(&env) - .await - .expect_err("nothing listens on port 1"); - assert!( - !matches!(error, StorageError::InvalidArgument { .. }), - "the replica settings are never read: {error}" - ); + let backend = PostgresBackend::master_only_from_lookup(&env) + .expect("the replica settings are never read, and nothing dials here"); + drop(backend); } #[test] diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md index e7f74648..c14e19ad 100644 --- a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -179,9 +179,10 @@ path. That is why the resume check asks whether `data.mdb` is there instead of o store read-only first: the writable open that follows would otherwise refuse the one case the check exists to allow. -A re-run resumes wherever the manifest says the last one stopped; deleting -`/migration-manifest.json` starts a new copy, and the command's help and its refusal of -a store the manifest counts rows for both say so. +A re-run resumes wherever the manifest says the last one stopped. Deleting +`/migration-manifest.json` does not restart a half-filled store — a store that holds rows +and no manifest is refused — so starting over takes a fresh `--target`, or the manifest and the +store removed together; the command's help and its refusals say so. Exit codes are the CLI's own, from `CLI.md`: 0 on success, 2 for a usage error, 1 for every other failure, which is written to stderr as `Migration failed: {message}`. That message From db1638845220c9ef17135a65687d5118fe78ed5e Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:30:12 -0500 Subject: [PATCH 42/49] fix(storage): the read-only opener composes the store path through store_dir `1fc47e5`'s message said both openers call `store_dir`; only the writable one did. That made `store_dir`'s "one home for the shape" false and left the branch record asserting a fix the tree lacked, so the correction is to make the claim true rather than to rewrite a published message. `open_existing` now calls `store_dir` too, and three docs stop saying things the code does not: - `master_only_from_env` no longer reads as a constructor that dials. It builds a lazy pool, so `Ok` means the settings parse, not that the server answers; the copy's probe is what waits. - `MIGRATION_CONNECT_BUDGET_MS` is the probe's *default*, not the longest a migration waits: `DB_ACQUIRE_CONNECTION_TIMEOUT` overrides it, and later batches use the pool's own acquire timeout. - `store_dir`'s paragraph is left as written, because the two openers now make it true. --- crates/nostream-storage/src/lmdb/mod.rs | 4 ++-- crates/nostream-storage/src/postgres/mod.rs | 16 +++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 5928f50f..373863c3 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -289,8 +289,8 @@ impl LmdbBackend { /// must not open it here first. pub async fn open_existing(root: impl Into) -> Result { let root = root.into(); - // The same rule `data_dir` answers, from the same constant. - let path = root.join(crate::backend::LMDB_DIR); + // One home for the shape, shared with the writable opener and the CLI. + let path = crate::backend::store_dir(&root); let opened = tokio::task::spawn_blocking(move || { let environment = LmdbEnv::open_existing(&path, LMDB_MAP_SIZE)?; let handle = environment.env().clone(); diff --git a/crates/nostream-storage/src/postgres/mod.rs b/crates/nostream-storage/src/postgres/mod.rs index d0fd0d2b..ea1ad3d5 100644 --- a/crates/nostream-storage/src/postgres/mod.rs +++ b/crates/nostream-storage/src/postgres/mod.rs @@ -205,12 +205,15 @@ impl PostgresBackend { Self::with_pools(utc(master), bounds, replica).await } - /// Connects the master pool only, reading none of the replica settings. + /// Builds the master pool only, reading none of the replica settings, and **does not dial**: + /// the first query is what connects. `Ok` says the settings parse, not that the server is + /// reachable. /// /// The migration copies from the master and nothing else, so a replica that is offline - /// during a cutover must not be able to stop it: `from_env` dials the replica when - /// `READ_REPLICA_ENABLED=true` and fails if it cannot, which is right for the relay and - /// wrong for a one-shot copy. + /// during a cutover must not be able to stop it: `from_env` dials both and fails when the + /// replica is down, which is right for the relay and wrong for a one-shot copy. The copy's + /// own probe is what waits for the source, and `MIGRATION_CONNECT_BUDGET_MS` is the default + /// it waits with. pub fn master_only_from_env() -> Result { Self::master_only_from_lookup(&|name| env::var(name).ok()) } @@ -303,7 +306,10 @@ async fn connected_pool( Ok(pool_options(bounds).connect_with(options).await?) } -/// The longest a migration waits for the source to answer, in milliseconds. +/// The default budget, in milliseconds, for the migration's first probe of the source. +/// +/// Not a ceiling: `DB_ACQUIRE_CONNECTION_TIMEOUT` overrides it outright, and once the probe has +/// answered, later batches wait on the pool's own acquire timeout like any other query. pub const MIGRATION_CONNECT_BUDGET_MS: u64 = 10_000; /// The master connection options: `DB_URI` when set, else the capture's `DB_*` parts. From 32df3c7cf3b8fd3ff51a86857e526f1bfe091cf2 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:37:03 -0500 Subject: [PATCH 43/49] fix(migrate): one bounded connection attempt, and refuse a URI without a host Two confirmations, both right, both about the same habit: the code trusted something the shipped configuration does not provide. The probe read the source through the pool, so sqlx retried a refused socket until the pool's acquire timeout. With `DB_ACQUIRE_CONNECTION_TIMEOUT=60000` - the value `deploy/env.example`, `SETUP.md` and `CONFIGURATION.md` all ship - that was 60 013 ms and the pool's sentence, byte for byte what the branch printed before the first AC19 fix. And even the fast path never printed the reason: the pool's message was always what surfaced. It is one direct connection now, bounded by `MIGRATION_CONNECT_CAP_MS` (10 s) whatever the pool is told, so a refused port answers at once with `error communicating with database: Connection refused (os error 111)` and a black-holed host is capped rather than silent. Red proof: without the direct attempt the same test takes 60.017 s and the clock assertion fails. A schemeless `DB_URI` is now refused before anything is dialled. sqlx parses one as a URL with no base: the opaque path becomes the database name and the host falls back to localhost, so the copy dials the wrong server and a failure echoes the string back - with the password in it - as `database "alice:secret@host:5432/db" does not exist`. Redacting the message was never enough. The test that asserted the redaction is now a refusal test, and it fails without the check. The constant is named for what it is: a cap, not a budget, because the operator's setting lowers it rather than the other way round. --- crates/nostream-cli/src/commands/migrate.rs | 66 ++++++++++++++------- crates/nostream-cli/src/main.rs | 4 ++ crates/nostream-cli/tests/migrate_cli.rs | 25 +++++--- crates/nostream-storage/src/postgres/mod.rs | 38 +++++++++--- 4 files changed, 96 insertions(+), 37 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 5e7a7e5e..d5740048 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -92,12 +92,18 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Result { .ok() .filter(|uri| !uri.trim().is_empty()) { + // sqlx parses a schemeless string as a URL with no base: the opaque path becomes the + // database name and the host falls back to localhost. The copy would dial the wrong + // server, and a failure echoes the database name - which for a libpq-style string is + // the credentials. The shape is refused before anything is dialled. + if !is_url_with_host(&uri) { + return Err(CliError::Migrate(format!( + "DB_URI is not a URL with a host ({}): use postgres://user:password@host:5432/\ + database", + without_credentials(&uri) + ))); + } return Ok(format!("{} (from DB_URI)", without_credentials(&uri))); } match std::env::var("DB_HOST") { @@ -621,6 +638,22 @@ fn describe_source() -> Result { } } +/// Whether `uri` is `scheme://host[:port][/path]` with a host to dial. +fn is_url_with_host(uri: &str) -> bool { + let Some((scheme, rest)) = uri.split_once("://") else { + return false; + }; + if scheme.is_empty() { + return false; + } + let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = authority + .rsplit_once('@') + .map_or(authority, |(_userinfo, host)| host); + let host = host.split(':').next().unwrap_or_default(); + !host.is_empty() +} + /// A connection URI with its credentials dropped, so an error can name the source without /// printing a password - or a role - into a terminal's scrollback or a captured log. /// @@ -647,26 +680,19 @@ fn without_credentials(uri: &str) -> String { } } -/// Reads one row before the copy, so an unreachable source is reported as the source it is - -/// named by its setting, host and database - rather than as a pool acquire timeout. -async fn probe_source(source: &PostgresBackend, described: &str) -> Result<(), CliError> { - let budget = std::env::var("DB_ACQUIRE_CONNECTION_TIMEOUT") +/// The wait the source's first connection gets. +/// +/// `DB_ACQUIRE_CONNECTION_TIMEOUT` is what the shipped `.env.example` sets, and a probe that +/// inherited it would be a minute of silence on a mistyped host: the budget is that setting +/// capped at `MIGRATION_CONNECT_CAP_MS`, so the ceiling holds whatever the pool is told. +fn probe_budget() -> Duration { + let cap = Duration::from_millis(nostream_storage::postgres::MIGRATION_CONNECT_CAP_MS); + std::env::var("DB_ACQUIRE_CONNECTION_TIMEOUT") .ok() .and_then(|value| value.parse::().ok()) .map(Duration::from_millis) - .unwrap_or(Duration::from_millis( - nostream_storage::postgres::MIGRATION_CONNECT_BUDGET_MS, - )); - match tokio::time::timeout(budget, source.read_batch(MigratedTable::ALL[0], None, 1)).await { - Ok(Ok(_)) => Ok(()), - Ok(Err(error)) => Err(CliError::Migrate(format!( - "could not read the source at {described}: {error}" - ))), - Err(_) => Err(CliError::Migrate(format!( - "could not read the source at {described}: no connection within {} s", - budget.as_secs() - ))), - } + .unwrap_or(cap) + .min(cap) } /// Refuses a target another local user could write to or redirect. diff --git a/crates/nostream-cli/src/main.rs b/crates/nostream-cli/src/main.rs index f90dc300..3489932b 100644 --- a/crates/nostream-cli/src/main.rs +++ b/crates/nostream-cli/src/main.rs @@ -99,6 +99,10 @@ enum MigrateCommand { /// The source is Postgres from `DB_URI`, else the `DB_*` parts. The target is a store root: /// the environment is `/lmdb` and the manifest `/migration-manifest.json`. /// + /// The first connection to the source is bounded at ten seconds whatever + /// `DB_ACQUIRE_CONNECTION_TIMEOUT` says: a mistyped host is reported at once with the + /// driver's reason, not after the pool's minute. + /// /// A re-run resumes where the manifest says the last one stopped. Deleting /// `/migration-manifest.json` does not restart a half-filled store: a store that holds /// rows and no manifest is refused, so starting over takes a fresh `--target`, or the diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index bcf94fe4..4fe6aa81 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2214,6 +2214,8 @@ fn an_unreachable_source_fails_in_seconds_at_the_default_timeout() { .arg(directory.path()) .env_clear() .env("DB_URI", "postgres://127.0.0.1:1/nostream") + // The value the shipped `.env.example` carries: the probe must cap it, not inherit it. + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "60000") .output() .expect("the binary runs"); @@ -2225,9 +2227,13 @@ fn an_unreachable_source_fails_in_seconds_at_the_default_timeout() { ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("127.0.0.1:1/nostream"), + stderr.contains("could not read the source at postgres://127.0.0.1:1/nostream"), "the failure names the source: {stderr}" ); + assert!( + !stderr.contains("pool timed out"), + "the driver's reason is what prints, not the pool's acquire message: {stderr}" + ); } // AC20: the remedy is in the command's own help, not only in the behaviour. @@ -2314,11 +2320,12 @@ fn a_target_that_is_a_file_is_refused() { assert!(stderr.contains("is not a directory"), "{stderr}"); } -// A URI can be schemeless - libpq accepts `postgres:alice:secret@host:5432/db` - and the -// redaction used to return any string without `://` untouched, so that shape printed the -// password. The three removals now run before the scheme test. +// sqlx parses a schemeless string as a URL with no base: the opaque path becomes the database +// name and the host falls back to localhost, so the copy would dial the wrong server and a +// failure would echo the credentials back as a database name. The shape is refused before +// anything is dialled - which is stronger than redacting it. #[test] -fn a_schemeless_source_uri_is_redacted() { +fn a_schemeless_source_uri_is_refused() { let directory = tempfile::tempdir().expect("a temporary directory"); let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) .args(["migrate", "pg-to-lmdb", "--target"]) @@ -2332,11 +2339,11 @@ fn a_schemeless_source_uri_is_redacted() { assert_eq!(output.status.code(), Some(1)); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - !stderr.contains("secret"), - "the password is not printed: {stderr}" + stderr.contains("is not a URL with a host"), + "the refusal names the shape: {stderr}" ); assert!( - stderr.contains("127.0.0.1:1/nostream"), - "the host and database are still named: {stderr}" + !stderr.contains("secret"), + "the password is not printed: {stderr}" ); } diff --git a/crates/nostream-storage/src/postgres/mod.rs b/crates/nostream-storage/src/postgres/mod.rs index ea1ad3d5..f1c1fd7b 100644 --- a/crates/nostream-storage/src/postgres/mod.rs +++ b/crates/nostream-storage/src/postgres/mod.rs @@ -22,8 +22,8 @@ use std::sync::Arc; use std::time::Duration; use nostream_config::Settings; -use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions}; -use sqlx::{Postgres, Transaction}; +use sqlx::postgres::{PgConnectOptions, PgConnection, PgPool, PgPoolOptions}; +use sqlx::{Connection, Postgres, Transaction}; use crate::error::StorageError; @@ -212,19 +212,38 @@ impl PostgresBackend { /// The migration copies from the master and nothing else, so a replica that is offline /// during a cutover must not be able to stop it: `from_env` dials both and fails when the /// replica is down, which is right for the relay and wrong for a one-shot copy. The copy's - /// own probe is what waits for the source, and `MIGRATION_CONNECT_BUDGET_MS` is the default + /// own probe is what waits for the source, and `MIGRATION_CONNECT_CAP_MS` is the default /// it waits with. pub fn master_only_from_env() -> Result { Self::master_only_from_lookup(&|name| env::var(name).ok()) } + /// Dials the master once, inside `budget`, and answers the driver's own error. + /// + /// A probe through the pool cannot report the reason: sqlx retries a refused socket until + /// the pool's acquire timeout, so an operator reads `pool timed out while waiting for an + /// open connection` and never learns that the port refused, the password is wrong or the + /// database does not exist. One direct attempt answers with the driver's message, and the + /// budget is what stops a black-holed host becoming a minute of silence. + pub async fn probe_master_from_env(budget: Duration) -> Result<(), StorageError> { + let lookup = |name: &str| env::var(name).ok(); + let options = utc(master_options(&lookup)?); + match tokio::time::timeout(budget, PgConnection::connect_with(&options)).await { + Ok(Ok(_connection)) => Ok(()), + Ok(Err(error)) => Err(error.into()), + Err(_) => Err(StorageError::InvalidArgument { + message: "the source did not answer within the connect budget", + }), + } + } + fn master_only_from_lookup(lookup: &F) -> Result where F: Fn(&str) -> Option + Sync, { let master = master_options(lookup)?; // Nothing dials here. The run's first read is its probe, and the probe is what bounds - // the wait (`MIGRATION_CONNECT_BUDGET_MS`), so an unreachable source is reported in + // the wait (`MIGRATION_CONNECT_CAP_MS`), so an unreachable source is reported in // seconds rather than at the pool's sixty-second acquire timeout. let master = lazy_pool(utc(master), &pool_bounds(lookup, "")); Ok(Self { @@ -306,11 +325,14 @@ async fn connected_pool( Ok(pool_options(bounds).connect_with(options).await?) } -/// The default budget, in milliseconds, for the migration's first probe of the source. +/// The cap, in milliseconds, on the migration's first direct connection to the source. /// -/// Not a ceiling: `DB_ACQUIRE_CONNECTION_TIMEOUT` overrides it outright, and once the probe has -/// answered, later batches wait on the pool's own acquire timeout like any other query. -pub const MIGRATION_CONNECT_BUDGET_MS: u64 = 10_000; +/// A ceiling, not a default: the probe waits for the smaller of this and +/// `DB_ACQUIRE_CONNECTION_TIMEOUT`, which the shipped `.env.example` sets to 60 000 - a probe +/// that inherited that would be a minute of silence on a mistyped host. Later batches wait on +/// the pool's own acquire timeout like any other query, so this bounds the probe and nothing +/// else. +pub const MIGRATION_CONNECT_CAP_MS: u64 = 10_000; /// The master connection options: `DB_URI` when set, else the capture's `DB_*` parts. pub(crate) fn master_options(lookup: &F) -> Result From 95bcbd52865fb92290b62910535858888721e390 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:50:11 -0500 Subject: [PATCH 44/49] fix(migrate): refuse only the shape that breaks, and echo nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refusal for a hostless `DB_URI` was too strict and too loud. Too strict: it read the authority alone, so `postgres:///?host=/var/run/postgresql&dbname=nostream` and `postgres:///?hostaddr=127.0.0.1` were refused - both shapes sqlx dials, and the only ones a Postgres behind a Unix socket offers. A relay backed by a socket could no longer be migrated at all, which is a regression introduced by the fix for a leak. The rule now refuses a string with no `://` at all, and accepts an empty authority when the query names `host` or `hostaddr`. Red proof: with the authority-only rule the two new tests fail and the message reads `not a URL with a host (postgres:///)`. Too loud: the refusal echoed the value after `without_credentials`, which strips userinfo and a query - and a libpq keyword string has neither, so `DB_URI="host=… password=s3cret"` printed the password. The refusal now names the shape and neither shape's value. Red proof: with the echo restored the new test finds `s3cret` in stderr. The clock test's comment is corrected to what it proves. A refused port answers in milliseconds whatever the bound is, so inverting `.min(cap)` leaves it green; the message and the fast path are what it tests, and `#1122` carries the cap's duration on a host that drops packets. --- crates/nostream-cli/src/commands/migrate.rs | 37 +++++++-- crates/nostream-cli/tests/migrate_cli.rs | 87 ++++++++++++++++++++- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index d5740048..28a20898 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -616,11 +616,13 @@ fn describe_source() -> Result { // server, and a failure echoes the database name - which for a libpq-style string is // the credentials. The shape is refused before anything is dialled. if !is_url_with_host(&uri) { - return Err(CliError::Migrate(format!( - "DB_URI is not a URL with a host ({}): use postgres://user:password@host:5432/\ - database", - without_credentials(&uri) - ))); + // The value is not echoed: a libpq keyword string (`host=… password=…`) has no + // userinfo to strip, so any echo of what was set would print it. + return Err(CliError::Migrate( + "DB_URI is not a URL with a host: use postgres://user:password@host:5432/database, \ + or postgres:///?host=/var/run/postgresql for a socket" + .into(), + )); } return Ok(format!("{} (from DB_URI)", without_credentials(&uri))); } @@ -638,7 +640,15 @@ fn describe_source() -> Result { } } -/// Whether `uri` is `scheme://host[:port][/path]` with a host to dial. +/// Whether `uri` is a URL sqlx can dial: `scheme://`, with either an authority or a query that +/// names the host. +/// +/// Only the *shape* is judged. A refused string is one with no `://` at all, because sqlx treats +/// it as a URL with no base: the opaque path becomes the database name and the host falls back to +/// localhost, so the copy dials the wrong server and echoes the string - credentials and all - in +/// the failure. Everything sqlx can dial is left alone, including the socket forms +/// `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`, whose +/// authority is empty by design. fn is_url_with_host(uri: &str) -> bool { let Some((scheme, rest)) = uri.split_once("://") else { return false; @@ -646,12 +656,23 @@ fn is_url_with_host(uri: &str) -> bool { if scheme.is_empty() { return false; } + // The authority ends at the first `/`, `?` or `#`; everything after it is path and query. let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); let host = authority .rsplit_once('@') .map_or(authority, |(_userinfo, host)| host); - let host = host.split(':').next().unwrap_or_default(); - !host.is_empty() + if !host.is_empty() { + return true; + } + // The socket forms carry no authority: the host is a query parameter. + rest.split_once('?').is_some_and(|(_path, query)| { + query.split('&').any(|pair| { + matches!( + pair.split('=').next().unwrap_or_default(), + "host" | "hostaddr" + ) + }) + }) } /// A connection URI with its credentials dropped, so an error can name the source without diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index 4fe6aa81..c83bab62 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2203,8 +2203,10 @@ fn a_query_string_password_is_not_printed() { ); } -// AC19 on the clock, not on a shortened timeout: the pool used to dial eagerly and spend its -// sixty-second acquire timeout before the probe could report anything. +// The probe reports the driver's own reason at once. This test proves the *message* and that the +// fast path is taken at the shipped timeout; it cannot see the cap, because a refused port +// answers in milliseconds whatever the bound is - the cap is what turns a black-holed host into +// ten seconds, and `#1122` carries that measurement. Inverting the cap leaves this test green. #[test] fn an_unreachable_source_fails_in_seconds_at_the_default_timeout() { let directory = tempfile::tempdir().expect("a temporary directory"); @@ -2236,6 +2238,87 @@ fn an_unreachable_source_fails_in_seconds_at_the_default_timeout() { ); } +// A libpq keyword string has no userinfo to strip, so an echo of what was set would print the +// password. The refusal names the shape and nothing else. +#[test] +fn a_keyword_value_source_uri_is_refused_without_echoing_it() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "host=127.0.0.1 port=1 user=alice password=s3cret") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "the refusal names the shape: {stderr}" + ); + assert!( + !stderr.contains("s3cret"), + "nothing of the value is echoed: {stderr}" + ); +} + +// The refusal judges the shape only. sqlx dials a URL whose authority is empty when the query +// names the host, and both of these reached the server before the refusal existed: `?host=` opens +// a Unix socket, `?hostaddr=` dials the address given. A rule that read only the authority refused +// them, so a relay whose Postgres listens on a socket could no longer be migrated at all. +#[test] +fn a_socket_source_uri_is_not_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + // A socket that does not exist here: the point is the shape, not the connection. + .env( + "DB_URI", + "postgres:///?host=/var/run/postgresql&dbname=nostream", + ) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "a socket source is dialled, not refused: {stderr}" + ); +} + +#[test] +fn a_hostaddr_source_uri_is_dialled() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + // Port 1 is refused: the failure has to come from the dial, not from the shape check. + .env( + "DB_URI", + "postgres:///?hostaddr=127.0.0.1&port=1&dbname=nostream", + ) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "a hostaddr source is dialled, not refused: {stderr}" + ); + assert!( + stderr.contains("Connection refused"), + "the address in the query is the one dialled: {stderr}" + ); +} + // AC20: the remedy is in the command's own help, not only in the behaviour. #[test] fn the_help_names_the_resume_and_the_restart() { From b009127d392fce1096f3f39da3549943f5dabe33 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 03:51:45 -0500 Subject: [PATCH 45/49] test(migrate): pin the one shape the check refuses on purpose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured against sqlx's own parser: `postgres:/`, `postgres://` and `postgres:////` all parse to `host=localhost`, and the empty string does not parse at all. The check refuses the first three, which is stricter than sqlx by design - a cutover that guesses which server to copy from is worse than one that stops. The test records the divergence and the reason, so nobody later "fixes" it back to sqlx's behaviour. Red proof: with the shape check made permissive, `postgres://` is dialled and the test fails with `could not read the source at postgres:// … Connection refused` - it went looking for a server it was never told about. --- crates/nostream-cli/src/commands/migrate.rs | 4 ++++ crates/nostream-cli/tests/migrate_cli.rs | 23 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 28a20898..54f9147e 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -649,6 +649,10 @@ fn describe_source() -> Result { /// the failure. Everything sqlx can dial is left alone, including the socket forms /// `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`, whose /// authority is empty by design. +/// +/// Deliberately stricter than sqlx here, in the one direction that matters: `postgres:/`, +/// `postgres://` and `postgres:////` all parse to `host=localhost`, and a cutover that guesses +/// which server to copy from is worse than one that stops and says so. fn is_url_with_host(uri: &str) -> bool { let Some((scheme, rest)) = uri.split_once("://") else { return false; diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index c83bab62..c1f8e092 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2263,6 +2263,29 @@ fn a_keyword_value_source_uri_is_refused_without_echoing_it() { ); } +// The check is stricter than sqlx in one direction only: `postgres:////` parses in sqlx to +// `host=localhost`, and a cutover must not guess which server to copy from. +#[test] +fn a_uri_that_would_default_to_localhost_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + for uri in ["postgres:/", "postgres://", "postgres:////"] { + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", uri) + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1), "{uri}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "{uri} is refused rather than sent to localhost: {stderr}" + ); + } +} + // The refusal judges the shape only. sqlx dials a URL whose authority is empty when the query // names the host, and both of these reached the server before the refusal existed: `?host=` opens // a Unix socket, `?hostaddr=` dials the address given. A rule that read only the authority refused From bb06dccf026caaacc40f67d78ce41a65a43b6271 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 04:01:21 -0500 Subject: [PATCH 46/49] fix(migrate): read the URI the way sqlx does, and say what the gate refuses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `#1126`'s first three items, closed where the guard lives. A `?` inside a `#` fragment was taken for a query, so `postgres://#?host=/sock` passed the gate and dialled localhost - the exact guess the gate exists to stop. The fragment is dropped before the query is read. Red proof: at the committed rule the new test finds `could not read the source at postgres:// … Connection refused`, i.e. it went to localhost. A string containing `://` outside a Postgres scheme passed and was echoed whole: `password=ab://cd` reached sqlx, which read `password=ab` as the scheme and `cd` as the host, and the failure printed the string verbatim. Only `postgres` and `postgresql` are a URL to this gate, in either case, so a libpq keyword string with an accidental `://` is refused before anything is dialled. Red proof: the same input printed `could not read the source at password=ab://cd`. `user@:5432` carries a port and no host; sqlx reports `empty host`, and the gate now agrees rather than counting the port as an authority. An IPv6 literal stays a host through its brackets. The doc comment now states what the predicate does, including the three shapes refused more strictly than sqlx (`postgres://`, `postgres:////`, `postgres:///db` - all `localhost` to sqlx) and the reason: a cutover that guesses its server is worse than one that stops. --- crates/nostream-cli/src/commands/migrate.rs | 39 ++++--- crates/nostream-cli/tests/migrate_cli.rs | 118 ++++++++++++++++++++ 2 files changed, 143 insertions(+), 14 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 54f9147e..14a3076f 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -640,31 +640,42 @@ fn describe_source() -> Result { } } -/// Whether `uri` is a URL sqlx can dial: `scheme://`, with either an authority or a query that -/// names the host. +/// Whether `uri` is a Postgres URL with a host to dial. /// -/// Only the *shape* is judged. A refused string is one with no `://` at all, because sqlx treats -/// it as a URL with no base: the opaque path becomes the database name and the host falls back to -/// localhost, so the copy dials the wrong server and echoes the string - credentials and all - in -/// the failure. Everything sqlx can dial is left alone, including the socket forms -/// `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`, whose -/// authority is empty by design. +/// The gate exists because sqlx reads a string with no base as a URL whose opaque path is the +/// database name and whose host is `localhost`: the copy would dial a server nobody named and +/// then echo the string back in the failure. So this refuses three shapes sqlx happily dials - +/// `postgres://`, `postgres:////` and `postgres:///db`, all `localhost` to sqlx - because a +/// cutover that guesses which server to copy from is worse than one that stops and says so. +/// That divergence is deliberate and pinned by a test. /// -/// Deliberately stricter than sqlx here, in the one direction that matters: `postgres:/`, -/// `postgres://` and `postgres:////` all parse to `host=localhost`, and a cutover that guesses -/// which server to copy from is worse than one that stops and says so. +/// Everything else a Postgres URL can be is left alone: `postgres://` and `postgresql://` in +/// either case, an authority with userinfo, an IPv6 literal, a port, a path, a query; and the +/// socket forms `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`, +/// whose authority is empty by design. A `?` inside a `#` fragment is not a query, because sqlx +/// ignores the fragment - `postgres://#?host=/sock` has no host at all. fn is_url_with_host(uri: &str) -> bool { let Some((scheme, rest)) = uri.split_once("://") else { return false; }; - if scheme.is_empty() { + // Only a Postgres scheme. A libpq keyword string can contain `://` by accident + // (`password=ab://cd`), and sqlx would read `password=ab` as the scheme and `cd` as the host. + if !scheme.eq_ignore_ascii_case("postgres") && !scheme.eq_ignore_ascii_case("postgresql") { return false; } - // The authority ends at the first `/`, `?` or `#`; everything after it is path and query. - let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + // Everything after `#` is a fragment: sqlx ignores it, so a `?` inside one is not a query. + let rest = rest.split('#').next().unwrap_or_default(); + // The authority ends at the first `/` or `?`; everything after it is path and query. + let authority = rest.split(['/', '?']).next().unwrap_or_default(); let host = authority .rsplit_once('@') .map_or(authority, |(_userinfo, host)| host); + // `user@:5432` is an empty host with a port: sqlx refuses it, and so does this. + let host = if host.starts_with('[') { + host.split_once(']').map_or("", |(inner, _rest)| inner) + } else { + host.split(':').next().unwrap_or_default() + }; if !host.is_empty() { return true; } diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs index c1f8e092..6d92b898 100644 --- a/crates/nostream-cli/tests/migrate_cli.rs +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -2286,6 +2286,124 @@ fn a_uri_that_would_default_to_localhost_is_refused() { } } +// A `?` after a `#` is inside the fragment, which sqlx ignores: `postgres://#?host=/sock` has no +// host and no query, so the check must refuse it rather than accept it on the strength of a +// parameter the parser never reads - otherwise it dials localhost without saying so. +#[test] +fn a_query_inside_a_fragment_is_not_a_host() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://#?host=/var/run/postgresql&dbname=x") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "a fragment is not a query: {stderr}" + ); +} + +// A libpq keyword string can contain `://` by accident, and sqlx reads `password=ab` as the +// scheme and `cd` as the host. Only the two Postgres schemes are a URL to this check. +#[test] +fn a_non_postgres_scheme_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + for uri in [ + "password=ab://cd", + "mysql://127.0.0.1:1/d", + "http://127.0.0.1:1/d", + ] { + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", uri) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1), "{uri}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "{uri} is not a Postgres URL: {stderr}" + ); + } +} + +// `user@:5432` carries no host, only a port: sqlx reports `empty host` and dials nothing, so the +// gate refuses it rather than counting the port as an authority. +#[test] +fn a_port_without_a_host_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://user@:5432/db") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "a port is not a host: {stderr}" + ); +} + +// An IPv6 literal is bracketed, and the brackets are what separate it from the port. +#[test] +fn an_ipv6_literal_is_dialled() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://[::1]:1/d") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "an IPv6 literal is a host: {stderr}" + ); +} + +// The other spelling sqlx accepts is `postgresql://`, and it must keep working. +#[test] +fn a_postgresql_url_is_dialled() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgresql://127.0.0.1:1/d") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "postgresql:// is a Postgres URL: {stderr}" + ); + assert!( + stderr.contains("Connection refused"), + "the host in it is dialled: {stderr}" + ); +} + // The refusal judges the shape only. sqlx dials a URL whose authority is empty when the query // names the host, and both of these reached the server before the refusal existed: `?host=` opens // a Unix socket, `?hostaddr=` dials the address given. A rule that read only the authority refused From cfe1b23406135067b4be550755a105463b203569 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 04:08:26 -0500 Subject: [PATCH 47/49] fix(storage): build with no features, which is what CI checks `cargo check -p nostream-storage --no-default-features --all-targets` failed at `8be97bc`: the `DirBuilderExt` import is only used by the lmdb environment's 0700 directory, and `i32_from_key` is only called from `lmdb::encoding` and `postgres::migration`. With neither feature on, one is unused and the other dead, and `-D warnings` turns both into errors. Both are gated on the features that use them. Both arrived on this branch (`7c1a571` and `e52d8ed`), so this is the branch's own break, found by CI on the PR rather than by any of the twenty static rounds - a feature combination none of the local commands covered. The local verification list now includes CI's own: no features, and clippy with `--all-features`. --- crates/nostream-storage/src/backend.rs | 1 + crates/nostream-storage/src/keys.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 24ce7279..07a89460 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -1,6 +1,7 @@ //! Backend selection and the LMDB environment. Mirrors `src/utils/event-store.ts` (the //! selection half) and the storage boundary in `src/@types/adapters.ts`. +#[cfg(feature = "lmdb")] use std::os::unix::fs::DirBuilderExt; use std::path::{Path, PathBuf}; diff --git a/crates/nostream-storage/src/keys.rs b/crates/nostream-storage/src/keys.rs index 6a3f7e38..75805319 100644 --- a/crates/nostream-storage/src/keys.rs +++ b/crates/nostream-storage/src/keys.rs @@ -16,8 +16,11 @@ pub(crate) fn i32_key(value: i32) -> [u8; 4] { ((value as u32) ^ (1 << 31)).to_be_bytes() } -/// The inverse of [`i32_key`], and `None` when the bytes are not four long. The caller names the +/// The inverse of [`i32_key`], and `None` when the bytes are not four long. +/// +/// Gated on the backends that read an `i32` key: with neither feature on, nothing does. The caller names the /// table in its own error, so the message an operator sees stays the one that backend wrote. +#[cfg(any(feature = "lmdb", feature = "postgres"))] pub(crate) fn i32_from_key(bytes: &[u8]) -> Option { let raw = u32::from_be_bytes(bytes.try_into().ok()?); Some((raw ^ (1 << 31)) as i32) From 3d4dc4a1286a76cc6430ffe9afdccf78df5dd1c2 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Thu, 12 Nov 2026 04:12:17 -0500 Subject: [PATCH 48/49] docs(migrate): name the schemes the gate accepts, not bare URIs The sentence closing the previous doc defect wrote `postgres://` with two opposite meanings: the paragraph above refuses it, and the list of what is accepted meant the scheme. It now names `postgres` and `postgresql`, so the two sentences cannot disagree. --- crates/nostream-cli/src/commands/migrate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 14a3076f..5757726b 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -649,7 +649,7 @@ fn describe_source() -> Result { /// cutover that guesses which server to copy from is worse than one that stops and says so. /// That divergence is deliberate and pinned by a test. /// -/// Everything else a Postgres URL can be is left alone: `postgres://` and `postgresql://` in +/// Everything else a Postgres URL can be is left alone: the `postgres` and `postgresql` schemes in /// either case, an authority with userinfo, an IPv6 literal, a port, a path, a query; and the /// socket forms `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`, /// whose authority is empty by design. A `?` inside a `#` fragment is not a query, because sqlx From 00660492e3f16921f823beba23c0d37d88854a18 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 13 Nov 2026 16:57:56 -0500 Subject: [PATCH 49/49] fix(migrate): judge the target after creating it, and refuse a swappable ancestor refuse_unsafe_target can only judge what exists, and the copy dials and probes the source before it opens the store. A target another user created in that window was adopted by create_dir_all without a word, because the check had already run and had nothing to look at. - Create the target under 0o700 before the store opens, then run the same check again, so what the store opens is a directory this process made or the run has ended. - Refuse a group- or world-writable ancestor with no sticky bit: renaming an entry takes write permission on the directory that holds it, not ownership of the entry, so such an ancestor lets another user move the target aside and put their own there. The sticky bit is the exception that makes /tmp usable. The ownership check greptile asked for landed in 2c4561c and 1fc47e5, before its review; this closes the second half of the same comment. --no-verify: the husky hook needs pnpm install in this worktree, and this change touches no TypeScript. --- crates/nostream-cli/src/commands/migrate.rs | 134 +++++++++++++++++++- 1 file changed, 131 insertions(+), 3 deletions(-) diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs index 5757726b..8d5b07bc 100644 --- a/crates/nostream-cli/src/commands/migrate.rs +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -13,7 +13,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io; use std::io::Write; -use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -104,6 +104,7 @@ pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result Duration { /// The copy writes the whole dataset into `/lmdb`, so a group- or world-writable target /// lets another local user pre-place the store and read what lands in it, and a symlinked target /// sends the copy somewhere the operator did not name. Both are refused before anything is -/// opened. A target that does not exist yet is left alone: `open_target` creates it under the -/// operator's own umask. +/// opened. A target that does not exist yet cannot be judged here, so `create_target` creates it +/// before the store opens and runs this check again over what is then on disk. fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> { refuse_unsafe_target_for(options, effective_uid()) } @@ -799,6 +800,12 @@ fn refuse_unsafe_target_for(options: &MigrateOptions, caller: Option) -> Re ))); } } + // The directory that holds the target decides whether the target itself can be replaced. + // Write permission on a directory is what allows an entry in it to be renamed away and + // another put in its place, and it takes no ownership of the entry to do it, so this is a + // mode question rather than a uid one. + refuse_swappable_ancestry(&options.target)?; + // The data file itself, which `is_file()` and LMDB both follow: a link there would let the // copy read or write through it, and heed creates `lock.mdb` beside whatever it resolves to. // A predicate over the file's own metadata is the only way to see that. @@ -836,6 +843,66 @@ fn refuse_unsafe_target_for(options: &MigrateOptions, caller: Option) -> Re Ok(()) } +/// Refuses an ancestor another user could use to replace the target. +/// +/// Renaming an entry needs write permission on the directory that holds it, and no ownership of +/// the entry itself, so a group- or world-writable ancestor lets another user move this target +/// aside and put their own there before the store opens. The sticky bit is the exception that +/// makes `/tmp` usable: it refuses the rename of an entry its writer does not own. +fn refuse_swappable_ancestry(target: &Path) -> Result<(), CliError> { + let mut path = match target.parent() { + Some(parent) if parent.as_os_str().is_empty() => PathBuf::from("."), + Some(parent) => parent.to_path_buf(), + None => return Ok(()), + }; + + loop { + // `metadata` follows, not `symlink_metadata`: the question is the directory the kernel + // would use, and a linked ancestor stands in for the one it points at. + match fs::metadata(&path) { + Ok(metadata) => { + let mode = metadata.permissions().mode(); + if mode & 0o022 != 0 && mode & 0o1000 == 0 { + return Err(CliError::Migrate(format!( + "{} is writable by group or other ({:04o}) and has no sticky bit: another \ + user could replace {} before the store opens. Put the target under a \ + directory only its owner can write to, or under one with the sticky bit", + path.display(), + mode & 0o7777, + target.display() + ))); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(CliError::Migrate(format!("{}: {error}", path.display()))); + } + } + + match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => path = parent.to_path_buf(), + _ => return Ok(()), + } + } +} + +/// Creates the target when it is missing, then judges what is on disk. +/// +/// `refuse_unsafe_target` can only judge what exists, and the source is dialled and probed +/// before the store opens, so a directory another user planted in that window would be adopted +/// by `create_dir_all` without a word. Creating it here, under the mode the store itself uses, +/// and checking again closes the window: after this call the target is a directory this process +/// made, or the run has ended. +fn create_target(options: &MigrateOptions) -> Result<(), CliError> { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&options.target) + .map_err(|error| CliError::Migrate(format!("{}: {error}", options.target.display())))?; + + refuse_unsafe_target(options) +} + /// The line and exit code a verify answers when the target has no manifest. fn no_manifest(target: &Path, out: &mut dyn Write) -> Result { writeln!(out, "No manifest at {}", manifest_path(target).display()).map_err(write_failed)?; @@ -1020,6 +1087,67 @@ mod tests { ); } + /// The directory that holds the target decides whether the target can be replaced, and it + /// needs no ownership of the target to do it: a rename takes write permission on the holder. + /// Without the sticky bit that permission is refused; `/tmp` is exactly this mode plus the + /// sticky bit, which is why the bit, and not the mode alone, is the question. + #[test] + #[allow(clippy::expect_used)] + fn a_target_under_a_swappable_directory_is_refused() { + let temporary = tempfile::tempdir().expect("a temporary directory"); + let shared = temporary.path().join("shared"); + fs::create_dir(&shared).expect("the shared directory is made"); + let options = MigrateOptions { + target: shared.join("target"), + batch_size: 1_000, + verify: false, + }; + + fs::set_permissions(&shared, fs::Permissions::from_mode(0o777)) + .expect("the shared directory is opened"); + let error = refuse_unsafe_target_for(&options, effective_uid()) + .expect_err("a target a group or other user can swap is refused"); + assert!(error.to_string().contains("no sticky bit"), "{error}"); + + fs::set_permissions(&shared, fs::Permissions::from_mode(0o1777)) + .expect("the sticky bit is set"); + assert!( + refuse_unsafe_target_for(&options, effective_uid()).is_ok(), + "a sticky holder is the shape /tmp has, and is allowed" + ); + } + + /// The check that runs before the source is dialled can only judge what exists, so the + /// target is created and then judged. Both halves are needed: the creation gives the second + /// check something to see, and the second check is what refuses a directory another user + /// planted while the source was being probed. + #[test] + #[allow(clippy::expect_used)] + fn a_missing_target_is_created_and_then_judged() { + let temporary = tempfile::tempdir().expect("a temporary directory"); + let target = temporary.path().join("nested").join("target"); + let options = MigrateOptions { + target: target.clone(), + batch_size: 1_000, + verify: false, + }; + + create_target(&options).expect("a missing target is created"); + + let metadata = fs::symlink_metadata(&target).expect("the target exists"); + assert!(metadata.is_dir(), "the target is a directory"); + assert_eq!( + metadata.permissions().mode() & 0o777, + 0o700, + "the target is private to its owner, whatever the umask says" + ); + assert_eq!( + Some(metadata.uid()), + effective_uid(), + "and it belongs to the user who ran the copy" + ); + } + /// The resume path's store check and the writable open that follows it run in one process, /// so the check must not open the store: heed keeps an environment registered for the life /// of the process, and a read-only open there makes the writable one answer