diff --git a/.changeset/pg-to-lmdb-migration.md b/.changeset/pg-to-lmdb-migration.md new file mode 100644 index 00000000..a845151c --- /dev/null +++ b/.changeset/pg-to-lmdb-migration.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f5b4d889..fff0fe61 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -273,12 +273,12 @@ jobs: -D clippy::todo -D clippy::unimplemented -D clippy::panic - name: Tests run: cargo test --workspace --all-features - # The two database-backed storage targets create and migrate their own scratch - # databases; the Cucumber half of the harness stays in scripts/storage-parity.sh. + # The database-backed storage targets create and migrate their own scratch databases; the + # Cucumber half of the harness stays in scripts/storage-parity.sh. - name: Storage parity against Postgres env: PARITY_ADMIN_URL: postgres://postgres:postgres@localhost:5432/postgres - run: cargo test -p nostream-storage --test migrations --test parity -- --ignored --test-threads=1 + run: cargo test -p nostream-storage --test migrations --test parity --test migration_pg -- --ignored --test-threads=1 - name: Storage builds without the Postgres driver run: cargo check -p nostream-storage --no-default-features --all-targets - name: Release build diff --git a/.gitignore b/.gitignore index c2156dc1..41f1dd99 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,7 @@ coverage-report/ # Rust /target/ + +# A migration target is the dataset: `--target .` writes it here, and this repository is public. +/lmdb/ +/migration-manifest.json diff --git a/CLAUDE.md b/CLAUDE.md index 034dd0b0..de773bfa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,17 +36,19 @@ cargo test --workspace --all-features cargo build --workspace --all-features --release ``` -CI additionally runs the two database-backed storage tests ("Storage parity against Postgres") +CI additionally runs the three database-backed storage tests ("Storage parity against Postgres") against a `postgres:16-alpine` service, and they are `#[ignore]`d so a checkout without a database still passes `cargo test --workspace`: ``` -cargo test -p nostream-storage --test migrations --test parity -- --ignored --test-threads=1 +cargo test -p nostream-storage --test migrations --test parity --test migration_pg \ + -- --ignored --test-threads=1 ``` `PARITY_ADMIN_URL` names the administrative database (CI sets `postgres://postgres:postgres@localhost:5432/postgres`); the targets create, migrate and -drop their own scratch databases. `scripts/storage-parity.sh` runs both plus the +drop their own scratch databases — `migration_pg` seeds one and copies it into a temporary LMDB +store. `scripts/storage-parity.sh` runs all three plus the TypeScript Cucumber suite against the sqlx-migrated schema — the complete local oracle. The LMDB arm of that oracle needs no database: `cargo test -p nostream-storage --test parity_lmdb -- --nocapture` replays the same fixture on a temporary store and runs inside the fifth @@ -88,7 +90,7 @@ resolver 3: | `nostream-relay` | connections, wire protocol, handlers, rate limiters, live dispatch, NIP-77 engine (the acceptor lives in `nostreamd`) | | `nostream-services` | payment processors and the invoice lifecycle, operator notifications, the web-of-trust graph, Tor, the scheduler and its workers, event import, NIP-66 monitors (#865) | | `nostream-admin` | admin API and auth | -| `nostream-cli` | operator command line (`nostream-cli config validate`, `nostream-cli import`) | +| `nostream-cli` | operator command line (`nostream-cli config validate`, `nostream-cli import`, `nostream-cli migrate pg-to-lmdb`) | | `nostreamd` | the daemon binary: runtime, telemetry, signals, the services composition root and the worker supervisor | - Every crate manifest repeats `version.workspace = true`, `edition.workspace = true`, diff --git a/Cargo.lock b/Cargo.lock index ef684388..062a8c2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1244,13 +1244,18 @@ name = "nostream-cli" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", + "heed", + "hex", "nostream-config", "nostream-core", "nostream-services", "nostream-storage", + "serde", "serde_json", "serde_yaml", + "sha2 0.11.1", "tempfile", "thiserror", "tokio", diff --git a/crates/nostream-cli/Cargo.toml b/crates/nostream-cli/Cargo.toml index 0aa379ff..fca0a721 100644 --- a/crates/nostream-cli/Cargo.toml +++ b/crates/nostream-cli/Cargo.toml @@ -14,16 +14,21 @@ path = "src/main.rs" [dependencies] clap.workspace = true anyhow = { workspace = true } +hex.workspace = true nostream-config.workspace = true nostream-core.workspace = true nostream-services.workspace = true nostream-storage.workspace = true +serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true +sha2.workspace = true thiserror.workspace = true tokio.workspace = true [dev-dependencies] +chrono.workspace = true +heed.workspace = true tempfile.workspace = true [lints] diff --git a/crates/nostream-cli/src/commands/migrate.rs b/crates/nostream-cli/src/commands/migrate.rs new file mode 100644 index 00000000..8d5b07bc --- /dev/null +++ b/crates/nostream-cli/src/commands/migrate.rs @@ -0,0 +1,1219 @@ +//! `migrate pg-to-lmdb`: the copy of the Postgres store into an LMDB one. +//! +//! The seam is `nostream_storage::migration` — a paged read of each source table and an exact +//! write of each row, neither of which the nine repository traits can express. This module owns +//! what the seam's caller owns: the manifest, the checksum, resume, the verification pass and +//! the run's own measurement (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`). +//! +//! The batch commits to the store first and the manifest is replaced atomically second, so a +//! crash between them replays one batch: every write is a put of the source's own key and +//! value, so the replay is idempotent and the merged entry is unchanged. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io; +use std::io::Write; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; +use nostream_storage::{LmdbBackend, PostgresBackend}; +use sha2::{Digest, Sha256}; + +use crate::config::CommandOutput; +use crate::error::CliError; + +/// The manifest's file name under the target root. +const MANIFEST_FILE: &str = "migration-manifest.json"; + +/// The manifest format this tool writes. +const MANIFEST_FORMAT: u32 = 1; + +/// The options one run carries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MigrateOptions { + /// The store root: the environment is `/lmdb`, the manifest `/migration-manifest.json`. + pub target: PathBuf, + /// Rows per batch, which is also the page size verify reads with. + pub batch_size: usize, + /// Verify the target against its manifest and copy nothing. + pub verify: bool, +} + +/// `migrate pg-to-lmdb`: the copy, or the verification pass when `verify` is set. +pub fn run(options: MigrateOptions, out: &mut dyn Write) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| CliError::Config(format!("tokio runtime: {error}")))?; + + runtime.block_on(async move { + refuse_unsafe_target(&options)?; + + if options.verify { + // Both answered before anything is opened, so a verify of a directory that holds no + // store is refused rather than given a fresh empty one. The store is asked about by + // its file for the same reason: R8's missing store is `data.mdb`'s absence, and a + // directory that never held a migration looks exactly like it. Verify creates no + // store and no manifest; LMDB's read-only open still recreates `lock.mdb` beside an + // existing `data.mdb`, which is the one thing it writes. + if !store_present(&options) { + return no_store(&options, out); + } + if read_manifest(&options.target)?.is_none() { + return no_manifest(&options.target, out); + } + let target = open_existing_target(&options.target).await?; + return verify_with(&target, &options, out).await; + } + + let existing = read_manifest(&options.target)?; + + // A complete manifest is verified rather than copied, and the source is never opened: + // a finished migration must not need Postgres to still be reachable. + if existing + .as_ref() + .is_some_and(|manifest| manifest.complete()) + { + if !store_present(&options) { + return no_store(&options, out); + } + let target = open_existing_target(&options.target).await?; + return verify_with(&target, &options, out).await; + } + + // R7's store requirement, answered before the source is dialled so a missing store is + // reported rather than a database error. The writable open below follows it in this + // process, which is why the check must not open the store itself. + check_resume_store(existing.as_ref(), &options)?; + // The source is named before it is dialled and probed before the copy: a run whose + // `DB_URI` is unset would otherwise resolve to `postgres@localhost:5432/postgres` and + // copy whatever that happens to hold, and one whose host is down would report the + // pool's acquire timeout a minute later instead of the database it could not reach. + let described = describe_source()?; + // One direct attempt first, bounded: the pool retries a refused socket until its + // acquire timeout, so only a direct connection answers with the reason. Then the + // master-only backend, because the copy reads the master and nothing else and a + // replica that is down during a cutover must not stop it. + PostgresBackend::probe_master_from_env(probe_budget()) + .await + .map_err(|error| { + CliError::Migrate(format!("could not read the source at {described}: {error}")) + })?; + let source = PostgresBackend::master_only_from_env().map_err(|error| { + CliError::Migrate(format!("could not read the source at {described}: {error}")) + })?; + create_target(&options)?; + let target = open_target(&options.target).await?; + migrate_with(&source, &target, &options, out).await + }) +} + +/// The copy over an injected source, target and writer, which is how the whole run is tested +/// without a process and without a database. +/// +/// Every batch's rows are written to the store and then folded into the manifest, which is +/// replaced after each batch; a batch that fails leaves the previous batch's `after`, `rows` and +/// `checksum` in place, so the next run resumes at the refused row. +pub async fn migrate_with( + source: &dyn MigrationSource, + target: &dyn MigrationTarget, + options: &MigrateOptions, + out: &mut dyn Write, +) -> Result { + refuse_zero_batch(options)?; + let existing = read_manifest(&options.target)?; + let mut manifest = existing.clone().unwrap_or_else(Manifest::fresh); + + // A store that already holds rows and carries no manifest is not this tool's. Copying over + // one leaves the displaced rows' index entries behind — a completed invoice keeps its + // `invoices_pending` entry — and verify scans the primary databases only, so it could never + // report them (R7). + if existing.is_none() + && let Some(table) = first_occupied_table(target).await? + { + return Err(CliError::Migrate(format!( + "{} already holds rows ({}): an empty target, or one that carries a manifest, is required", + options.target.display(), + table.name() + ))); + } + + // The same premise from the other side: a manifest that counts rows must find them. Asking + // about the store by its file tells a missing store from a present one, not a store this run + // wrote from one truncated, or written by something else, that still has a `data.mdb`. The + // copy would then add the remaining rows to a table that never held the ones the manifest + // counts, and report success over a store missing them; verify would catch it, but only if + // someone runs it. One row tells an empty table, and only counted tables are read. + if let Some(recorded) = &existing { + for table in MigratedTable::ALL { + let counted = recorded.get(table).rows; + if counted == 0 { + continue; + } + let held = target + .scan_batch(table, None, 1) + .await + .map_err(|error| table_failed(table, &error))?; + if held.is_empty() { + return Err(CliError::Migrate(format!( + "{} counts {counted} rows and the store holds none: the store beside the \ + manifest is not the one the run wrote to", + table.name() + ))); + } + } + } + + // R6: written before the first `read_batch`, so a store this tool has begun to fill always + // carries a manifest and a crash after the first batch resumes rather than being refused. + if existing.is_none() { + write_manifest(&options.target, &manifest)?; + } + + if manifest.complete() { + return verify_with(target, options, out).await; + } + + let started = Instant::now(); + // The wall clock the summary reports: the last value is the one taken just after the last + // `write_batch` of this run. + let mut measured: Option = None; + let mut total = 0u64; + let mut events = 0u64; + + for table in MigratedTable::ALL { + let entry = manifest.get(table); + if entry.complete { + continue; + } + let mut rows = entry.rows; + let mut sum = RowSum::from_hex(&entry.checksum)?; + let mut after = decode_key(&entry.after)?; + + loop { + let batch = source + .read_batch(table, after.as_deref(), options.batch_size) + .await + .map_err(|error| table_failed(table, &error))?; + let count = batch.len(); + let written = target + .write_batch(&batch) + .await + .map_err(|error| table_failed(table, &error))?; + measured = Some(started.elapsed()); + + for row in &batch { + sum.add(row)?; + } + rows += written; + total += written; + if table == MigratedTable::Events { + events += written; + } + if let Some(last) = batch.last() { + after = Some(last.key()); + } + + let complete = count < options.batch_size; + let entry = manifest.entry(table); + entry.rows = rows; + entry.checksum = sum.to_hex(); + entry.after = after.as_deref().map(hex::encode).unwrap_or_default(); + entry.complete = complete; + write_manifest(&options.target, &manifest)?; + + if complete { + break; + } + } + } + + let ms = u64::try_from(measured.unwrap_or_default().as_millis()).unwrap_or(u64::MAX); + let rate = events.saturating_mul(1_000) / ms.max(1); + writeln!(out, "Migrated {total} rows in {ms} ms").map_err(write_failed)?; + writeln!(out, "Events throughput: {rate} events/s").map_err(write_failed)?; + // Where the copy landed: a target that was redirected or mistyped is visible here rather + // than as a store the operator cannot find. + writeln!( + out, + "Store: {}", + fs::canonicalize(store_path(options)) + .unwrap_or_else(|_| store_path(options)) + .display() + ) + .map_err(write_failed)?; + + Ok(CommandOutput::default()) +} + +/// The verification pass: fold each table in the store the way the copy folded it, and compare +/// the nine entries with the manifest. +/// +/// Every table is paged to its end, so a table longer than one page is read rather than cut at +/// its first. The pass writes nothing to the store and never opens the source. +pub async fn verify_with( + target: &dyn MigrationTarget, + options: &MigrateOptions, + out: &mut dyn Write, +) -> Result { + refuse_zero_batch(options)?; + let Some(manifest) = read_manifest(&options.target)? else { + return no_manifest(&options.target, out); + }; + + let mut failures: Vec = Vec::new(); + let mut total = 0u64; + + for table in MigratedTable::ALL { + let entry = manifest.get(table); + let mut rows = 0u64; + let mut sum = RowSum::default(); + let mut after: Option> = None; + + loop { + let page = target + .scan_batch(table, after.as_deref(), options.batch_size) + .await + .map_err(|error| table_failed(table, &error))?; + let count = page.len(); + for row in &page { + sum.add(row)?; + } + rows += count as u64; + if let Some(last) = page.last() { + after = Some(last.key()); + } + if count < options.batch_size { + break; + } + } + total += rows; + + if rows != entry.rows { + failures.push(format!( + "[{}] manifest {} rows, target {rows} rows", + table.name(), + entry.rows + )); + } else if sum.to_hex() != entry.checksum { + failures.push(format!( + "[{}] checksum mismatch (manifest {}, target {})", + table.name(), + entry.checksum, + sum.to_hex() + )); + } else if !entry.complete { + failures.push(format!("[{}] the manifest is incomplete", table.name())); + } + } + + if failures.is_empty() { + writeln!( + out, + "Verified {} tables, {total} rows", + MigratedTable::ALL.len() + ) + .map_err(write_failed)?; + return Ok(CommandOutput::default()); + } + + for line in &failures { + writeln!(out, "{line}").map_err(write_failed)?; + } + Ok(CommandOutput { + lines: Vec::new(), + exit_code: 1, + }) +} + +/// The manifest: one entry per table, and the two tables this tool does not move. +/// +/// It carries no timestamp, rate or path, so two runs over the same source produce the same +/// bytes and a resumed run's manifest is the clean run's. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct Manifest { + format: u32, + not_migrated: Vec, + tables: BTreeMap, +} + +impl Manifest { + /// The manifest a first run starts from: every table empty and incomplete. + fn fresh() -> Self { + Self { + format: MANIFEST_FORMAT, + not_migrated: MigratedTable::NOT_MIGRATED + .iter() + .map(|name| (*name).to_owned()) + .collect(), + tables: MigratedTable::ALL + .iter() + .map(|table| (table.name().to_owned(), TableEntry::default())) + .collect(), + } + } + + /// Whether the manifest records that a run did anything: a row written, a cursor moved, or + /// a table finished. A manifest that records none of that cannot have written a row (R7), + /// so a fresh store beside it is honest; one that records progress is only meaningful + /// beside the store the progress went into. + fn records_progress(&self) -> bool { + self.tables + .values() + .any(|entry| entry.rows != 0 || !entry.after.is_empty() || entry.complete) + } + + /// This table's entry, as the empty one when the manifest carries none. + fn get(&self, table: MigratedTable) -> TableEntry { + self.tables.get(table.name()).cloned().unwrap_or_default() + } + + /// This table's entry, created empty when the manifest carries none. + fn entry(&mut self, table: MigratedTable) -> &mut TableEntry { + self.tables.entry(table.name().to_owned()).or_default() + } + + /// Whether every table has been copied to its end. + fn complete(&self) -> bool { + MigratedTable::ALL + .iter() + .all(|table| self.get(*table).complete) + } +} + +/// One table's accumulated state: what the copy has written and where it resumes. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct TableEntry { + /// The rows the table's batches have written. + rows: u64, + /// The hex of the table's running row sum; sixty-four `0` characters when it is empty. + checksum: String, + /// The hex of the last written key; empty before the first batch. + after: String, + /// True once a read returned fewer rows than the batch size. + complete: bool, +} + +impl Default for TableEntry { + fn default() -> Self { + Self { + rows: 0, + checksum: RowSum::default().to_hex(), + after: String::new(), + complete: false, + } + } +} + +/// The manifest's 256-bit row sum. +/// +/// For every row it adds `SHA-256(len32 ‖ key ‖ canonical)` into a wrapping 256-bit sum. The sum +/// is order-independent, needs constant memory, and composes across a resume +/// (`sum(A ∪ B) = sum(A) + sum(B)`); it detects an added, removed or altered row and is stated +/// as a corruption and completeness check, not as a cryptographic commitment +/// (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct RowSum([u64; 4]); + +impl RowSum { + /// Adds one row's digest. + fn add(&mut self, row: &MigrationRow) -> Result<(), CliError> { + let key = row.key(); + let length = u32::try_from(key.len()) + .map_err(|_| CliError::Migrate("a row key is longer than 4 GiB".to_owned()))?; + let canonical = row + .canonical() + .map_err(|error| CliError::Migrate(error.to_string()))?; + + let mut hasher = Sha256::new(); + hasher.update(length.to_be_bytes()); + hasher.update(&key); + hasher.update(&canonical); + let digest = hasher.finalize(); + let (words, _) = digest.as_chunks::<8>(); + sum_words(&mut self.0, words); + Ok(()) + } + + /// The sum a manifest entry carries, as its sixty-four hex characters. + fn from_hex(text: &str) -> Result { + let mut bytes = [0u8; 32]; + hex::decode_to_slice(text, &mut bytes).map_err(|error| { + CliError::Migrate(format!( + "the manifest's checksum is not 64 hexadecimal characters: {error}" + )) + })?; + let mut limbs = [0u64; 4]; + let (words, _) = bytes.as_chunks::<8>(); + for (limb, word) in limbs.iter_mut().zip(words) { + *limb = u64::from_be_bytes(*word); + } + Ok(Self(limbs)) + } + + /// The sum as sixty-four lowercase hex characters. + fn to_hex(self) -> String { + let mut bytes = [0u8; 32]; + let (words, _) = bytes.as_chunks_mut::<8>(); + for (word, limb) in words.iter_mut().zip(self.0) { + *word = limb.to_be_bytes(); + } + hex::encode(bytes) + } +} + +/// Adds four big-endian words into a 256-bit limb sum, modulo 2^256. +/// +/// The manifest documents the checksum as the sum of the row digests modulo 2^256 +/// (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`). Adding the four words +/// independently is a different number whenever a limb wraps, and an independent checker +/// written from the documentation would compute a checksum this tool calls wrong, so the +/// carry has to cross the word boundary. Limb 3 is the least significant — `to_hex` writes +/// limb 0 first — so the carry runs from index 3 up to 0, and the carry out of the top limb +/// is dropped, which is what makes the result the sum modulo 2^256. +fn sum_words(limbs: &mut [u64; 4], words: &[[u8; 8]]) { + let mut carry = 0u128; + for (index, word) in words.iter().enumerate().rev() { + let sum = u128::from(limbs[index]) + u128::from(u64::from_be_bytes(*word)) + carry; + limbs[index] = sum as u64; + carry = sum >> 64; + } +} + +/// The manifest's path under a target root. +fn manifest_path(target: &Path) -> PathBuf { + target.join(MANIFEST_FILE) +} + +/// A comma-separated list, or `none`, for a refusal that names what it found. +fn named_list(names: &[&str]) -> String { + if names.is_empty() { + "none".to_owned() + } else { + names.join(", ") + } +} + +/// The manifest at the target, or `None` when there is none. +fn read_manifest(target: &Path) -> Result, CliError> { + let path = manifest_path(target); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(CliError::Migrate(format!("{}: {error}", path.display()))), + }; + let manifest: Manifest = serde_json::from_slice(&bytes) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + if manifest.format != MANIFEST_FORMAT { + return Err(CliError::Migrate(format!( + "{}: manifest format {} is not supported; this tool reads and writes {MANIFEST_FORMAT}", + path.display(), + manifest.format + ))); + } + + // A manifest is this tool's only when it describes all nine tables and the two it leaves + // alone. `Manifest::get` answers an empty entry for a table the file does not name, so a + // partial file would restart every table at `rows: 0` over a store of unknown contents and + // skip R7's refusal entirely. + let named: BTreeSet<&str> = manifest.tables.keys().map(String::as_str).collect(); + let expected: BTreeSet<&str> = MigratedTable::ALL + .iter() + .map(|table| table.name()) + .collect(); + if named != expected { + let missing: Vec<&str> = expected.difference(&named).copied().collect(); + let unknown: Vec<&str> = named.difference(&expected).copied().collect(); + return Err(CliError::Migrate(format!( + "{}: the manifest does not name this tool's nine tables (missing: {}; unknown: {})", + path.display(), + named_list(&missing), + named_list(&unknown) + ))); + } + if manifest.not_migrated != MigratedTable::NOT_MIGRATED { + let named: Vec<&str> = manifest.not_migrated.iter().map(String::as_str).collect(); + return Err(CliError::Migrate(format!( + "{}: the manifest's not_migrated is not the two tables this tool leaves alone (it \ + names: {})", + path.display(), + named_list(&named) + ))); + } + Ok(Some(manifest)) +} + +/// Replaces the manifest through a temporary file, so a reader never sees a half-written one. +/// +/// The file is created exclusively and written 0600, and both halves are load-bearing. The +/// manifest records the last key written per table, and for `invite_codes` that key is the +/// invite code itself - a bearer credential NIP-43 accepts - so the manifest must not be the +/// world-readable copy of it; the store's own files are heed's 0600. Creating exclusively +/// refuses an entry already at the temporary path, which is how a planted symlink would +/// otherwise redirect the write and be renamed into place. +fn write_manifest(target: &Path, manifest: &Manifest) -> Result<(), CliError> { + let path = manifest_path(target); + let temporary = path.with_extension("json.tmp"); + let bytes = serde_json::to_vec_pretty(manifest) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + + match fs::remove_file(&temporary) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(CliError::Migrate(format!( + "{}: {error}", + temporary.display() + ))); + } + } + + // The rename is the commit point, so it needs both halves of the durability it implies: + // the bytes on disk before the name moves, and the directory entry durable after it. + // Without the file sync, a power loss can leave the new name pointing at a truncated + // manifest; `read_manifest` then refuses it, and the nonempty-target rule refuses to + // migrate over the store it describes, so the operator's only way forward is a discard. + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?; + file.write_all(&bytes) + .and_then(|()| file.sync_all()) + .map_err(|error| CliError::Migrate(format!("{}: {error}", temporary.display())))?; + drop(file); + + fs::rename(&temporary, &path) + .map_err(|error| CliError::Migrate(format!("{}: {error}", path.display())))?; + sync_directory_of(&path) +} + +/// Makes a rename inside a directory durable. +/// +/// A rename changes the directory, not the file, and an unsynced directory can lose that +/// entry on power loss even though the file's own content was synced. +fn sync_directory_of(path: &Path) -> Result<(), CliError> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let directory = fs::File::open(parent) + .map_err(|error| CliError::Migrate(format!("{}: {error}", parent.display())))?; + directory + .sync_all() + .map_err(|error| CliError::Migrate(format!("{}: {error}", parent.display()))) +} + +/// The source the run reads, as the setting that named it: `db.internal:5432/nostream (from +/// DB_URI)`. An unconfigured source is refused here, because `from_env` would otherwise fall +/// back to a local server's `postgres` database and copy whatever it holds. +fn describe_source() -> Result { + if let Some(uri) = std::env::var("DB_URI") + .ok() + .filter(|uri| !uri.trim().is_empty()) + { + // sqlx parses a schemeless string as a URL with no base: the opaque path becomes the + // database name and the host falls back to localhost. The copy would dial the wrong + // server, and a failure echoes the database name - which for a libpq-style string is + // the credentials. The shape is refused before anything is dialled. + if !is_url_with_host(&uri) { + // The value is not echoed: a libpq keyword string (`host=… password=…`) has no + // userinfo to strip, so any echo of what was set would print it. + return Err(CliError::Migrate( + "DB_URI is not a URL with a host: use postgres://user:password@host:5432/database, \ + or postgres:///?host=/var/run/postgresql for a socket" + .into(), + )); + } + return Ok(format!("{} (from DB_URI)", without_credentials(&uri))); + } + match std::env::var("DB_HOST") { + Ok(host) => Ok(format!( + "{host}:{port}/{database} (from DB_HOST, DB_NAME)", + port = std::env::var("DB_PORT").unwrap_or_else(|_| "5432".to_owned()), + database = std::env::var("DB_NAME").unwrap_or_else(|_| "postgres".to_owned()), + )), + Err(_) => Err(CliError::Migrate( + "no source is configured: set DB_URI (for example \ + postgres://user:pass@host:5432/nostream), or DB_HOST and DB_NAME. Nothing was read." + .to_owned(), + )), + } +} + +/// Whether `uri` is a Postgres URL with a host to dial. +/// +/// The gate exists because sqlx reads a string with no base as a URL whose opaque path is the +/// database name and whose host is `localhost`: the copy would dial a server nobody named and +/// then echo the string back in the failure. So this refuses three shapes sqlx happily dials - +/// `postgres://`, `postgres:////` and `postgres:///db`, all `localhost` to sqlx - because a +/// cutover that guesses which server to copy from is worse than one that stops and says so. +/// That divergence is deliberate and pinned by a test. +/// +/// Everything else a Postgres URL can be is left alone: the `postgres` and `postgresql` schemes in +/// either case, an authority with userinfo, an IPv6 literal, a port, a path, a query; and the +/// socket forms `postgres:///?host=/var/run/postgresql` and `postgres:///?hostaddr=127.0.0.1`, +/// whose authority is empty by design. A `?` inside a `#` fragment is not a query, because sqlx +/// ignores the fragment - `postgres://#?host=/sock` has no host at all. +fn is_url_with_host(uri: &str) -> bool { + let Some((scheme, rest)) = uri.split_once("://") else { + return false; + }; + // Only a Postgres scheme. A libpq keyword string can contain `://` by accident + // (`password=ab://cd`), and sqlx would read `password=ab` as the scheme and `cd` as the host. + if !scheme.eq_ignore_ascii_case("postgres") && !scheme.eq_ignore_ascii_case("postgresql") { + return false; + } + // Everything after `#` is a fragment: sqlx ignores it, so a `?` inside one is not a query. + let rest = rest.split('#').next().unwrap_or_default(); + // The authority ends at the first `/` or `?`; everything after it is path and query. + let authority = rest.split(['/', '?']).next().unwrap_or_default(); + let host = authority + .rsplit_once('@') + .map_or(authority, |(_userinfo, host)| host); + // `user@:5432` is an empty host with a port: sqlx refuses it, and so does this. + let host = if host.starts_with('[') { + host.split_once(']').map_or("", |(inner, _rest)| inner) + } else { + host.split(':').next().unwrap_or_default() + }; + if !host.is_empty() { + return true; + } + // The socket forms carry no authority: the host is a query parameter. + rest.split_once('?').is_some_and(|(_path, query)| { + query.split('&').any(|pair| { + matches!( + pair.split('=').next().unwrap_or_default(), + "host" | "hostaddr" + ) + }) + }) +} + +/// A connection URI with its credentials dropped, so an error can name the source without +/// printing a password - or a role - into a terminal's scrollback or a captured log. +/// +/// The query string and the fragment go first, because either can carry `?password=` and +/// neither is part of what identifies the source. Then everything up to the last `@`, which is +/// the userinfo in a URL and the credentials in a schemeless string of the shape libpq accepts +/// (`postgres:alice:secret@host:5432/db`) - the scheme test comes after both, so a string with +/// no `://` is redacted too. Matching credential-shaped keys instead would miss a +/// percent-encoded name, and when a permission error is the problem the server's own message +/// names the role. +fn without_credentials(uri: &str) -> String { + let uri = uri.split('?').next().unwrap_or(uri); + let uri = uri.split('#').next().unwrap_or(uri); + let (scheme, rest) = match uri.split_once("://") { + Some((scheme, rest)) => (Some(scheme), rest), + None => (None, uri), + }; + let rest = rest + .rsplit_once('@') + .map_or(rest, |(_userinfo, after)| after); + match scheme { + Some(scheme) => format!("{scheme}://{rest}"), + None => rest.to_owned(), + } +} + +/// The wait the source's first connection gets. +/// +/// `DB_ACQUIRE_CONNECTION_TIMEOUT` is what the shipped `.env.example` sets, and a probe that +/// inherited it would be a minute of silence on a mistyped host: the budget is that setting +/// capped at `MIGRATION_CONNECT_CAP_MS`, so the ceiling holds whatever the pool is told. +fn probe_budget() -> Duration { + let cap = Duration::from_millis(nostream_storage::postgres::MIGRATION_CONNECT_CAP_MS); + std::env::var("DB_ACQUIRE_CONNECTION_TIMEOUT") + .ok() + .and_then(|value| value.parse::().ok()) + .map(Duration::from_millis) + .unwrap_or(cap) + .min(cap) +} + +/// Refuses a target another local user could write to or redirect. +/// +/// The copy writes the whole dataset into `/lmdb`, so a group- or world-writable target +/// lets another local user pre-place the store and read what lands in it, and a symlinked target +/// sends the copy somewhere the operator did not name. Both are refused before anything is +/// opened. A target that does not exist yet cannot be judged here, so `create_target` creates it +/// before the store opens and runs this check again over what is then on disk. +fn refuse_unsafe_target(options: &MigrateOptions) -> Result<(), CliError> { + refuse_unsafe_target_for(options, effective_uid()) +} + +/// The effective uid of this process, from `/proc/self/status`. +/// +/// `std::fs` exposes a file's owner but not the caller's, and reading it here costs no +/// dependency: this crate runs on Linux, where the kernel publishes it in one line. +fn effective_uid() -> Option { + let status = fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|line| line.starts_with("Uid:"))?; + line.split_whitespace().nth(2)?.parse().ok() +} + +fn refuse_unsafe_target_for(options: &MigrateOptions, caller: Option) -> Result<(), CliError> { + for path in [&options.target, &store_path(options)] { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(CliError::Migrate(format!("{}: {error}", path.display()))); + } + }; + if metadata.file_type().is_symlink() { + return Err(CliError::Migrate(format!( + "{} is a symlink: the target must be the directory the copy lands in", + path.display() + ))); + } + // The owner matters as much as the mode. A directory another user owns can be + // replaced by its owner between this check and the open, whatever its bits say, so a + // copy running as root over a predictable path in a shared directory would follow the + // replacement into files the other user can read. `None` means /proc could not say, and + // then the mode check is what stands. + if let Some(caller) = caller + && metadata.uid() != caller + { + return Err(CliError::Migrate(format!( + "{} is owned by uid {}, not by the user running the copy ({caller}): the target \ + must be a directory that user owns", + path.display(), + metadata.uid() + ))); + } + if !metadata.is_dir() { + return Err(CliError::Migrate(format!( + "{} is not a directory: --target takes the store root, and the environment is \ + created under it", + path.display() + ))); + } + let mode = metadata.permissions().mode(); + if mode & 0o022 != 0 { + return Err(CliError::Migrate(format!( + "{} is writable by group or other ({:04o}): the target must be a directory only \ + its owner can write to", + path.display(), + mode & 0o7777 + ))); + } + } + // The directory that holds the target decides whether the target itself can be replaced. + // Write permission on a directory is what allows an entry in it to be renamed away and + // another put in its place, and it takes no ownership of the entry to do it, so this is a + // mode question rather than a uid one. + refuse_swappable_ancestry(&options.target)?; + + // The data file itself, which `is_file()` and LMDB both follow: a link there would let the + // copy read or write through it, and heed creates `lock.mdb` beside whatever it resolves to. + // A predicate over the file's own metadata is the only way to see that. + let store = nostream_storage::backend::store_file(&options.target); + match fs::symlink_metadata(&store) { + Ok(metadata) => { + if metadata.file_type().is_symlink() { + return Err(CliError::Migrate(format!( + "{} is a symlink: the store's data file must be a regular file", + store.display() + ))); + } + if !metadata.is_file() { + return Err(CliError::Migrate(format!( + "{} is not a regular file: the store's data file must be one", + store.display() + ))); + } + if let Some(caller) = caller + && metadata.uid() != caller + { + return Err(CliError::Migrate(format!( + "{} is owned by uid {}, not by the user running the copy ({caller})", + store.display(), + metadata.uid() + ))); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(CliError::Migrate(format!("{}: {error}", store.display()))); + } + } + + Ok(()) +} + +/// Refuses an ancestor another user could use to replace the target. +/// +/// Renaming an entry needs write permission on the directory that holds it, and no ownership of +/// the entry itself, so a group- or world-writable ancestor lets another user move this target +/// aside and put their own there before the store opens. The sticky bit is the exception that +/// makes `/tmp` usable: it refuses the rename of an entry its writer does not own. +fn refuse_swappable_ancestry(target: &Path) -> Result<(), CliError> { + let mut path = match target.parent() { + Some(parent) if parent.as_os_str().is_empty() => PathBuf::from("."), + Some(parent) => parent.to_path_buf(), + None => return Ok(()), + }; + + loop { + // `metadata` follows, not `symlink_metadata`: the question is the directory the kernel + // would use, and a linked ancestor stands in for the one it points at. + match fs::metadata(&path) { + Ok(metadata) => { + let mode = metadata.permissions().mode(); + if mode & 0o022 != 0 && mode & 0o1000 == 0 { + return Err(CliError::Migrate(format!( + "{} is writable by group or other ({:04o}) and has no sticky bit: another \ + user could replace {} before the store opens. Put the target under a \ + directory only its owner can write to, or under one with the sticky bit", + path.display(), + mode & 0o7777, + target.display() + ))); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(CliError::Migrate(format!("{}: {error}", path.display()))); + } + } + + match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => path = parent.to_path_buf(), + _ => return Ok(()), + } + } +} + +/// Creates the target when it is missing, then judges what is on disk. +/// +/// `refuse_unsafe_target` can only judge what exists, and the source is dialled and probed +/// before the store opens, so a directory another user planted in that window would be adopted +/// by `create_dir_all` without a word. Creating it here, under the mode the store itself uses, +/// and checking again closes the window: after this call the target is a directory this process +/// made, or the run has ended. +fn create_target(options: &MigrateOptions) -> Result<(), CliError> { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&options.target) + .map_err(|error| CliError::Migrate(format!("{}: {error}", options.target.display())))?; + + refuse_unsafe_target(options) +} + +/// The line and exit code a verify answers when the target has no manifest. +fn no_manifest(target: &Path, out: &mut dyn Write) -> Result { + writeln!(out, "No manifest at {}", manifest_path(target).display()).map_err(write_failed)?; + Ok(CommandOutput { + lines: Vec::new(), + exit_code: 1, + }) +} + +/// The line and exit code a verify answers when the target has a manifest but no store. +fn no_store(options: &MigrateOptions, out: &mut dyn Write) -> Result { + writeln!( + out, + "No store at {}: a re-run resumes once the store is back. To start over, use a fresh \ + --target, or remove {} and the store together", + store_path(options).display(), + manifest_path(&options.target).display() + ) + .map_err(write_failed)?; + Ok(CommandOutput { + lines: Vec::new(), + exit_code: 1, + }) +} + +/// The store's directory under a target root: the environment `LmdbBackend::open` would create. +fn store_path(options: &MigrateOptions) -> PathBuf { + nostream_storage::backend::store_dir(&options.target) +} + +/// Whether the target already holds an LMDB environment, asked the way R8 asks it: `/ +/// lmdb/data.mdb` is a file. A directory alone is not a store, and neither is a lock file. +fn store_present(options: &MigrateOptions) -> bool { + nostream_storage::backend::store_exists(&options.target) +} + +/// The first table of a manifest-less target that already holds a row, if any: one row read per +/// table, which is the probe R7 asks for. +async fn first_occupied_table( + target: &dyn MigrationTarget, +) -> Result, CliError> { + for table in MigratedTable::ALL { + let rows = target + .scan_batch(table, None, 1) + .await + .map_err(|error| table_failed(table, &error))?; + if !rows.is_empty() { + return Ok(Some(table)); + } + } + Ok(None) +} + +/// A batch of zero never advances a cursor. The parser refuses it, and these two entry points are +/// public, so they refuse it themselves rather than loop. +fn refuse_zero_batch(options: &MigrateOptions) -> Result<(), CliError> { + if options.batch_size == 0 { + return Err(CliError::Migrate( + "--batch-size must be at least 1".to_owned(), + )); + } + Ok(()) +} + +/// The store requirement R7 puts on a resume. +/// +/// A manifest that records progress is only meaningful beside the store the progress went into: +/// the writable open that follows this check creates a missing store, and the copy would then +/// skip the tables already counted and the rows already summed, exiting 0 against a store +/// missing every row the manifest claims. +/// +/// The question is asked by the store's file and not by opening it. A read-only open registers +/// the environment with heed process-wide, and the writable open of the same path in the same +/// process then answers `BadOpenOptions` — so a probe here would make a resume fail exactly +/// when its store is present. `tests::the_resume_check_leaves_the_store_openable` drives both +/// steps for that reason. +fn check_resume_store( + existing: Option<&Manifest>, + options: &MigrateOptions, +) -> Result<(), CliError> { + if existing.is_some_and(|manifest| manifest.records_progress()) && !store_present(options) { + return Err(CliError::Migrate(format!( + "No store at {}: the manifest records progress, so a re-run resumes once the store is \ + back. To start over, use a fresh --target, or remove {} and the store together", + store_path(options).display(), + manifest_path(&options.target).display() + ))); + } + Ok(()) +} + +/// Opens the store a copy writes, creating it when it is not there. +async fn open_target(root: &Path) -> Result { + LmdbBackend::open(root) + .await + .map_err(|error| CliError::Migrate(error.to_string())) +} + +/// Opens the store a verify reads, read-only and creating nothing (R8). +async fn open_existing_target(root: &Path) -> Result { + LmdbBackend::open_existing(root) + .await + .map_err(|error| CliError::Migrate(error.to_string())) +} + +/// A failure of one table's read or write, naming the table so an operator knows which row +/// stopped the run. +fn table_failed(table: MigratedTable, error: &dyn std::fmt::Display) -> CliError { + CliError::Migrate(format!("{}: {error}", table.name())) +} + +/// One hex resume cursor, or `None` before the table's first batch. +fn decode_key(text: &str) -> Result>, CliError> { + if text.is_empty() { + return Ok(None); + } + hex::decode(text).map(Some).map_err(|error| { + CliError::Migrate(format!( + "the manifest's resume cursor is not hexadecimal: {error}" + )) + }) +} + +/// A write failure is an error, not a success with nothing delivered. +fn write_failed(error: std::io::Error) -> CliError { + CliError::Migrate(format!("unable to write the migration's output: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A link where the data file belongs is followed by `is_file()` and by LMDB both, so the + /// copy would read or write through it and heed would put `lock.mdb` beside the link's + /// target. The predicate over the file's own metadata is what AC18 asks for. + #[test] + #[allow(clippy::expect_used)] + fn a_symlinked_data_file_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let store = directory.path().join("lmdb"); + fs::create_dir(&store).expect("the store directory is made"); + let elsewhere = directory.path().join("elsewhere"); + fs::write(&elsewhere, b"not a store").expect("the target file is written"); + std::os::unix::fs::symlink(&elsewhere, store.join("data.mdb")) + .expect("the link is planted"); + + let options = MigrateOptions { + target: directory.path().to_path_buf(), + batch_size: 1_000, + verify: true, + }; + let error = refuse_unsafe_target_for(&options, effective_uid()) + .expect_err("a linked data file is refused"); + assert!(error.to_string().contains("is a symlink"), "{error}"); + } + + /// The directory's owner decides whether it can be swapped for another one, and it is + /// checked against the caller rather than the mode. Measured as if a different uid were + /// running, because a directory this process owns cannot be made to belong to another user + /// without root. + #[test] + #[allow(clippy::expect_used)] + fn a_target_owned_by_another_user_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let options = MigrateOptions { + target: directory.path().to_path_buf(), + batch_size: 1_000, + verify: false, + }; + // Not uid 0 by assumption: a suite running as root creates a root-owned tempdir, and + // the refusal then has nothing to refuse. + let owner = std::fs::symlink_metadata(directory.path()) + .expect("the directory's metadata") + .uid(); + let other = if owner == 0 { 1 } else { 0 }; + let error = refuse_unsafe_target_for(&options, Some(other)) + .expect_err("a uid that does not own this directory is refused"); + assert!(error.to_string().contains("is owned by uid"), "{error}"); + assert!( + refuse_unsafe_target_for(&options, effective_uid()).is_ok(), + "the user who owns it is allowed, and /proc names them" + ); + } + + /// The directory that holds the target decides whether the target can be replaced, and it + /// needs no ownership of the target to do it: a rename takes write permission on the holder. + /// Without the sticky bit that permission is refused; `/tmp` is exactly this mode plus the + /// sticky bit, which is why the bit, and not the mode alone, is the question. + #[test] + #[allow(clippy::expect_used)] + fn a_target_under_a_swappable_directory_is_refused() { + let temporary = tempfile::tempdir().expect("a temporary directory"); + let shared = temporary.path().join("shared"); + fs::create_dir(&shared).expect("the shared directory is made"); + let options = MigrateOptions { + target: shared.join("target"), + batch_size: 1_000, + verify: false, + }; + + fs::set_permissions(&shared, fs::Permissions::from_mode(0o777)) + .expect("the shared directory is opened"); + let error = refuse_unsafe_target_for(&options, effective_uid()) + .expect_err("a target a group or other user can swap is refused"); + assert!(error.to_string().contains("no sticky bit"), "{error}"); + + fs::set_permissions(&shared, fs::Permissions::from_mode(0o1777)) + .expect("the sticky bit is set"); + assert!( + refuse_unsafe_target_for(&options, effective_uid()).is_ok(), + "a sticky holder is the shape /tmp has, and is allowed" + ); + } + + /// The check that runs before the source is dialled can only judge what exists, so the + /// target is created and then judged. Both halves are needed: the creation gives the second + /// check something to see, and the second check is what refuses a directory another user + /// planted while the source was being probed. + #[test] + #[allow(clippy::expect_used)] + fn a_missing_target_is_created_and_then_judged() { + let temporary = tempfile::tempdir().expect("a temporary directory"); + let target = temporary.path().join("nested").join("target"); + let options = MigrateOptions { + target: target.clone(), + batch_size: 1_000, + verify: false, + }; + + create_target(&options).expect("a missing target is created"); + + let metadata = fs::symlink_metadata(&target).expect("the target exists"); + assert!(metadata.is_dir(), "the target is a directory"); + assert_eq!( + metadata.permissions().mode() & 0o777, + 0o700, + "the target is private to its owner, whatever the umask says" + ); + assert_eq!( + Some(metadata.uid()), + effective_uid(), + "and it belongs to the user who ran the copy" + ); + } + + /// The resume path's store check and the writable open that follows it run in one process, + /// so the check must not open the store: heed keeps an environment registered for the life + /// of the process, and a read-only open there makes the writable one answer + /// `BadOpenOptions` — a resume failing exactly when its store is present. Nothing but both + /// steps together can see it. + #[tokio::test] + #[allow(clippy::expect_used, clippy::unwrap_used)] + async fn the_resume_check_leaves_the_store_openable() { + let directory = tempfile::tempdir().expect("a temporary directory"); + LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let mut manifest = Manifest::fresh(); + manifest.entry(MigratedTable::Events).rows = 1; + let options = MigrateOptions { + target: directory.path().to_path_buf(), + batch_size: 1_000, + verify: false, + }; + + check_resume_store(Some(&manifest), &options) + .expect("the store the manifest needs is there"); + open_target(&options.target) + .await + .expect("the writable open after the check succeeds"); + } + + /// The documented checksum is the sum modulo 2^256, so a wrap in the least significant + /// limb carries into the one above it. Adding the four words independently — which is + /// what this replaced — gives a different number whenever a limb wraps, and a checker + /// written from the documentation would then disagree with the tool. + #[test] + fn a_wrapped_limb_carries_into_the_next_one() { + let mut limbs = [0u64, 0, 0, u64::MAX]; + let words = [[0u8; 8], [0u8; 8], [0u8; 8], 1u64.to_be_bytes()]; + + sum_words(&mut limbs, &words); + + assert_eq!( + limbs, + [0, 0, 1, 0], + "the carry left the least significant limb" + ); + } + + /// The carry out of the most significant limb is dropped, which is the `modulo 2^256` in + /// the documented definition: four saturated limbs plus one sum to zero, not to a carry. + #[test] + fn the_carry_out_of_the_top_limb_is_dropped() { + let mut limbs = [u64::MAX, u64::MAX, u64::MAX, u64::MAX]; + let words = [[0u8; 8], [0u8; 8], [0u8; 8], 1u64.to_be_bytes()]; + + sum_words(&mut limbs, &words); + + assert_eq!(limbs, [0, 0, 0, 0], "the sum wraps at 2^256"); + } + + /// Limb 0 is the most significant, because `to_hex` writes it first; a word added into + /// it must land there and not at the other end. + #[test] + fn the_first_word_is_the_most_significant_limb() { + let mut limbs = [0u64; 4]; + let words = [1u64.to_be_bytes(), [0u8; 8], [0u8; 8], [0u8; 8]]; + + sum_words(&mut limbs, &words); + + assert_eq!(limbs, [1, 0, 0, 0]); + } +} diff --git a/crates/nostream-cli/src/commands/mod.rs b/crates/nostream-cli/src/commands/mod.rs index 3fe4416e..645c0dc8 100644 --- a/crates/nostream-cli/src/commands/mod.rs +++ b/crates/nostream-cli/src/commands/mod.rs @@ -7,6 +7,7 @@ pub mod export; pub mod import; pub mod info; pub mod invite; +pub mod migrate; pub mod seed; pub mod setup; pub mod start; diff --git a/crates/nostream-cli/src/error.rs b/crates/nostream-cli/src/error.rs index 3263dcb3..96a5826a 100644 --- a/crates/nostream-cli/src/error.rs +++ b/crates/nostream-cli/src/error.rs @@ -16,4 +16,9 @@ pub enum CliError { /// The import could not run: a bad argument, a missing file or a storage failure. #[error("{0}")] Import(String), + + /// The migration could not run: a refused row, an unusable manifest or a store that would + /// not open. + #[error("{0}")] + Migrate(String), } diff --git a/crates/nostream-cli/src/main.rs b/crates/nostream-cli/src/main.rs index c04af460..3489932b 100644 --- a/crates/nostream-cli/src/main.rs +++ b/crates/nostream-cli/src/main.rs @@ -49,6 +49,11 @@ enum Command { Info, /// Mint and manage invite codes. Invite, + /// Copy the Postgres store into an LMDB store. + Migrate { + #[command(subcommand)] + command: MigrateCommand, + }, /// Seed the database. Seed, /// Run the first-time setup flow. @@ -87,6 +92,40 @@ enum ConfigCommand { }, } +#[derive(Debug, Subcommand)] +enum MigrateCommand { + /// Copy every table from Postgres into an LMDB store, with the relay stopped. + /// + /// The source is Postgres from `DB_URI`, else the `DB_*` parts. The target is a store root: + /// the environment is `/lmdb` and the manifest `/migration-manifest.json`. + /// + /// The first connection to the source is bounded at ten seconds whatever + /// `DB_ACQUIRE_CONNECTION_TIMEOUT` says: a mistyped host is reported at once with the + /// driver's reason, not after the pool's minute. + /// + /// A re-run resumes where the manifest says the last one stopped. Deleting + /// `/migration-manifest.json` does not restart a half-filled store: a store that holds + /// rows and no manifest is refused, so starting over takes a fresh `--target`, or the + /// manifest and the store removed together. Nothing may write to either store while a run + /// is in progress, so the relay must be stopped on both before the first batch is read. + PgToLmdb { + /// The store root (the environment lives under `/lmdb`). + #[arg(long, value_name = "DIR")] + target: PathBuf, + /// Rows per batch, and the page size verify reads with. + #[arg( + long, + value_name = "N", + default_value_t = 1_000, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..) + )] + batch_size: usize, + /// Verify the store against its manifest and copy nothing. + #[arg(long)] + verify: bool, + }, +} + fn main() -> ExitCode { let cli = Cli::parse(); let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); @@ -109,6 +148,21 @@ fn main() -> ExitCode { } Command::Info => commands::info::run().map(|_| CommandOutput::default()), Command::Invite => commands::invite::run().map(|_| CommandOutput::default()), + Command::Migrate { command } => match command { + MigrateCommand::PgToLmdb { + target, + batch_size, + verify, + } => { + let options = commands::migrate::MigrateOptions { + target, + batch_size, + verify, + }; + let mut stdout = std::io::stdout().lock(); + commands::migrate::run(options, &mut stdout) + } + }, Command::Seed => commands::seed::run().map(|_| CommandOutput::default()), Command::Setup => commands::setup::run().map(|_| CommandOutput::default()), Command::Start => commands::start::run().map(|_| CommandOutput::default()), @@ -136,5 +190,9 @@ fn main() -> ExitCode { eprintln!("Import failed: {message}"); ExitCode::from(1) } + Err(CliError::Migrate(message)) => { + eprintln!("Migration failed: {message}"); + ExitCode::from(1) + } } } diff --git a/crates/nostream-cli/tests/migrate_cli.rs b/crates/nostream-cli/tests/migrate_cli.rs new file mode 100644 index 00000000..6d92b898 --- /dev/null +++ b/crates/nostream-cli/tests/migrate_cli.rs @@ -0,0 +1,2573 @@ +//! The `migrate pg-to-lmdb` command's contract: the copy, the manifest, resume, verify and the +//! run's own throughput measurement. +//! +//! Every test drives `migrate_with`/`verify_with` over a constructed source and a real +//! `LmdbBackend` on a `TempDir`, so no database and no process is involved; the one test that +//! needs a process runs the binary and reads its stderr. +//! +//! These targets may unwrap and assert by panicking: a failed assertion is a failed test. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::process::Command; +use std::sync::Mutex; +use std::time::Duration; + +use chrono::{DateTime, TimeZone, Utc}; +use heed::types::Bytes; +use serde_json::{Value, json}; + +use nostream_cli::commands::migrate::{self, MigrateOptions}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::StorageError; +use nostream_storage::migration::{ + MigratedTable, MigratedUser, MigrationRow, MigrationSource, MigrationTarget, +}; +use nostream_storage::repositories::{ + DeliveryLogEntry, DeliveryLogFilters, DvmJob, DvmJobRepository, DvmJobStatus, EventRecord, + EventRepository, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, + InvoiceUnit, NewReport, Nip05Verification, Nip05VerificationRepository, + NotificationDeliveryLogRepository, NotificationDeliveryStatus, NotificationTargetType, + OutboxMessage, OutboxStatus, Report, ReportRepository, ReportType, StorageFuture, User, + UserRepository, collect_records, +}; + +// --------------------------------------------------------------------------------------- +// Fixtures. +// --------------------------------------------------------------------------------------- + +/// One whole-second instant, 2020-01-01T00:00:00Z plus `offset` seconds, so every fixture +/// timestamp is at the microsecond precision the store keeps. +fn at(offset: i64) -> DateTime { + Utc.timestamp_opt(1_577_836_800 + offset, 0) + .single() + .expect("a whole second") +} + +fn instant(text: &str) -> DateTime { + DateTime::parse_from_rfc3339(text) + .expect("a fixture instant") + .with_timezone(&Utc) +} + +fn event_id(seed: u8) -> EventId { + EventId::from_byte_array([seed; 32]) +} + +fn pubkey(seed: u8) -> Pubkey { + Pubkey::from_byte_array([seed; 32]) +} + +/// A live, unexpiring text note. +fn event_record(seed: u8, content: &str) -> EventRecord { + EventRecord { + row_id: None, + event: Event { + id: event_id(seed), + pubkey: pubkey(1), + created_at: 1_600_000_000 + i64::from(seed), + kind: Kind::TEXT_NOTE, + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([9; 64]), + }, + first_seen: Some(at(0)), + deleted_at: None, + expires_at: None, + event_deduplication: None, + remote_address: None, + } +} + +fn dvm_job(seed: u8) -> DvmJob { + DvmJob { + id: event_id(seed), + requester_pubkey: pubkey(2), + kind: Kind(5100), + worker_index: Some(1), + status: DvmJobStatus::Completed, + result_event_id: Some(event_id(200 + seed)), + error: None, + picked_up_at: Some(at(10)), + completed_at: Some(at(20)), + created_at: at(i64::from(seed)), + updated_at: at(30), + } +} + +fn invite(seed: u8) -> InviteCode { + InviteCode { + code: format!("code-{seed}"), + created_by: Some(pubkey(3)), + claimed_by: None, + expires_at: Some(at(100)), + remaining_uses: Some(2), + created_at: at(i64::from(seed)), + updated_at: at(40), + } +} + +/// The third invoice is the pending one, which is the row the pending index has to hold. +fn invoice(seed: u8) -> Invoice { + let pending = seed == 3; + Invoice { + id: format!("00000000-0000-4000-8000-00000000000{seed}"), + pubkey: pubkey(4), + bolt11: format!("lnbc1invoice{seed}"), + amount_requested: 1_000, + amount_paid: (!pending).then_some(1_000), + unit: InvoiceUnit::Msats, + status: if pending { + InvoiceStatus::Pending + } else { + InvoiceStatus::Completed + }, + description: "an invoice".to_owned(), + confirmed_at: (!pending).then(|| at(50)), + expires_at: Some(at(60)), + updated_at: at(70), + created_at: at(i64::from(seed)), + verify_url: None, + } +} + +fn nip05(seed: u8) -> Nip05Verification { + Nip05Verification { + pubkey: pubkey(20 + seed), + nip05: format!("user{seed}@example.com"), + domain: "example.com".to_owned(), + is_verified: true, + last_verified_at: Some(at(80)), + last_checked_at: at(90), + failure_count: 1, + created_at: at(i64::from(seed)), + updated_at: at(100), + } +} + +fn outbox_id(seed: u8) -> String { + format!("10000000-0000-4000-8000-00000000000{seed}") +} + +fn outbox(seed: u8) -> OutboxMessage { + OutboxMessage { + id: outbox_id(seed), + event_type: "invoice.paid".to_owned(), + payload: json!({ "amount": 1_000 }), + status: OutboxStatus::Delivered, + attempt_count: 1, + available_at: at(120), + last_error: None, + delivered_at: Some(at(130)), + created_at: at(i64::from(seed)), + updated_at: at(140), + } +} + +fn delivery_log(seed: u8) -> DeliveryLogEntry { + DeliveryLogEntry { + id: format!("20000000-0000-4000-8000-00000000000{seed}"), + outbox_id: Some(outbox_id(seed)), + event_type: "invoice.paid".to_owned(), + target_id: "telegram:7175022".to_owned(), + target_type: NotificationTargetType::Telegram, + status: NotificationDeliveryStatus::Success, + attempt_number: 1, + error_snippet: None, + created_at: at(110 + i64::from(seed)), + } +} + +fn report(seed: u8) -> Report { + Report { + id: i32::from(seed), + event_id: event_id(40 + seed), + reporter_pubkey: pubkey(5), + reported_pubkey: Some(pubkey(6)), + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.5, + actionable: true, + created_at: at(i64::from(seed)), + } +} + +fn user(seed: u8) -> MigratedUser { + MigratedUser { + user: User { + pubkey: pubkey(30 + seed), + is_admitted: true, + is_vanished: false, + balance: 0, + created_at: at(i64::from(seed)), + updated_at: at(150), + }, + tos_accepted_at: None, + } +} + +/// Three rows per table, which is the source every whole-run test copies. +fn source_rows() -> Vec { + let mut rows = Vec::new(); + for seed in 1..=3 { + rows.push(MigrationRow::DvmJob(dvm_job(seed))); + rows.push(MigrationRow::Event(event_record(seed, "hello"))); + rows.push(MigrationRow::InviteCode(invite(seed))); + rows.push(MigrationRow::Invoice(invoice(seed))); + rows.push(MigrationRow::Nip05Verification(nip05(seed))); + rows.push(MigrationRow::DeliveryLogEntry(delivery_log(seed))); + rows.push(MigrationRow::OutboxMessage(outbox(seed))); + rows.push(MigrationRow::Report(report(seed))); + rows.push(MigrationRow::User(user(seed))); + } + rows +} + +fn rows_of(rows: &[MigrationRow], table: MigratedTable) -> Vec { + let mut selected: Vec = rows + .iter() + .filter(|row| row.table() == table) + .cloned() + .collect(); + selected.sort_by_key(MigrationRow::key); + selected +} + +// --------------------------------------------------------------------------------------- +// The injected source. +// --------------------------------------------------------------------------------------- + +/// A source that serves a fixed row set, in key order, and records every call it is asked for. +struct StubSource { + rows: Vec, + /// The ordinal of the call that answers `Err`, if any. + fail_at: Option, + calls: Mutex>)>>, +} + +impl StubSource { + fn new(rows: Vec) -> Self { + Self { + rows, + fail_at: None, + calls: Mutex::new(Vec::new()), + } + } + + fn failing_at(rows: Vec, ordinal: usize) -> Self { + Self { + fail_at: Some(ordinal), + ..Self::new(rows) + } + } + + fn calls(&self) -> Vec<(MigratedTable, Option>)> { + self.calls + .lock() + .expect("the counter is not poisoned") + .clone() + } +} + +impl MigrationSource for StubSource { + fn read_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let ordinal = { + let mut calls = self.calls.lock().expect("the counter is not poisoned"); + calls.push((table, after.map(<[u8]>::to_vec))); + calls.len() - 1 + }; + if self.fail_at == Some(ordinal) { + return Err(StorageError::InvalidArgument { + message: "the stub's read failed", + }); + } + + let mut rows = rows_of(&self.rows, table); + if let Some(after) = after { + rows.retain(|row| row.key().as_slice() > after); + } + rows.truncate(limit); + Ok(rows) + }) + } +} + +// --------------------------------------------------------------------------------------- +// Helpers. +// --------------------------------------------------------------------------------------- + +fn options(directory: &std::path::Path, batch_size: usize) -> MigrateOptions { + MigrateOptions { + target: directory.to_owned(), + batch_size, + verify: false, + } +} + +fn manifest_bytes(directory: &std::path::Path) -> Vec { + std::fs::read(directory.join("migration-manifest.json")).expect("the manifest is written") +} + +/// The manifest of the shape R6 writes: nine entries, `rows: 0`, the zero checksum, `after: ""`, +/// and `complete` as the caller asks. +fn write_manifest_fixture(directory: &std::path::Path, complete: bool) { + let manifest = json!({ + "format": 1, + "not_migrated": MigratedTable::NOT_MIGRATED, + "tables": MigratedTable::ALL + .iter() + .map(|table| ( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": complete}), + )) + .collect::>(), + }); + std::fs::write( + directory.join("migration-manifest.json"), + serde_json::to_vec(&manifest).expect("the fixture manifest serialises"), + ) + .expect("the fixture manifest is written"); +} + +fn manifest(directory: &std::path::Path) -> Value { + serde_json::from_slice(&manifest_bytes(directory)).expect("the manifest is JSON") +} + +fn output_text(out: Vec) -> String { + String::from_utf8(out).expect("the command writes utf-8") +} + +/// `time_key`: the flip-signed seconds, then the event id — the shape the two event tail +/// indexes are keyed by. +fn time_key(seconds: i64, id: EventId) -> Vec { + let mut key = ((seconds as u64) ^ (1 << 63)).to_be_bytes().to_vec(); + key.extend_from_slice(id.as_bytes()); + key +} + +/// Every key in one named database, through the public environment handle. +fn database_entries(backend: &LmdbBackend, name: &str) -> Vec<(Vec, Vec)> { + let txn = backend.env().read_txn().expect("a read transaction"); + let database = backend + .env() + .open_database::(&txn, Some(name)) + .expect("the database opens") + .expect("the database exists"); + database + .iter(&txn) + .expect("the database iterates") + .map(|entry| { + let (key, value) = entry.expect("a stored entry"); + (key.to_vec(), value.to_vec()) + }) + .collect() +} + +fn database_keys(backend: &LmdbBackend, name: &str) -> Vec> { + database_entries(backend, name) + .into_iter() + .map(|(key, _)| key) + .collect() +} + +/// Pages one table of the store to its end. +async fn scan_all(target: &dyn MigrationTarget, table: MigratedTable) -> Vec { + let mut rows = Vec::new(); + let mut after: Option> = None; + loop { + let page = target + .scan_batch(table, after.as_deref(), 100) + .await + .expect("the store scans"); + let count = page.len(); + if let Some(last) = page.last() { + after = Some(last.key()); + } + rows.extend(page); + if count < 100 { + return rows; + } + } +} + +// --------------------------------------------------------------------------------------- +// AC2: the whole copy, and a second run that verifies. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_clean_run_copies_every_table_and_a_second_run_writes_nothing() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let source = StubSource::new(source_rows()); + let options = options(directory.path(), 1_000); + + let mut out = Vec::new(); + let output = migrate::migrate_with(&source, &target, &options, &mut out) + .await + .expect("the copy runs"); + assert_eq!(output.exit_code, 0); + + let written = manifest(directory.path()); + assert_eq!(written["format"], 1); + assert_eq!( + written["not_migrated"], + json!(["event_tags", "user_subscriptions"]) + ); + let tables = written["tables"] + .as_object() + .expect("the manifest names tables"); + assert_eq!(tables.len(), 9, "one entry per migrated table"); + for table in MigratedTable::ALL { + let entry = &tables[table.name()]; + assert_eq!(entry["rows"], 3, "{} rows", table.name()); + assert_eq!(entry["complete"], true, "{} is complete", table.name()); + } + + // Each table scans to exactly the source's rows. + for table in MigratedTable::ALL { + assert_eq!( + scan_all(&target, table).await, + rows_of(&source_rows(), table), + "{} scans to the source's rows", + table.name() + ); + } + + // One row per table read back through that table's own read. + let filters = [Filter { + limit: Some(10), + ..Filter::default() + }]; + let events = collect_records( + EventRepository::find_by_filters(&target, &filters) + .await + .expect("the filter read opens"), + ) + .await + .expect("the filter read drains"); + assert_eq!(events.len(), 3); + for seed in 1..=3 { + assert!( + events.contains(&event_record(seed, "hello")), + "event {seed} reads back" + ); + } + + assert_eq!( + UserRepository::find_by_pubkey(&target, &pubkey(31)) + .await + .expect("the user reads") + .expect("the user is stored"), + user(1).user + ); + assert_eq!( + InvoiceRepository::find_by_id(&target, &invoice(1).id) + .await + .expect("the invoice reads") + .expect("the invoice is stored"), + invoice(1) + ); + assert_eq!( + InvoiceRepository::find_pending_invoices(&target, 0, 10) + .await + .expect("the pending index reads"), + vec![invoice(3)] + ); + assert_eq!( + Nip05VerificationRepository::find_by_pubkey(&target, &nip05(1).pubkey) + .await + .expect("the verification reads"), + Some(nip05(1)) + ); + assert_eq!( + InviteCodeRepository::find_by_code(&target, &invite(1).code) + .await + .expect("the code reads"), + Some(invite(1)) + ); + assert_eq!( + DvmJobRepository::find_by_id(&target, &dvm_job(1).id) + .await + .expect("the job reads"), + Some(dvm_job(1)) + ); + assert_eq!( + ReportRepository::find_by_event_id(&target, &report(1).event_id) + .await + .expect("the report reads"), + vec![report(1)] + ); + let mut logged: Vec = + NotificationDeliveryLogRepository::find_recent(&target, 10, &DeliveryLogFilters::default()) + .await + .expect("the log reads") + .into_iter() + .map(|entry| entry.id) + .collect(); + logged.sort(); + assert_eq!( + logged, + vec![delivery_log(1).id, delivery_log(2).id, delivery_log(3).id] + ); + + // The second run: a source that would answer rows and counts its calls. + let counting = StubSource::new(Vec::new()); + let before = manifest_bytes(directory.path()); + let usage = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + let mut out = Vec::new(); + let output = migrate::migrate_with(&counting, &target, &options, &mut out) + .await + .expect("the second run succeeds"); + assert_eq!(output.exit_code, 0); + assert_eq!( + counting.calls().len(), + 0, + "the second run never asked the source for a row" + ); + assert_eq!( + manifest_bytes(directory.path()), + before, + "the manifest is unchanged" + ); + for table in MigratedTable::ALL { + assert_eq!( + scan_all(&target, table).await.len(), + 3, + "{} still holds three rows", + table.name() + ); + } + assert_eq!( + target + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + usage, + "the store was not written" + ); + assert!( + output_text(out).contains("Verified 9 tables"), + "the second run verified rather than short-circuiting" + ); + + // The one read that writes is made after that second run, because it would move a checksum. + let created = ReportRepository::create( + &target, + &NewReport { + event_id: event_id(1), + reporter_pubkey: pubkey(2), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }, + ) + .await + .expect("a report is created"); + assert_eq!(created.id, 4, "the migrated ids raised the sequence"); +} + +// --------------------------------------------------------------------------------------- +// AC3: resume. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn an_interrupted_run_resumes_to_the_clean_run_manifest() { + // The clean run over source S at --batch-size 1. + let clean_directory = tempfile::tempdir().expect("a temporary directory"); + let clean_target = LmdbBackend::open(clean_directory.path()) + .await + .expect("the store opens"); + let clean_options = options(clean_directory.path(), 1); + migrate::migrate_with( + &StubSource::new(source_rows()), + &clean_target, + &clean_options, + &mut Vec::new(), + ) + .await + .expect("the clean run succeeds"); + let clean = manifest_bytes(clean_directory.path()); + + // The interrupted run: the second read_batch answers Err. + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let options = options(directory.path(), 1); + let failing = StubSource::failing_at(source_rows(), 1); + let error = migrate::migrate_with(&failing, &target, &options, &mut Vec::new()) + .await + .expect_err("the second read fails"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a migration failure is the variant `main` prints as `Migration failed:` and exits 1 for" + ); + assert_eq!( + error.to_string(), + "dvm_jobs: invalid argument: the stub's read failed" + ); + + let interrupted = manifest(directory.path()); + let dvm = &interrupted["tables"]["dvm_jobs"]; + assert_eq!(dvm["rows"], 1); + assert_eq!(dvm["complete"], false); + let first_key = dvm_job(1).id.as_bytes().to_vec(); + assert_eq!(dvm["after"], hex::encode(&first_key)); + + // The healthy source over the same target, at --batch-size 1 too. + let resumed = StubSource::new(source_rows()); + let mut out = Vec::new(); + let output = migrate::migrate_with(&resumed, &target, &options, &mut out) + .await + .expect("the resumed run succeeds"); + let calls = resumed.calls(); + + // 1. The run measured the resume rather than restarting the table. + assert_eq!(calls[0].0, MigratedTable::DvmJobs); + assert_eq!( + calls[0].1, + Some(first_key), + "the first call resumes at the manifest's cursor" + ); + // 2. The cursor advances per batch. + assert_eq!(calls[1].0, MigratedTable::DvmJobs); + assert_eq!(calls[1].1, Some(dvm_job(2).id.as_bytes().to_vec())); + + // 3. The resumed run lands on the clean run's manifest, byte for byte. + assert_eq!(output.exit_code, 0); + assert_eq!(manifest_bytes(directory.path()), clean); +} + +// --------------------------------------------------------------------------------------- +// AC4: one changed column moves only its table's checksum. +// --------------------------------------------------------------------------------------- + +/// Copies `rows` into a fresh store and answers its manifest. +async fn manifest_of(rows: Vec) -> Value { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::new(rows), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + manifest(directory.path()) +} + +#[tokio::test] +async fn one_changed_source_row_changes_only_its_table_checksum() { + let clean = manifest_of(source_rows()).await; + + let mut richer = source_rows(); + for row in &mut richer { + if let MigrationRow::User(migrated) = row + && migrated.user.pubkey == user(1).user.pubkey + { + migrated.user.balance = 999; + } + } + let balance = manifest_of(richer).await; + + let mut accepted = source_rows(); + for row in &mut accepted { + if let MigrationRow::User(migrated) = row + && migrated.user.pubkey == user(1).user.pubkey + { + migrated.tos_accepted_at = Some(at(7)); + } + } + let tos = manifest_of(accepted).await; + + for variant in [&balance, &tos] { + assert_eq!( + variant["tables"]["users"]["rows"], + clean["tables"]["users"]["rows"] + ); + assert_ne!( + variant["tables"]["users"]["checksum"], + clean["tables"]["users"]["checksum"] + ); + for table in MigratedTable::ALL { + if table == MigratedTable::Users { + continue; + } + assert_eq!( + variant["tables"][table.name()]["checksum"], + clean["tables"][table.name()]["checksum"], + "{} is unchanged", + table.name() + ); + } + } +} + +// --------------------------------------------------------------------------------------- +// AC5: verify pages the whole table. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn verify_accepts_the_runs_own_target_and_rejects_a_changed_row() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let rows: Vec = (1..=5) + .map(|seed| MigrationRow::Event(event_record(seed, "hello"))) + .collect(); + // Five events at --batch-size 2 is three pages, so a verify that reads one page sees two. + let options = options(directory.path(), 2); + migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut Vec::new()) + .await + .expect("the copy succeeds"); + + let before = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + let mut out = Vec::new(); + let output = migrate::verify_with(&target, &options, &mut out) + .await + .expect("the verify runs"); + assert_eq!(output.exit_code, 0); + assert!(output_text(out).contains("Verified 9 tables")); + + // A same-length content change under the same key: the count cannot move. + let mut changed = event_record(1, "hello"); + changed.event.content = "jello".to_owned(); + target + .write_batch(&[MigrationRow::Event(changed)]) + .await + .expect("the changed row is written"); + + let mut out = Vec::new(); + let output = migrate::verify_with(&target, &options, &mut out) + .await + .expect("the verify runs"); + assert_eq!(output.exit_code, 1); + let text = output_text(out); + let line = text + .lines() + .find(|line| line.starts_with("[events] checksum mismatch (manifest ")) + .unwrap_or_else(|| panic!("a checksum line is printed, got:\n{text}")); + let body = line + .strip_prefix("[events] checksum mismatch (manifest ") + .expect("the line's prefix") + .trim_end_matches(')'); + let (manifest_hex, target_hex) = body.split_once(", target ").expect("both checksums"); + assert_eq!(manifest_hex.len(), 64, "the manifest's checksum: {line}"); + assert_eq!(target_hex.len(), 64, "the target's checksum: {line}"); + + assert_eq!( + target + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + before, + "verify wrote nothing to the store" + ); +} + +// --------------------------------------------------------------------------------------- +// AC6: no manifest. +// --------------------------------------------------------------------------------------- + +/// The directory's entries with their kind, sorted by name: what a verify must leave as it found +/// it. +fn entries(path: &std::path::Path) -> Vec<(String, bool)> { + let mut entries: Vec<(String, bool)> = std::fs::read_dir(path) + .expect("the directory reads") + .map(|entry| { + let entry = entry.expect("an entry reads"); + ( + entry.file_name().to_string_lossy().into_owned(), + entry.file_type().expect("the entry's type").is_dir(), + ) + }) + .collect(); + entries.sort(); + entries +} + +// `run` builds a runtime of its own, so this test cannot be a `#[tokio::test]`: the async half +// is driven by a runtime this test owns, and the two `run` calls happen outside any `block_on`. +#[test] +fn verify_creates_nothing_and_rejects_a_target_with_no_store_or_no_manifest() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the test runtime builds"); + let directory = tempfile::tempdir().expect("a temporary directory"); + + // Half one: a target with no store. The listing is the same afterwards. + let before = entries(directory.path()); + let mut out = Vec::new(); + let output = migrate::run( + MigrateOptions { + verify: true, + ..options(directory.path(), 1_000) + }, + &mut out, + ) + .expect("a missing store is an answer, not a failure"); + assert_eq!(output.exit_code, 1); + let text = output_text(out); + assert!(text.contains("No store at"), "got: {text}"); + assert!( + text.contains(&directory.path().join("lmdb").display().to_string()), + "the line names the store it looked for: {text}" + ); + assert_eq!(entries(directory.path()), before, "verify created nothing"); + + // Half two: a store a copy filled, and whose manifest has since been removed. + let copy = options(directory.path(), 1_000); + let (usage, counts) = runtime.block_on(async { + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + ©, + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + let usage = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + let mut counts = Vec::new(); + for table in MigratedTable::ALL { + counts.push(scan_all(&target, table).await.len()); + } + (usage, counts) + }); + std::fs::remove_file(directory.path().join("migration-manifest.json")) + .expect("the manifest is removed"); + + let mut out = Vec::new(); + let output = migrate::run( + MigrateOptions { + verify: true, + ..options(directory.path(), 1_000) + }, + &mut out, + ) + .expect("a missing manifest is an answer, not a failure"); + assert_eq!(output.exit_code, 1); + let text = output_text(out); + assert!(text.contains("No manifest at"), "got: {text}"); + assert!( + text.contains( + &directory + .path() + .join("migration-manifest.json") + .display() + .to_string() + ), + "the line names the manifest it looked for: {text}" + ); + + runtime.block_on(async { + let reopened = LmdbBackend::open(directory.path()) + .await + .expect("the store reopens"); + for (table, count) in MigratedTable::ALL.iter().zip(counts) { + assert_eq!( + scan_all(&reopened, *table).await.len(), + count, + "{} is unchanged", + table.name() + ); + } + assert_eq!( + reopened + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + usage, + "the store's bytes are unchanged" + ); + }); +} + +// --------------------------------------------------------------------------------------- +// AC17: a crash between the first batch's commit and its manifest write. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_crash_between_the_first_batch_and_the_manifest_write_resumes_to_the_clean_manifest() { + let table = MigratedTable::ALL[0]; + + // The uninterrupted run: the manifest and the counts a resume has to land on. + let clean = tempfile::tempdir().expect("a temporary directory"); + let clean_store = LmdbBackend::open(clean.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::new(source_rows()), + &clean_store, + &options(clean.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the clean copy succeeds"); + let clean_manifest = manifest_bytes(clean.path()); + let mut clean_counts = Vec::new(); + for table in MigratedTable::ALL { + clean_counts.push(scan_all(&clean_store, table).await.len()); + } + drop(clean_store); + + // The crash state, reproduced rather than killed: the fresh manifest R6 writes before its + // first batch, then that batch's rows, and nothing else. A source that fails on its first + // read leaves exactly that manifest behind, and its entries are asserted below. + let crashed = tempfile::tempdir().expect("a temporary directory"); + let store = LmdbBackend::open(crashed.path()) + .await + .expect("the store opens"); + let options = options(crashed.path(), 1_000); + let error = migrate::migrate_with( + &StubSource::failing_at(source_rows(), 0), + &store, + &options, + &mut Vec::new(), + ) + .await + .expect_err("the first read fails"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a source failure is the variant `main` reports as `Migration failed:`" + ); + + let fresh = manifest_bytes(crashed.path()); + let fresh: Value = serde_json::from_slice(&fresh).expect("the fresh manifest is json"); + assert_eq!(fresh["format"], 1); + assert_eq!( + fresh["not_migrated"], + json!(["event_tags", "user_subscriptions"]) + ); + let tables = fresh["tables"].as_object().expect("the tables object"); + assert_eq!(tables.len(), 9, "one entry per table"); + for entry in tables.values() { + assert_eq!(entry["rows"], 0); + assert_eq!(entry["checksum"], "0".repeat(64)); + assert_eq!(entry["after"], ""); + assert_eq!(entry["complete"], false); + } + + let source = StubSource::new(source_rows()); + let first = MigrationSource::read_batch(&source, table, None, 200) + .await + .expect("the first batch reads"); + assert!(!first.is_empty(), "the first table has rows"); + store + .write_batch(&first) + .await + .expect("the first batch is stored"); + + // The resume replays that batch from the fresh manifest's own cursor. + let resumed = StubSource::new(source_rows()); + let output = migrate::migrate_with(&resumed, &store, &options, &mut Vec::new()) + .await + .expect("the resume succeeds"); + assert_eq!(output.exit_code, 0); + assert_eq!( + resumed.calls().first(), + Some(&(table, None)), + "the resume starts at the fresh manifest's empty cursor" + ); + for (table, count) in MigratedTable::ALL.iter().zip(clean_counts) { + assert_eq!( + scan_all(&store, *table).await.len(), + count, + "{} count", + table.name() + ); + } + assert_eq!( + manifest_bytes(crashed.path()), + clean_manifest, + "the resume lands on the uninterrupted run's manifest" + ); +} + +// --------------------------------------------------------------------------------------- +// AC7: the run's own measurement. +// --------------------------------------------------------------------------------------- + +/// One event whose id is its index, so a thousand of them are a thousand keys in key order. +fn indexed_event(index: u64) -> MigrationRow { + let mut record = event_record(1, "hello"); + let mut id = [0u8; 32]; + id[..8].copy_from_slice(&index.to_be_bytes()); + record.event.id = EventId::from_byte_array(id); + MigrationRow::Event(record) +} + +/// The two summary lines, parsed. +fn summary(text: &str) -> (u64, u64, u64, String) { + let mut lines = text.lines(); + let first = lines.next().expect("the rows line"); + let second = lines.next().expect("the throughput line"); + let third = lines.next().expect("the store line"); + assert_eq!(lines.next(), None, "exactly three summary lines"); + + let (rows_text, rest) = first + .strip_prefix("Migrated ") + .and_then(|rest| rest.split_once(" rows in ")) + .expect("the rows line's shape"); + let ms_text = rest.strip_suffix(" ms").expect("the milliseconds"); + let rate_text = second + .strip_prefix("Events throughput: ") + .and_then(|rest| rest.strip_suffix(" events/s")) + .expect("the throughput line's shape"); + + let store = third + .strip_prefix("Store: ") + .expect("the store line's shape") + .to_owned(); + + ( + rows_text.parse().expect("a row count"), + ms_text.parse().expect("a millisecond count"), + rate_text.parse().expect("a rate"), + store, + ) +} + +/// A target that sleeps inside every batch it writes, so the run has work its own clock cannot +/// miss: five batches of 200 rows pay at least 100 ms. +struct SleepingTarget { + inner: LmdbBackend, +} + +impl MigrationTarget for SleepingTarget { + fn write_batch<'a>(&'a self, rows: &'a [MigrationRow]) -> StorageFuture<'a, u64> { + Box::pin(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + MigrationTarget::write_batch(&self.inner, rows).await + }) + } + + fn scan_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + MigrationTarget::scan_batch(&self.inner, table, after, limit) + } +} + +#[tokio::test] +async fn the_run_prints_a_rate_consistent_with_its_own_events_and_milliseconds() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let store = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let target = SleepingTarget { inner: store }; + // 1 000 events at --batch-size 200 is five full batches plus the closing short read, and the + // target sleeps 20 ms in each, so the copy cannot finish in less than 100 ms. + let rows: Vec = (0..1_000).map(indexed_event).collect(); + let options = options(directory.path(), 200); + + let mut out = Vec::new(); + migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut out) + .await + .expect("the copy succeeds"); + let (rows, ms, rate, store) = summary(&output_text(out)); + assert_eq!( + store, + std::fs::canonicalize(directory.path().join("lmdb")) + .expect("the store exists") + .display() + .to_string(), + "the summary names the canonical store the copy landed in" + ); + + assert_eq!(rows, 1_000); + assert!( + ms >= 100, + "the run's clock includes the 100 ms the target slept: printed {ms} ms" + ); + assert_eq!( + rate, + 1_000 * 1_000 / ms.max(1), + "the rate is the printed numbers'" + ); + assert!(rate > 0); +} + +// --------------------------------------------------------------------------------------- +// AC8: a migrated event keeps what the traits would lose. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_migrated_event_keeps_first_seen_deduplication_and_its_deleted_state() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut live = event_record(1, "live"); + live.first_seen = Some(instant("2020-01-01T00:00:00Z")); + live.remote_address = Some("203.0.113.7".parse().expect("a fixture address")); + live.event_deduplication = Some(vec![json!("abc")]); + live.expires_at = Some(1_893_456_000); + + let mut deleted = event_record(2, "deleted"); + deleted.deleted_at = Some(at(500)); + + let mut expired = event_record(3, "expired"); + expired.expires_at = Some(1_577_836_800); + + let rows = vec![ + MigrationRow::Event(live.clone()), + MigrationRow::Event(deleted.clone()), + MigrationRow::Event(expired.clone()), + ]; + migrate::migrate_with( + &StubSource::new(rows.clone()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + + assert_eq!( + scan_all(&target, MigratedTable::Events).await, + rows_of(&rows, MigratedTable::Events) + ); + + // The criterion's split is the store's own: `lmdb/events.rs:775-807` builds a filter page with + // `live: false`, so a soft-deleted or expired row is readable by filter, and + // `count_by_filters` is the read that applies the live predicate (ADR 0011 D4, pinned by + // `parity_lmdb::a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count`). The + // count is the half that proves B and C are stored *and* marked. + // + // A copy that read its **source** through `find_by_filters` would lose B and C instead: the + // Postgres composer adds `deleted_at IS NULL AND (expires_at IS NULL OR expires_at > $now)` + // (`postgres/sql.rs:265`), which is why the seam's reader has no filter at all. + let filters = [Filter { + limit: Some(10), + ..Filter::default() + }]; + let read = collect_records( + EventRepository::find_by_filters(&target, &filters) + .await + .expect("the filter read opens"), + ) + .await + .expect("the filter read drains"); + assert_eq!(read.len(), 3, "every stored row reads back by filter"); + let live_read = read + .iter() + .find(|record| record.event.id == live.event.id) + .expect("the live row reads back"); + assert_eq!(live_read.first_seen, live.first_seen); + assert_eq!(live_read.remote_address, live.remote_address); + assert_eq!(live_read.event_deduplication, live.event_deduplication); + assert_eq!(live_read.expires_at, live.expires_at); + let deleted_read = read + .iter() + .find(|record| record.event.id == deleted.event.id) + .expect("the soft-deleted row reads back"); + assert_eq!(deleted_read.deleted_at, deleted.deleted_at); + let expired_read = read + .iter() + .find(|record| record.event.id == expired.event.id) + .expect("the expired row reads back"); + assert_eq!(expired_read.expires_at, expired.expires_at); + assert_eq!( + EventRepository::count_by_filters(&target, &filters) + .await + .expect("the count runs"), + 1, + "the live predicate holds for the count, so only the live row is counted" + ); + + // Both tail indexes are stated separately: a value-only copy passes every read above. + let deleted_at = deleted.deleted_at.expect("the fixture is deleted"); + assert_eq!( + database_keys(&target, "events_deleted"), + vec![time_key(deleted_at.timestamp(), deleted.event.id)], + "one entry, keyed by the deleted instant in whole seconds" + ); + + let mut expiring = database_keys(&target, "events_expiring"); + let mut expected = vec![ + time_key( + live.expires_at.expect("the live row expires"), + live.event.id, + ), + time_key( + expired.expires_at.expect("the expired row expires"), + expired.event.id, + ), + ]; + expiring.sort(); + expected.sort(); + assert_eq!(expiring, expected); +} + +// --------------------------------------------------------------------------------------- +// AC9: a refused row stops the run. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_refused_row_stops_the_run_and_keeps_the_resume_point() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + /// A tag whose key is past LMDB's 511-byte bound: `2 + 601 + 8 + 32` bytes. + fn refused(seed: u8) -> EventRecord { + let mut record = event_record(seed, "refused"); + record.event.tags = vec![vec!["t".to_owned(), "a".repeat(600)]]; + record + } + + let rows: Vec = (1..=2) + .map(|seed| MigrationRow::Event(event_record(seed, "fine"))) + .chain(std::iter::once(MigrationRow::Event(refused(3)))) + .collect(); + // The refused row is in the second batch of two. + let options = options(directory.path(), 2); + let error = migrate::migrate_with(&StubSource::new(rows), &target, &options, &mut Vec::new()) + .await + .expect_err("the refused key fails the batch"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a migration failure is the variant `main` prints as `Migration failed:` and exits 1 for" + ); + assert!( + error.to_string().starts_with("events: "), + "the message names the table whose write failed, not a batch ordinal: {error}" + ); + + let written = manifest(directory.path()); + let events = &written["tables"]["events"]; + assert_eq!(events["complete"], false); + assert_eq!(events["rows"], 2); + assert_eq!( + events["after"], + hex::encode(event_record(2, "fine").event.id.as_bytes()) + ); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 2, + "the first batch is the store's content" + ); + + // The resume point is usable: a healthy source continues where the refused row stopped. + let resumed = StubSource::new(vec![MigrationRow::Event(event_record(3, "fine"))]); + let output = migrate::migrate_with(&resumed, &target, &options, &mut Vec::new()) + .await + .expect("the run resumes after the refusal"); + assert_eq!(output.exit_code, 0); + assert_eq!(resumed.calls()[0].0, MigratedTable::Events); + assert_eq!( + resumed.calls()[0].1, + Some(event_record(2, "fine").event.id.as_bytes().to_vec()) + ); +} + +// --------------------------------------------------------------------------------------- +// AC16: a target that already holds rows and carries no manifest. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_target_with_rows_and_no_manifest_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + // Three rows in one table, through the seam's own write, and no manifest anywhere. + let seeded: Vec = (1..=3) + .map(|seed| MigrationRow::Event(event_record(seed, "seeded"))) + .collect(); + assert_eq!( + target.write_batch(&seeded).await.expect("the seed writes"), + 3 + ); + let manifest = directory.path().join("migration-manifest.json"); + assert!(!manifest.exists(), "the seed writes no manifest"); + + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("a target with rows and no manifest is refused"); + assert!( + matches!(error, nostream_cli::CliError::Migrate(_)), + "a migration failure is the variant `main` prints as `Migration failed:` and exits 1 for" + ); + let message = error.to_string(); + assert!(message.contains("already holds rows"), "{message}"); + assert!( + message.contains(&directory.path().display().to_string()), + "the message names the target: {message}" + ); + assert_eq!(source.calls().len(), 0, "nothing was read"); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 3, + "the seeded rows are untouched" + ); + for table in MigratedTable::ALL { + if table == MigratedTable::Events { + continue; + } + assert_eq!( + scan_all(&target, table).await.len(), + 0, + "{} is untouched", + table.name() + ); + } + assert!(!manifest.exists(), "no manifest was written"); + + // The control: the same source into an empty directory proceeds. + let empty = tempfile::tempdir().expect("a temporary directory"); + let clean = LmdbBackend::open(empty.path()) + .await + .expect("the store opens"); + let output = migrate::migrate_with( + &StubSource::new(source_rows()), + &clean, + &options(empty.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("an empty target proceeds"); + assert_eq!(output.exit_code, 0); + assert!( + empty.path().join("migration-manifest.json").exists(), + "the control run wrote its manifest" + ); +} + +// --------------------------------------------------------------------------------------- +// A manifest of another format, a batch of zero, and a verify with no store. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_manifest_of_another_format_is_refused_before_anything_is_written() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + std::fs::write( + directory.path().join("migration-manifest.json"), + br#"{"format":2,"not_migrated":[],"tables":{}}"#, + ) + .expect("the fixture manifest is written"); + let usage = target + .map_usage() + .await + .expect("the usage reads") + .used_bytes; + + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("format 2 is refused"); + let message = error.to_string(); + assert!(message.contains("migration-manifest.json"), "{message}"); + assert!(message.contains("format 2"), "{message}"); + assert_eq!(source.calls().len(), 0, "nothing was read"); + assert_eq!( + target + .map_usage() + .await + .expect("the usage reads") + .used_bytes, + usage, + "nothing was written" + ); + + let error = migrate::verify_with(&target, &options(directory.path(), 1_000), &mut Vec::new()) + .await + .expect_err("format 2 is refused by verify too"); + assert!(error.to_string().contains("format 2"), "{error}"); +} + +#[tokio::test] +async fn a_manifest_that_does_not_name_the_nine_tables_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + // A store with three rows already in it: a partial manifest must not let a copy restart + // every table at `rows: 0` over them, which is what R7's refusal exists to prevent. + let seeded: Vec = (1..=3) + .map(|seed| MigrationRow::Event(event_record(seed, "seeded"))) + .collect(); + assert_eq!( + target.write_batch(&seeded).await.expect("the seed writes"), + 3 + ); + let manifest = directory.path().join("migration-manifest.json"); + + std::fs::write(&manifest, br#"{"format":1,"not_migrated":[],"tables":{}}"#) + .expect("the fixture manifest is written"); + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("a manifest that names no tables is refused"); + let message = error.to_string(); + assert!(message.contains("nine tables"), "got: {message}"); + for table in MigratedTable::ALL { + assert!( + message.contains(table.name()), + "the refusal names {}: {message}", + table.name() + ); + } + assert_eq!(source.calls().len(), 0, "nothing was read"); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 3, + "the seeded rows are untouched" + ); + + // The nine tables, and a `not_migrated` that is not this tool's. + let mut tables = serde_json::Map::new(); + for table in MigratedTable::ALL { + tables.insert( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}), + ); + } + let fixture = json!({"format": 1, "not_migrated": ["event_tags"], "tables": tables}); + std::fs::write( + &manifest, + serde_json::to_vec(&fixture).expect("the fixture serialises"), + ) + .expect("the fixture manifest is written"); + let error = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("the wrong not_migrated list is refused"); + assert!(error.to_string().contains("not_migrated"), "{error}"); + assert_eq!( + scan_all(&target, MigratedTable::Events).await.len(), + 3, + "the seeded rows are untouched" + ); + + // Nine names, one of them swapped for a name that is not a table. A check that counted the + // entries would accept this, so the refusal has to be about which names they are. + let mut tables = serde_json::Map::new(); + for table in MigratedTable::ALL.iter().skip(1) { + tables.insert( + table.name().to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}), + ); + } + tables.insert( + "not_a_table".to_owned(), + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}), + ); + let fixture = + json!({"format": 1, "not_migrated": MigratedTable::NOT_MIGRATED, "tables": tables}); + std::fs::write( + &manifest, + serde_json::to_vec(&fixture).expect("the fixture serialises"), + ) + .expect("the fixture manifest is written"); + let error = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("one swapped name is refused even though the count is right"); + let message = error.to_string(); + assert!(message.contains("not_a_table"), "{message}"); + assert!( + message.contains(MigratedTable::ALL[0].name()), + "the refusal names the table that is missing: {message}" + ); +} + +#[tokio::test] +async fn a_zero_batch_is_refused_instead_of_looping() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let source = StubSource::new(source_rows()); + let error = migrate::migrate_with( + &source, + &target, + &options(directory.path(), 0), + &mut Vec::new(), + ) + .await + .expect_err("a batch of zero is refused"); + assert!(error.to_string().contains("--batch-size"), "{error}"); + assert_eq!(source.calls().len(), 0, "nothing was read"); + + let error = migrate::verify_with(&target, &options(directory.path(), 0), &mut Vec::new()) + .await + .expect_err("a batch of zero is refused by verify too"); + assert!(error.to_string().contains("--batch-size"), "{error}"); +} + +#[test] +fn verify_refuses_a_manifest_without_a_store_instead_of_creating_one() { + let directory = tempfile::tempdir().expect("a temporary directory"); + write_manifest_fixture(directory.path(), true); + let store = directory.path().join("lmdb"); + + for verify in [true, false] { + let mut out = Vec::new(); + let output = migrate::run( + MigrateOptions { + verify, + ..options(directory.path(), 1_000) + }, + &mut out, + ) + .expect("a missing store is an answer, not a failure"); + assert_eq!(output.exit_code, 1); + let text = output_text(out); + assert!(text.contains("No store at"), "{text}"); + assert!( + text.contains(&store.display().to_string()), + "the line names the store: {text}" + ); + assert!( + !store.exists(), + "verify created no store (verify = {verify})" + ); + } +} + +// --------------------------------------------------------------------------------------- +// The seam's index writes, asserted where the store keeps them. +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn a_migrated_replaceable_event_writes_the_pointer_its_kind_index_is_keyed_by() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut record = event_record(7, "the first value"); + record.event.kind = Kind(10000); + // A stored, empty deduplication list: the author and kind alone, with no `d` element. + record.event_deduplication = Some(Vec::new()); + target + .write_batch(&[MigrationRow::Event(record.clone())]) + .await + .expect("the event writes"); + + let entries = database_entries(&target, "replaceable"); + assert_eq!(entries.len(), 1, "one pointer for the author and kind"); + let (key, value) = &entries[0]; + assert_eq!( + value.as_slice(), + record.event.id.as_bytes(), + "the pointer carries the event id" + ); + assert_eq!( + key.len(), + 36, + "pubkey ++ kind, with no deduplication element" + ); + assert_eq!(&key[..32], record.event.pubkey.as_bytes()); + + // A second event of the same author and kind replaces the pointer at the same key, so the + // key is the author and kind rather than the id. + let mut second = record.clone(); + second.event.id = event_id(9); + target + .write_batch(&[MigrationRow::Event(second.clone())]) + .await + .expect("the replacement writes"); + let entries = database_entries(&target, "replaceable"); + assert_eq!(entries.len(), 1, "one pointer per author and kind"); + assert_eq!(entries[0].0, *key, "the same key"); + assert_eq!(entries[0].1.as_slice(), second.event.id.as_bytes()); +} + +#[tokio::test] +async fn a_migrated_delivery_without_an_outbox_id_stores_no_outbox_pointer() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut orphan = delivery_log(1); + orphan.outbox_id = None; + let linked = delivery_log(2); + target + .write_batch(&[ + MigrationRow::DeliveryLogEntry(orphan.clone()), + MigrationRow::DeliveryLogEntry(linked.clone()), + ]) + .await + .expect("the rows write"); + + let rows = scan_all(&target, MigratedTable::NotificationDeliveryLog).await; + let stored = rows + .iter() + .find_map(|row| match row { + MigrationRow::DeliveryLogEntry(entry) if entry.id == orphan.id => Some(entry), + _ => None, + }) + .expect("the row without an outbox id is stored"); + assert_eq!(stored.outbox_id, None, "the NULL outbox id survives"); + assert_eq!(stored.event_type, orphan.event_type); + + let pointers = database_entries(&target, "delivery_log_by_outbox"); + assert_eq!(pointers.len(), 1, "only the row with an outbox id has one"); + assert!( + pointers[0] + .0 + .windows(linked.id.len()) + .any(|window| window == linked.id.as_bytes()), + "the pointer names the linked row" + ); +} + +#[tokio::test] +async fn a_migrated_job_writes_the_status_index_its_finder_reads() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + // Same `created_at`, one status apart, so the keys can differ only in the status. + let mut submitted = dvm_job(3); + submitted.status = DvmJobStatus::Submitted; + submitted.created_at = at(5); + let mut completed = dvm_job(4); + completed.created_at = at(5); + for job in [&submitted, &completed] { + target + .write_batch(&[MigrationRow::DvmJob(job.clone())]) + .await + .expect("the job writes"); + } + + let entries = database_entries(&target, "dvm_jobs_by_status_created_at"); + assert_eq!(entries.len(), 2, "one pointer per job"); + let pointer_for = |job: &DvmJob| { + entries + .iter() + .find(|(key, _)| key.ends_with(job.id.as_bytes())) + .map(|(key, value)| (key.clone(), value.clone())) + .expect("a pointer for the job") + }; + for job in [&submitted, &completed] { + let (_, value) = pointer_for(job); + assert!(value.is_empty(), "a status pointer carries no value"); + } + + // Both jobs have the same `created_at`, so the bytes before the id are the status: they + // differ, and the finder's status prefix is what the writer put there. + let submitted_prefix = { + let (key, _) = pointer_for(&submitted); + key[..key.len() - 32].to_vec() + }; + let completed_prefix = { + let (key, _) = pointer_for(&completed); + key[..key.len() - 32].to_vec() + }; + assert_ne!( + submitted_prefix, completed_prefix, + "the status leads the key, after the timestamp part" + ); +} + +// --------------------------------------------------------------------------------------- +// AC18: a manifest that records progress over a missing store. +// --------------------------------------------------------------------------------------- + +// The first half needs the binary's stderr and `run` builds a runtime of its own, so this is a +// plain test with a runtime it owns. +#[test] +fn a_manifest_that_records_progress_over_a_missing_store_is_refused() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the test runtime builds"); + + // Half one: a copy at --batch-size 2 that stops after its first batch, then its store gone. + let partial = tempfile::tempdir().expect("a temporary directory"); + let partial_options = options(partial.path(), 2); + let error = runtime.block_on(async { + let target = LmdbBackend::open(partial.path()) + .await + .expect("the store opens"); + migrate::migrate_with( + &StubSource::failing_at(source_rows(), 1), + &target, + &partial_options, + &mut Vec::new(), + ) + .await + }); + assert!( + matches!(error, Err(nostream_cli::CliError::Migrate(_))), + "the stub fails the second read_batch" + ); + let manifest: Value = serde_json::from_slice(&manifest_bytes(partial.path())) + .expect("the partial run's manifest parses"); + let dvm = &manifest["tables"]["dvm_jobs"]; + assert_eq!(dvm["rows"], 2, "the first batch is recorded"); + assert!( + !dvm["after"].as_str().expect("a hex cursor").is_empty(), + "and so is its cursor" + ); + let store = partial.path().join("lmdb"); + std::fs::remove_dir_all(&store).expect("the store is removed"); + assert!(!store.exists()); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(partial.path()) + .args(["--batch-size", "2"]) + .env_clear() + .output() + .expect("the binary runs"); + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("Migration failed:"), "got: {stderr}"); + assert!( + stderr.contains(&store.display().to_string()), + "the line names /lmdb: {stderr}" + ); + assert!(!store.exists(), "the refused run created nothing"); + assert!( + stderr.contains("a fresh --target"), + "the refusal names the remedy: {stderr}" + ); + + // Half two: the manifest R6 writes, with no progress in it, over an empty directory. + let fresh = tempfile::tempdir().expect("a temporary directory"); + write_manifest_fixture(fresh.path(), false); + let fresh_store = fresh.path().join("lmdb"); + + // `run` takes the copy path rather than the guard: a source it cannot parse fails it, not a + // store, and it creates nothing. The URL is unparsable so no dial is attempted. + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(fresh.path()) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + assert_eq!( + output.status.code(), + Some(1), + "there is no database to read" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("No store"), + "a zero-progress manifest is not refused: {stderr}" + ); + assert!( + !stderr.contains(&fresh_store.display().to_string()), + "the failure is not about the store: {stderr}" + ); + assert!(!fresh_store.exists()); + + // And the copy that run would have driven completes over the same target. + let copy = options(fresh.path(), 2); + let (counts, complete) = runtime.block_on(async { + let target = LmdbBackend::open(fresh.path()) + .await + .expect("the store opens"); + let output = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + ©, + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + assert_eq!(output.exit_code, 0); + let mut counts = Vec::new(); + for table in MigratedTable::ALL { + counts.push(scan_all(&target, table).await.len()); + } + let manifest: Value = serde_json::from_slice(&manifest_bytes(fresh.path())) + .expect("the finished manifest parses"); + let complete = manifest["tables"] + .as_object() + .expect("the tables object") + .values() + .all(|entry| entry["complete"] == true); + (counts, complete) + }); + for count in counts { + assert_eq!(count, 3, "nine tables at three rows"); + } + assert!(complete, "every table is complete"); +} + +/// The process boundary the two failure tests above stop at: `main` prints +/// `Migration failed: {message}` and exits 1. No source is reachable here, so the run fails on +/// the way to the copy, which is the same arm. +#[test] +fn a_failed_run_reports_migration_failed_and_exits_1() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env("DB_URI", "postgres://127.0.0.1:1/nostream") + // Nothing listens on port 1, so the failure is immediate; without this the pool still + // retries to the 60 s acquire timeout, and this one test would be a minute of every run. + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.starts_with("Migration failed: "), + "the failure is reported as one: {stderr}" + ); +} + +// --------------------------------------------------------------------------------------- +// A process that never opened the store verifies it. +// --------------------------------------------------------------------------------------- + +// Every other verify test runs in the process that opened the store writable, where heed's +// process-global registry already holds valid database handles, so a read-only opener that +// never commits its read transaction looks fine. The binary is the only way to get a process +// that has never opened the store, which is the shape a real `--verify` runs in. +#[test] +fn the_binary_verifies_a_store_this_process_wrote() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("the test runtime builds"); + + let exit_code = runtime.block_on(async { + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let output = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + output.exit_code + }); + assert_eq!(exit_code, 0, "the copy completes"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .arg("--verify") + .env("DB_URI", "not-a-url") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(0), + "a fresh process verifies what the store holds: stdout {stdout} stderr {stderr}" + ); + assert!( + stdout.starts_with("Verified 9 tables, "), + "the verify reports its work: {stdout}" + ); +} + +// --------------------------------------------------------------------------------------- +// The checksum is the number the document says it is. +// --------------------------------------------------------------------------------------- + +// D4 states the row sum as the sum of `SHA-256(len32 ‖ key ‖ canonical)` per row, modulo 2^256, +// and the value exists so an independent checker can be written from that sentence. This +// literal was computed outside this crate from a single `events` row's bytes on exactly that +// framing, so a missing length prefix or a different order fails here instead of passing every +// criterion: the unit tests pin `sum_words`, not `add`. +#[tokio::test] +async fn the_row_checksum_matches_an_independent_computation() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let source = StubSource::new(vec![MigrationRow::Event(event_record(1, "golden"))]); + migrate::migrate_with( + &source, + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + + let written = manifest(directory.path()); + let events = &written["tables"][MigratedTable::Events.name()]; + assert_eq!(events["rows"], 1, "one row was copied"); + assert_eq!( + events["checksum"], "d81f052af7bfeb879aca5d630db8730d7168d6ca5cb2f5fc59f1a1c749150fbe", + "the checksum an independent checker computes for this row" + ); +} + +// --------------------------------------------------------------------------------------- +// A manifest that counts rows must find them in the store beside it. +// --------------------------------------------------------------------------------------- + +// Asking about the store by its file tells a missing store from a present one, not a store the +// run wrote from one that lost its tables: a `data.mdb` can be there with none of the nine +// tables in it, and the copy would then add the remaining rows to a table that never held the +// ones the manifest counts, reporting success over a store missing them. +#[tokio::test] +async fn a_manifest_that_counts_rows_over_a_store_without_them_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + let mut tables = serde_json::Map::new(); + for table in MigratedTable::ALL { + let entry = if table.name() == MigratedTable::Events.name() { + json!({"rows": 5, "checksum": "ab".repeat(32), "after": "", "complete": false}) + } else { + json!({"rows": 0, "checksum": "0".repeat(64), "after": "", "complete": false}) + }; + tables.insert(table.name().to_owned(), entry); + } + let fixture = + json!({"format": 1, "not_migrated": MigratedTable::NOT_MIGRATED, "tables": tables}); + std::fs::write( + directory.path().join("migration-manifest.json"), + serde_json::to_vec(&fixture).expect("the fixture serialises"), + ) + .expect("the fixture manifest is written"); + + let error = migrate::migrate_with( + &StubSource::new(source_rows()), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect_err("a store that never held the counted rows is refused"); + let message = error.to_string(); + assert!(message.contains("counts 5 rows"), "{message}"); + assert!(message.contains(MigratedTable::Events.name()), "{message}"); +} + +// --------------------------------------------------------------------------------------- +// The target and the manifest are not readable by anyone else. +// --------------------------------------------------------------------------------------- + +// The manifest records the last key written per table, and for `invite_codes` that key is the +// invite code itself - a bearer credential NIP-43 accepts. The store's files are heed's 0600, so +// the manifest was the world-readable copy of it. The planted link is the second half: a +// temporary path that already exists must not be written through and renamed into place. +#[tokio::test] +async fn the_manifest_is_private_and_its_temporary_path_is_not_followed() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let victim = directory.path().join("victim"); + std::fs::write(&victim, b"not the manifest").expect("the victim file is written"); + let temporary = directory.path().join("migration-manifest.json.tmp"); + std::os::unix::fs::symlink(&victim, &temporary).expect("the link is planted"); + + migrate::migrate_with( + &StubSource::new(vec![MigrationRow::Event(event_record(1, "golden"))]), + &target, + &options(directory.path(), 1_000), + &mut Vec::new(), + ) + .await + .expect("the copy succeeds"); + + assert_eq!( + std::fs::read(&victim).expect("the victim still reads"), + b"not the manifest", + "the planted link's target is untouched" + ); + let manifest = std::fs::symlink_metadata(directory.path().join("migration-manifest.json")) + .expect("the manifest exists"); + assert!( + !manifest.file_type().is_symlink(), + "the manifest is a file, not the link" + ); + assert_eq!( + manifest.permissions().mode() & 0o777, + 0o600, + "the manifest is readable by its owner alone" + ); +} + +// A group- or world-writable target lets another local user pre-place the store and read what +// lands in it, so the copy refuses before opening anything. +#[test] +fn a_group_writable_target_is_refused() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().expect("a temporary directory"); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o777)) + .expect("the target is made world-writable"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("writable by group or other"), + "the refusal says why: {stderr}" + ); +} + +// A symlinked target sends the copy somewhere the operator did not name. +#[test] +fn a_symlinked_target_is_refused() { + let root = tempfile::tempdir().expect("a temporary directory"); + let real = root.path().join("real"); + std::fs::create_dir(&real).expect("the real directory is made"); + let link = root.path().join("link"); + std::os::unix::fs::symlink(&real, &link).expect("the link is planted"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(&link) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("is a symlink"), "{stderr}"); +} + +// --------------------------------------------------------------------------------------- +// AC19: the source is named, and an unconfigured one fails at once. +// --------------------------------------------------------------------------------------- + +// `from_env` falls back to `postgres@localhost:5432/postgres` when nothing is set, so a run with +// no `DB_URI` dials whatever local server is listening and copies it, and when nothing is +// listening it waits out the pool's acquire timeout. +#[test] +fn an_unconfigured_source_is_refused_at_once() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let started = std::time::Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("DB_URI"), "the refusal names it: {stderr}"); + assert!(stderr.contains("DB_HOST"), "and the parts: {stderr}"); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the refusal does not wait out the pool: {:?}", + started.elapsed() + ); +} + +// An unreachable host is reported as the database it could not read, with the password dropped. +#[test] +fn an_unreachable_source_is_named_without_its_password() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://alice:secret@127.0.0.1:1/nostream") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("127.0.0.1:1/nostream"), + "the failure names the host and database it resolved to: {stderr}" + ); + assert!(stderr.contains("DB_URI"), "and the setting: {stderr}"); + assert!( + !stderr.contains("secret"), + "the password is not printed: {stderr}" + ); + assert!( + !stderr.contains("alice"), + "the role is not printed either, only the host and database: {stderr}" + ); +} + +// A URI can carry the password in a query parameter instead of the userinfo, and the description +// is printed into an error that reaches stderr and any captured log. +#[test] +fn a_query_string_password_is_not_printed() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env( + "DB_URI", + "postgres://127.0.0.1:1/nostream?password=secret&sslmode=require", + ) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("127.0.0.1:1/nostream"), + "the failure still names the source: {stderr}" + ); + assert!( + !stderr.contains("secret"), + "the query-string password is not printed: {stderr}" + ); +} + +// The probe reports the driver's own reason at once. This test proves the *message* and that the +// fast path is taken at the shipped timeout; it cannot see the cap, because a refused port +// answers in milliseconds whatever the bound is - the cap is what turns a black-holed host into +// ten seconds, and `#1122` carries that measurement. Inverting the cap leaves this test green. +#[test] +fn an_unreachable_source_fails_in_seconds_at_the_default_timeout() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let started = std::time::Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://127.0.0.1:1/nostream") + // The value the shipped `.env.example` carries: the probe must cap it, not inherit it. + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "60000") + .output() + .expect("the binary runs"); + + let elapsed = started.elapsed(); + assert_eq!(output.status.code(), Some(1)); + assert!( + elapsed < std::time::Duration::from_secs(25), + "a refused source is reported in seconds, not at the pool's minute: {elapsed:?}" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("could not read the source at postgres://127.0.0.1:1/nostream"), + "the failure names the source: {stderr}" + ); + assert!( + !stderr.contains("pool timed out"), + "the driver's reason is what prints, not the pool's acquire message: {stderr}" + ); +} + +// A libpq keyword string has no userinfo to strip, so an echo of what was set would print the +// password. The refusal names the shape and nothing else. +#[test] +fn a_keyword_value_source_uri_is_refused_without_echoing_it() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "host=127.0.0.1 port=1 user=alice password=s3cret") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "the refusal names the shape: {stderr}" + ); + assert!( + !stderr.contains("s3cret"), + "nothing of the value is echoed: {stderr}" + ); +} + +// The check is stricter than sqlx in one direction only: `postgres:////` parses in sqlx to +// `host=localhost`, and a cutover must not guess which server to copy from. +#[test] +fn a_uri_that_would_default_to_localhost_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + for uri in ["postgres:/", "postgres://", "postgres:////"] { + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", uri) + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1), "{uri}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "{uri} is refused rather than sent to localhost: {stderr}" + ); + } +} + +// A `?` after a `#` is inside the fragment, which sqlx ignores: `postgres://#?host=/sock` has no +// host and no query, so the check must refuse it rather than accept it on the strength of a +// parameter the parser never reads - otherwise it dials localhost without saying so. +#[test] +fn a_query_inside_a_fragment_is_not_a_host() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://#?host=/var/run/postgresql&dbname=x") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "a fragment is not a query: {stderr}" + ); +} + +// A libpq keyword string can contain `://` by accident, and sqlx reads `password=ab` as the +// scheme and `cd` as the host. Only the two Postgres schemes are a URL to this check. +#[test] +fn a_non_postgres_scheme_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + for uri in [ + "password=ab://cd", + "mysql://127.0.0.1:1/d", + "http://127.0.0.1:1/d", + ] { + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", uri) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1), "{uri}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "{uri} is not a Postgres URL: {stderr}" + ); + } +} + +// `user@:5432` carries no host, only a port: sqlx reports `empty host` and dials nothing, so the +// gate refuses it rather than counting the port as an authority. +#[test] +fn a_port_without_a_host_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://user@:5432/db") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "a port is not a host: {stderr}" + ); +} + +// An IPv6 literal is bracketed, and the brackets are what separate it from the port. +#[test] +fn an_ipv6_literal_is_dialled() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres://[::1]:1/d") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "an IPv6 literal is a host: {stderr}" + ); +} + +// The other spelling sqlx accepts is `postgresql://`, and it must keep working. +#[test] +fn a_postgresql_url_is_dialled() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgresql://127.0.0.1:1/d") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "postgresql:// is a Postgres URL: {stderr}" + ); + assert!( + stderr.contains("Connection refused"), + "the host in it is dialled: {stderr}" + ); +} + +// The refusal judges the shape only. sqlx dials a URL whose authority is empty when the query +// names the host, and both of these reached the server before the refusal existed: `?host=` opens +// a Unix socket, `?hostaddr=` dials the address given. A rule that read only the authority refused +// them, so a relay whose Postgres listens on a socket could no longer be migrated at all. +#[test] +fn a_socket_source_uri_is_not_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + // A socket that does not exist here: the point is the shape, not the connection. + .env( + "DB_URI", + "postgres:///?host=/var/run/postgresql&dbname=nostream", + ) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "a socket source is dialled, not refused: {stderr}" + ); +} + +#[test] +fn a_hostaddr_source_uri_is_dialled() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + // Port 1 is refused: the failure has to come from the dial, not from the shape check. + .env( + "DB_URI", + "postgres:///?hostaddr=127.0.0.1&port=1&dbname=nostream", + ) + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("is not a URL with a host"), + "a hostaddr source is dialled, not refused: {stderr}" + ); + assert!( + stderr.contains("Connection refused"), + "the address in the query is the one dialled: {stderr}" + ); +} + +// AC20: the remedy is in the command's own help, not only in the behaviour. +#[test] +fn the_help_names_the_resume_and_the_restart() { + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--help"]) + .output() + .expect("the binary runs"); + + assert!(output.status.success(), "{}", output.status); + let help = String::from_utf8_lossy(&output.stdout); + assert!( + help.contains("resumes"), + "the help says a re-run resumes: {help}" + ); + assert!( + help.contains("fresh `--target`"), + "and what starting over actually takes: {help}" + ); + assert!( + help.contains("relay must be stopped"), + "and that the relay must be stopped: {help}" + ); +} + +// AC7: the summary names the store the copy landed in, so a path that went through a link is +// visible as the real one rather than echoed back as the argument. +#[tokio::test] +async fn the_summary_names_the_canonical_store_not_the_argument() { + let root = tempfile::tempdir().expect("a temporary directory"); + let real = root.path().join("real"); + std::fs::create_dir(&real).expect("the real directory is made"); + let link = root.path().join("link"); + std::os::unix::fs::symlink(&real, &link).expect("the link is planted"); + let argument = link.join("store"); + std::fs::create_dir(&argument).expect("the target is made through the link"); + + let target = LmdbBackend::open(&argument).await.expect("the store opens"); + let mut out = Vec::new(); + migrate::migrate_with( + &StubSource::new(vec![MigrationRow::Event(event_record(1, "golden"))]), + &target, + &options(&argument, 1_000), + &mut out, + ) + .await + .expect("the copy succeeds"); + + let (_, _, _, store) = summary(&output_text(out)); + assert_ne!( + store, + argument.join("lmdb").display().to_string(), + "the argument went through a link, so it is not what is reported" + ); + assert_eq!( + store, + std::fs::canonicalize(real.join("store").join("lmdb")) + .expect("the store exists") + .display() + .to_string(), + "the summary names the real store" + ); +} + +// U4: a `--target` that is a file is named as one, rather than reported as the operating +// system's `Not a directory (os error 20)` from a path the operator never wrote. +#[test] +fn a_target_that_is_a_file_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let file = directory.path().join("not-a-dir"); + std::fs::write(&file, b"x").expect("the file is written"); + + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(&file) + .env_clear() + .env("DB_URI", "not-a-url") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("is not a directory"), "{stderr}"); +} + +// sqlx parses a schemeless string as a URL with no base: the opaque path becomes the database +// name and the host falls back to localhost, so the copy would dial the wrong server and a +// failure would echo the credentials back as a database name. The shape is refused before +// anything is dialled - which is stronger than redacting it. +#[test] +fn a_schemeless_source_uri_is_refused() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let output = Command::new(env!("CARGO_BIN_EXE_nostream-cli")) + .args(["migrate", "pg-to-lmdb", "--target"]) + .arg(directory.path()) + .env_clear() + .env("DB_URI", "postgres:alice:secret@127.0.0.1:1/nostream") + .env("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000") + .output() + .expect("the binary runs"); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("is not a URL with a host"), + "the refusal names the shape: {stderr}" + ); + assert!( + !stderr.contains("secret"), + "the password is not printed: {stderr}" + ); +} diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 980ec522..07a89460 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -1,12 +1,14 @@ //! Backend selection and the LMDB environment. Mirrors `src/utils/event-store.ts` (the //! selection half) and the storage boundary in `src/@types/adapters.ts`. +#[cfg(feature = "lmdb")] +use std::os::unix::fs::DirBuilderExt; use std::path::{Path, PathBuf}; use nostream_core::EventStoreBackend; #[cfg(feature = "lmdb")] -use heed::{Env, EnvOpenOptions}; +use heed::{Env, EnvFlags, EnvOpenOptions}; #[cfg(feature = "lmdb")] use crate::error::StorageError; @@ -18,11 +20,33 @@ use crate::lmdb::{LMDB_MAX_DBS, LMDB_MAX_READERS}; /// database it replaces. pub const LMDB_DIR: &str = "lmdb"; +/// The file heed keeps an environment's data in, under [`LMDB_DIR`]. The store never changes +/// it: another name would be a different environment to every existing reader of the path. +pub const LMDB_DATA_FILE: &str = "data.mdb"; + +/// The store directory inside `root`: `/lmdb`. +/// +/// One home for the shape, so the openers, the CLI's target check and its messages cannot +/// disagree about where the store is. +pub fn store_dir(root: &Path) -> PathBuf { + root.join(LMDB_DIR) +} + +/// The `data.mdb` of the store in `root`, whether or not it is there. +pub fn store_file(root: &Path) -> PathBuf { + store_dir(root).join(LMDB_DATA_FILE) +} + +/// Whether `root` holds a store this crate opened: `data.mdb` is a file. +pub fn store_exists(root: &Path) -> bool { + store_file(root).is_file() +} + /// Resolves the data directory under `root` for one backend. A backend that keeps its data /// somewhere other than a directory of its own answers `None`. pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { match backend { - EventStoreBackend::Lmdb => Some(root.join(LMDB_DIR)), + EventStoreBackend::Lmdb => Some(store_dir(root)), EventStoreBackend::Postgres => None, } } @@ -48,9 +72,11 @@ pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { /// location before this environment is wired into settings. /// - **`NO_LOCK` requires self-managed concurrency; a broken lock file means /// synchronization issues and possible UB** — split responsibility. The code guarantees -/// its half: no env flags are passed (`EnvOpenOptions::new()` only), so `NO_LOCK` is -/// never in play and this crate never reads, writes or removes the lock file. The -/// operator must supply the rest: the data directory's lock file must not be broken. +/// its half: the only flag it passes is `READ_ONLY`, for the verify opener, so `NO_LOCK` +/// is never in play and LMDB's own locking runs. The lock file belongs to LMDB: this crate +/// never reads, writes or removes it, and a read-only open recreates a missing `lock.mdb` +/// beside `data.mdb`. The operator must supply the rest: the data directory's lock file +/// must not be broken. /// - **One `map_size` per path for the life of the process** — upheld by heed + propagated: /// heed's per-process `OPENED_ENV` registry returns the same `Env` for a second open with /// equal options and `Error::BadOpenOptions` when they differ, and `open` surfaces that @@ -69,14 +95,46 @@ impl LmdbEnv { /// for the backend's named databases and its reader slots. pub fn open(path: impl Into, map_size: usize) -> Result { let path = path.into(); - std::fs::create_dir_all(&path).map_err(|source| StorageError::CreateDirectory { - path: path.clone(), - source, - })?; + // 0700, not the umask's answer: the directory holds a relay's whole dataset, and on a + // host whose umask is 002 `create_dir_all` leaves it group-writable, which is both a + // wider audience than the store's own 0600 files and something the migration's target + // check refuses on the next run. + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&path) + .map_err(|source| StorageError::CreateDirectory { + path: path.clone(), + source, + })?; + let env = Self::open_with(&path, map_size, EnvFlags::empty())?; + Ok(Self { env, path }) + } + /// Opens an existing LMDB environment at `path` read-only, and creates nothing. + /// + /// The directory is not made and the file is not created: a path with no environment is + /// heed's error rather than a new file. `READ_ONLY` takes no write lock, and this + /// constructor opens no transaction of its own. + /// + /// heed answers `Error::BadOpenOptions` for a second open of one path with different + /// options, and its registry holds the environment process-wide, so dropping a handle does + /// not free the path: a caller that already opened this path writable cannot then open it + /// read-only, or the other way round, and a caller that needs the writable one afterwards + /// must not open it here first. + pub fn open_existing(path: impl Into, map_size: usize) -> Result { + let path = path.into(); + let env = Self::open_with(&path, map_size, EnvFlags::READ_ONLY)?; + Ok(Self { env, path }) + } + + /// The one place this crate opens an environment, so the `unsafe` precondition list above + /// is answered once and `open` and `open_existing` cannot drift apart. + fn open_with(path: &Path, map_size: usize, flags: EnvFlags) -> Result { // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on: - // no env flags (LMDB's locking runs), transactions never escape the closure they are - // opened in, and heed's per-process dedup turns a second open into the same Env or + // no env flags beyond `READ_ONLY` when the caller passes it (LMDB's locking runs, and + // the write half of the list is vacuous), transactions never escape the closure they + // are opened in, and heed's per-process dedup turns a second open into the same Env or // BadOpenOptions. The network-filesystem precondition rests on the operator passing a // local path; see the list above. #[allow(unsafe_code)] @@ -85,13 +143,14 @@ impl LmdbEnv { .map_size(map_size) .max_dbs(LMDB_MAX_DBS) .max_readers(LMDB_MAX_READERS) - .open(&path) + .flags(flags) + .open(path) } .map_err(|source| StorageError::OpenLmdb { - path: path.clone(), + path: path.to_path_buf(), source, })?; - Ok(Self { env, path }) + Ok(env) } /// The environment handle, for the repository implementations. diff --git a/crates/nostream-storage/src/error.rs b/crates/nostream-storage/src/error.rs index e0f1edbc..020683a8 100644 --- a/crates/nostream-storage/src/error.rs +++ b/crates/nostream-storage/src/error.rs @@ -53,6 +53,13 @@ pub enum StorageError { #[error("lmdb operation failed: {0}")] Lmdb(#[from] heed::Error), + /// A named database a caller required is absent from the environment. The migration + /// tool's verify opens a store it did not create, so a missing database is an error rather + /// than a new one. + #[cfg(feature = "lmdb")] + #[error("the store has no {name} database")] + MissingDatabase { name: &'static str }, + /// An LMDB environment could not be opened at the requested path. #[cfg(feature = "lmdb")] #[error("failed to open the lmdb environment at {path}: {source}")] diff --git a/crates/nostream-storage/src/keys.rs b/crates/nostream-storage/src/keys.rs new file mode 100644 index 00000000..75805319 --- /dev/null +++ b/crates/nostream-storage/src/keys.rs @@ -0,0 +1,33 @@ +//! The key codecs both backends and the migration seam share. +//! +//! A key's byte order *is* the sort order LMDB and PostgreSQL both rely on, so an arithmetic +//! slip here is not a compile error and not a visible wrong value: it is a cursor that seeks to +//! the wrong place, or a checksum that disagrees after a full copy. These lived in +//! `lmdb::encoding` and were restated in `migration` and `postgres::migration`, each with a +//! comment saying it had to compile without the other driver. One feature-free home removes the +//! restatement and the drift it invited; `lmdb::encoding` re-exports these so its callers are +//! unchanged. + +use chrono::{DateTime, Utc}; + +/// An `i32` ordered by its bytes, with the sign bit flipped so a negative value sorts before +/// zero rather than after every positive one. +pub(crate) fn i32_key(value: i32) -> [u8; 4] { + ((value as u32) ^ (1 << 31)).to_be_bytes() +} + +/// The inverse of [`i32_key`], and `None` when the bytes are not four long. +/// +/// Gated on the backends that read an `i32` key: with neither feature on, nothing does. The caller names the +/// table in its own error, so the message an operator sees stays the one that backend wrote. +#[cfg(any(feature = "lmdb", feature = "postgres"))] +pub(crate) fn i32_from_key(bytes: &[u8]) -> Option { + let raw = u32::from_be_bytes(bytes.try_into().ok()?); + Some((raw ^ (1 << 31)) as i32) +} + +/// The instant a row carries, truncated to the microsecond precision every PostgreSQL +/// `timestamp` column has, so a value written by either backend reads back the same. +pub(crate) fn micros(value: DateTime) -> DateTime { + DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) +} diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index ddee1d90..d87c3ca8 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -16,20 +16,28 @@ //! - `src/factories/event-repository-factory.ts` //! //! What it deliberately does **not** own: the relay handlers that call these traits, the -//! payments service, the daemon's backend factory and the `nostream-cli migrate pg-to-lmdb` -//! tool. The cache is not a cache server: no Redis client is added and -//! `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not ported — the decision and -//! what would overturn it are ADR `0010-services-worker-model-and-cache.md`. Both backends are -//! complete: Postgres against the 37-migration schema, and LMDB against the same column -//! contract for 9 of the capture's 10 repositories — `user_subscriptions` has a TypeScript -//! repository and no trait yet, and `events.first_seen` is stored but never indexed -//! (`docs/rewrite/decisions/0011-lmdb-backend.md`). +//! payments service, the daemon's backend factory and the tool behind +//! `nostream-cli migrate pg-to-lmdb`. It owns that tool's seam — [`migration`], the paged source +//! read and the exact-per-row target write — which exists for that one caller and for no other; +//! the copy, its manifest and its verification live in the CLI. The cache is not a cache server: +//! no Redis client is added and `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not +//! ported — the decision and what would overturn it are ADR +//! `0010-services-worker-model-and-cache.md`. Both backends are complete: Postgres against the +//! 37-migration schema, and LMDB against the same column contract for 9 of the capture's 10 +//! repositories — `user_subscriptions` has a TypeScript repository and no trait yet, and is the +//! one of those ten the migration does not move (`event_tags` is not moved either: it is the +//! trigger-maintained index whose LMDB `tags` database the event write path keeps), and +//! `events.first_seen` is stored but never indexed +//! (`docs/rewrite/decisions/0011-lmdb-backend.md`, +//! `docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`). pub mod backend; pub mod error; pub mod event_store; +mod keys; #[cfg(feature = "lmdb")] pub mod lmdb; +pub mod migration; #[cfg(feature = "postgres")] pub mod postgres; pub mod records; diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index d60eafe0..16ccfd17 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -53,11 +53,7 @@ pub(crate) fn invalid(table: &'static str, message: String) -> StorageError { } } -/// The instant a row carries, truncated to the microsecond precision every PostgreSQL -/// `timestamp` column has, so a value written by either backend reads back the same. -pub(crate) fn micros(value: DateTime) -> DateTime { - DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) -} +pub(crate) use crate::keys::{i32_key, micros}; /// An `i64` ordered by its bytes: the sign bit is flipped so a negative value sorts before /// zero rather than after every positive one. @@ -71,15 +67,11 @@ pub(crate) fn i64_from_key(bytes: &[u8]) -> Result { Ok((raw ^ (1 << 63)) as i64) } -/// An `i32` ordered by its bytes, sign-flipped for the same reason as [`i64_key`]. -pub(crate) fn i32_key(value: i32) -> [u8; 4] { - ((value as u32) ^ (1 << 31)).to_be_bytes() -} - -/// The inverse of [`i32_key`]. +/// The inverse of [`i32_key`]. The arithmetic lives in `crate::keys` with the forward one; the +/// error is this module's, so the message an operator sees is unchanged. pub(crate) fn i32_from_key(bytes: &[u8]) -> Result { - let raw = u32::from_be_bytes(fixed::<4>("events", bytes)?); - Ok((raw ^ (1 << 31)) as i32) + crate::keys::i32_from_key(bytes) + .ok_or_else(|| invalid("events", format!("4 bytes expected, {} found", bytes.len()))) } fn fixed(table: &'static str, bytes: &[u8]) -> Result<[u8; N], StorageError> { diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index c1281b4f..b262a842 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -247,6 +247,39 @@ fn write_event( Ok(()) } +/// Writes one migrated event: every entry [`write_event`] writes, the `replaceable` pointer +/// [`write_upserted`] writes, and the `events_deleted` entry [`soft_delete`] writes when the row +/// carries `deleted_at`. +/// +/// Nothing is stamped, cleared or defaulted: a migrated row keeps the `first_seen`, +/// `deleted_at`, `expires_at`, `event_deduplication` and `remote_address` the source carries. A +/// row that arrives soft-deleted or expired never passed through [`soft_delete`], so both tail +/// entries are this writer's own work. +pub(super) fn write_migrated( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + write_event(txn, databases, record)?; + let kind = kind_of(record)?; + if encoding::replaceable_kind(kind) { + let deduplication = record.deduplication(); + databases.replaceable.put( + txn, + &encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?, + record.event.id.as_bytes(), + )?; + } + if let Some(deleted_at) = record.deleted_at { + databases.events_deleted.put( + txn, + &encoding::time_key(deleted_at.timestamp(), &record.event.id), + &[], + )?; + } + Ok(()) +} + /// Writes only the `events` value, which is what a soft delete changes. fn put_value( txn: &mut RwTxn<'_>, diff --git a/crates/nostream-storage/src/lmdb/invoices.rs b/crates/nostream-storage/src/lmdb/invoices.rs index 4bfd667f..287e7893 100644 --- a/crates/nostream-storage/src/lmdb/invoices.rs +++ b/crates/nostream-storage/src/lmdb/invoices.rs @@ -33,7 +33,10 @@ fn pending_key(created_at: DateTime, id: &str) -> Vec { } /// Writes a row and moves its pending entry, when the status or the row changed. -fn write( +/// +/// The migration's writer too ([`super::migration`]), with no previous row: it puts the source's +/// own `amount_paid` and `confirmed_at` and the pending entry the row's status asks for. +pub(super) fn write( txn: &mut RwTxn<'_>, databases: &Databases, invoice: &Invoice, diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs index bfcd2807..f8b4ef9c 100644 --- a/crates/nostream-storage/src/lmdb/metadata.rs +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -83,6 +83,25 @@ pub(crate) fn nip05_upsert( Ok(1) } +/// Writes one verification row and its pending-check entry as the source carries them. +/// +/// [`nip05_upsert`] keeps a stored `created_at` and stamps `updated_at`; a migrated row arrives +/// with both of its own, so this puts the row it is given and its own index entry. +pub(super) fn nip05_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + verification: &Nip05Verification, +) -> Result<(), StorageError> { + let value = rows::encode("nip05_verifications", verification)?; + databases + .nip05 + .put(txn, &nip05_key(&verification.pubkey), &value)?; + databases + .nip05_by_last_checked_at + .put(txn, &nip05_pending_key(verification), &[])?; + Ok(()) +} + /// `Nip05VerificationRepository::find_pending_verifications`: rows last checked before the /// cutoff with fewer than `max_failures`, oldest check first. pub(crate) fn nip05_find_pending( @@ -162,7 +181,7 @@ pub(crate) fn invite_find_by_code( rows::decode("invite_codes", value).map(Some) } -fn invite_write( +pub(super) fn invite_write( txn: &mut RwTxn<'_>, databases: &Databases, invite: &InviteCode, @@ -346,7 +365,7 @@ pub(crate) fn dvm_find_by_id( rows::decode("dvm_jobs", value).map(Some) } -fn dvm_write( +pub(super) fn dvm_write( txn: &mut RwTxn<'_>, databases: &Databases, job: &DvmJob, @@ -549,7 +568,28 @@ fn next_report_id(txn: &mut RwTxn<'_>, databases: &Databases) -> Result, + databases: &Databases, + id: i32, +) -> Result<(), StorageError> { + let wanted = u64::try_from(id).unwrap_or(0); + let current: u64 = match databases.meta.get(txn, REPORT_SEQUENCE)? { + Some(value) => rows::decode("meta", value)?, + None => 0, + }; + if wanted > current { + let value = rows::encode("meta", &wanted)?; + databases.meta.put(txn, REPORT_SEQUENCE, &value)?; + } + Ok(()) +} + +pub(super) fn report_write( txn: &mut RwTxn<'_>, databases: &Databases, report: &Report, diff --git a/crates/nostream-storage/src/lmdb/migration.rs b/crates/nostream-storage/src/lmdb/migration.rs new file mode 100644 index 00000000..ab7c5957 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/migration.rs @@ -0,0 +1,179 @@ +//! `MigrationTarget` against LMDB: one write transaction per batch, one migration-only writer +//! per table. +//! +//! Every writer puts the source's own key and value and the index entries that table's existing +//! write path writes, and stamps, clears and defaults nothing. That is what makes a replayed +//! batch idempotent: the same row written twice is a put of the same key with the same value. +//! +//! What each writer is handed is the row's normalised form ([`MigrationRow::normalised`]), so +//! every instant is at the microsecond precision the value codec stores, and an event's +//! `row_id` — the one column the store does not keep — is already `None`. + +use heed::types::Bytes; +use heed::{Database, RoTxn, RwTxn}; + +use super::rows::StoredUser; +use super::{Databases, LmdbBackend, encoding, events, invoices, metadata, notifications, rows}; +use crate::error::StorageError; +use crate::migration::{MigratedTable, MigratedUser, MigrationRow, MigrationTarget}; +use crate::repositories::StorageFuture; + +impl MigrationTarget for LmdbBackend { + fn write_batch<'a>(&'a self, rows: &'a [MigrationRow]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let Some(first) = rows.first() else { + return Ok(0); + }; + let table = first.table(); + if rows.iter().any(|row| row.table() != table) { + return Err(StorageError::InvalidArgument { + message: "a migration batch must hold one table's rows", + }); + } + + // The blocking closure owns what it touches, so the batch moves into it: one + // transaction for the whole page, which either lands or does not. + let batch = rows.to_vec(); + self.write(move |txn, databases| write_batch(txn, databases, &batch)) + .await?; + Ok(rows.len() as u64) + }) + } + + fn scan_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let after = after.map(<[u8]>::to_vec); + self.read(move |txn, databases| scan(txn, databases, table, after.as_deref(), limit)) + .await + }) + } +} + +/// Writes one batch, and raises `meta`'s `reports.next_id` past the batch's largest report so a +/// later insert cannot collide with a migrated row. +fn write_batch( + txn: &mut RwTxn<'_>, + databases: &Databases, + batch: &[MigrationRow], +) -> Result<(), StorageError> { + let mut largest_report = 0; + for row in batch { + match row.normalised() { + MigrationRow::DvmJob(job) => metadata::dvm_write(txn, databases, &job, None)?, + MigrationRow::Event(record) => events::write_migrated(txn, databases, &record)?, + MigrationRow::InviteCode(invite) => metadata::invite_write(txn, databases, &invite)?, + MigrationRow::Invoice(invoice) => invoices::write(txn, databases, &invoice, None)?, + MigrationRow::Nip05Verification(verification) => { + metadata::nip05_write(txn, databases, &verification)?; + } + MigrationRow::DeliveryLogEntry(entry) => { + notifications::delivery_write(txn, databases, &entry)?; + } + MigrationRow::OutboxMessage(message) => { + notifications::outbox_write(txn, databases, &message, None)?; + } + MigrationRow::Report(report) => { + metadata::report_write(txn, databases, &report)?; + largest_report = largest_report.max(report.id); + } + MigrationRow::User(user) => { + users_store(txn, databases, &user)?; + } + } + } + metadata::raise_report_sequence(txn, databases, largest_report) +} + +/// Writes one `users` row: the whole [`StoredUser`], including the `tos_accepted_at` no trait +/// write carries. +fn users_store( + txn: &mut RwTxn<'_>, + databases: &Databases, + migrated: &MigratedUser, +) -> Result<(), StorageError> { + let user = &migrated.user; + super::users::store( + txn, + databases, + &StoredUser { + pubkey: user.pubkey, + is_admitted: user.is_admitted, + is_vanished: user.is_vanished, + balance: user.balance, + tos_accepted_at: migrated.tos_accepted_at, + created_at: user.created_at, + updated_at: user.updated_at, + }, + ) +} + +/// One page of a table, in the store's key order, resuming exclusively after `after`. +fn scan( + txn: &RoTxn<'_>, + databases: &Databases, + table: MigratedTable, + after: Option<&[u8]>, + limit: usize, +) -> Result, StorageError> { + let database = database(databases, table); + let bounds = encoding::resume_bounds(after, b"", b""); + let mut rows = Vec::new(); + for entry in database.range(txn, &bounds)? { + if rows.len() >= limit { + break; + } + let (_, value) = entry?; + rows.push(decode(table, value)?); + } + Ok(rows) +} + +/// The database one table's rows live in. +fn database(databases: &Databases, table: MigratedTable) -> Database { + match table { + MigratedTable::DvmJobs => databases.dvm_jobs, + MigratedTable::Events => databases.events, + MigratedTable::InviteCodes => databases.invite_codes, + MigratedTable::Invoices => databases.invoices, + MigratedTable::Nip05Verifications => databases.nip05, + MigratedTable::NotificationDeliveryLog => databases.notification_delivery_log, + MigratedTable::NotificationOutbox => databases.outbox, + MigratedTable::Reports => databases.reports, + MigratedTable::Users => databases.users, + } +} + +/// One stored value, as the row the seam carries. +fn decode(table: MigratedTable, value: &[u8]) -> Result { + match table { + MigratedTable::DvmJobs => rows::decode("dvm_jobs", value).map(MigrationRow::DvmJob), + MigratedTable::Events => rows::decode_event(value).map(MigrationRow::Event), + MigratedTable::InviteCodes => { + rows::decode("invite_codes", value).map(MigrationRow::InviteCode) + } + MigratedTable::Invoices => rows::decode("invoices", value).map(MigrationRow::Invoice), + MigratedTable::Nip05Verifications => { + rows::decode("nip05_verifications", value).map(MigrationRow::Nip05Verification) + } + MigratedTable::NotificationDeliveryLog => { + rows::decode("notification_delivery_log", value).map(MigrationRow::DeliveryLogEntry) + } + MigratedTable::NotificationOutbox => { + rows::decode("notification_outbox", value).map(MigrationRow::OutboxMessage) + } + MigratedTable::Reports => rows::decode("reports", value).map(MigrationRow::Report), + MigratedTable::Users => { + let stored: StoredUser = rows::decode("users", value)?; + let tos_accepted_at = stored.tos_accepted_at; + Ok(MigrationRow::User(MigratedUser { + user: stored.user(), + tos_accepted_at, + })) + } + } +} diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 61b89184..373863c3 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -11,6 +11,7 @@ mod encoding; mod events; mod invoices; mod metadata; +mod migration; mod notifications; mod rows; mod users; @@ -165,6 +166,58 @@ impl Databases { meta: env.create_database(txn, Some("meta"))?, }) } + /// Opens every database by name, refusing a store that lacks one. + /// + /// The migration tool's verify reads a store it did not create, so `create_database` is + /// wrong there: a missing database must be an error rather than a new one. + fn open(env: &Env, txn: &RoTxn<'_>) -> Result { + Ok(Self { + events: open_named(env, txn, "events")?, + events_by_created_at: open_named(env, txn, "events_by_created_at")?, + events_by_author: open_named(env, txn, "events_by_author")?, + events_deleted: open_named(env, txn, "events_deleted")?, + events_expiring: open_named(env, txn, "events_expiring")?, + replaceable: open_named(env, txn, "replaceable")?, + tags: open_named(env, txn, "tags")?, + users: open_named(env, txn, "users")?, + invoices: open_named(env, txn, "invoices")?, + invoices_pending: open_named(env, txn, "invoices_pending")?, + nip05: open_named(env, txn, "nip05_verifications")?, + nip05_by_last_checked_at: open_named(env, txn, "nip05_by_last_checked_at")?, + invite_codes: open_named(env, txn, "invite_codes")?, + invite_codes_by_created_at: open_named(env, txn, "invite_codes_by_created_at")?, + invite_codes_by_expires_at: open_named(env, txn, "invite_codes_by_expires_at")?, + dvm_jobs: open_named(env, txn, "dvm_jobs")?, + dvm_jobs_by_status_created_at: open_named(env, txn, "dvm_jobs_by_status_created_at")?, + reports: open_named(env, txn, "reports")?, + reports_by_event_id: open_named(env, txn, "reports_by_event_id")?, + reports_actionable_by_created_at: open_named( + env, + txn, + "reports_actionable_by_created_at", + )?, + reports_actionable_event: open_named(env, txn, "reports_actionable_event")?, + reports_actionable_pubkey: open_named(env, txn, "reports_actionable_pubkey")?, + outbox: open_named(env, txn, "notification_outbox")?, + outbox_by_status_created_at: open_named(env, txn, "outbox_by_status_created_at")?, + outbox_by_status_updated_at: open_named(env, txn, "outbox_by_status_updated_at")?, + notification_delivery_log: open_named(env, txn, "notification_delivery_log")?, + delivery_log_by_created_at: open_named(env, txn, "delivery_log_by_created_at")?, + delivery_log_by_outbox: open_named(env, txn, "delivery_log_by_outbox")?, + meta: open_named(env, txn, "meta")?, + }) + } +} + +/// One named database of an existing environment, or [`StorageError::MissingDatabase`] when the +/// store does not hold it. +fn open_named( + env: &Env, + txn: &RoTxn<'_>, + name: &'static str, +) -> Result, StorageError> { + env.open_database(txn, Some(name))? + .ok_or(StorageError::MissingDatabase { name }) } /// How much of the fixed map one store has used (`Env::non_free_pages_size`). @@ -192,8 +245,8 @@ impl LmdbBackend { /// write. pub async fn open(root: impl Into) -> Result { let root = root.into(); - // The same rule `data_dir` answers, from the same constant. - let path = root.join(crate::backend::LMDB_DIR); + // One home for the shape, shared with the writable opener and the CLI. + let path = crate::backend::store_dir(&root); let opened = tokio::task::spawn_blocking(move || { let environment = LmdbEnv::open(&path, LMDB_MAP_SIZE)?; let handle = environment.env().clone(); @@ -222,6 +275,50 @@ impl LmdbBackend { }) } + /// Opens the existing store under `/lmdb` for reading only, and creates nothing. + /// + /// The migration tool's verify path uses this instead of [`open`](Self::open): that one + /// `create_dir_all`s and `create_database`s, so verifying a store that is not there answers + /// with a fresh empty environment and every table short. Here a missing environment or a + /// missing database is an error. + /// + /// heed's process-global registry answers `BadOpenOptions` for a second open of one path + /// with different options, and it keeps the environment registered for the life of the + /// process, so dropping a handle does not free the path: a process that holds this root + /// writable cannot also open it read-only, and one that needs the writable root afterwards + /// must not open it here first. + pub async fn open_existing(root: impl Into) -> Result { + let root = root.into(); + // One home for the shape, shared with the writable opener and the CLI. + let path = crate::backend::store_dir(&root); + let opened = tokio::task::spawn_blocking(move || { + let environment = LmdbEnv::open_existing(&path, LMDB_MAP_SIZE)?; + let handle = environment.env().clone(); + let transaction = handle.read_txn()?; + let databases = Databases::open(&handle, &transaction)?; + // heed only publishes the database handles a transaction opened once that + // transaction commits; a dropped read transaction leaves them unusable, and a + // fresh process - which has no earlier committed open to inherit them from - + // answers `Invalid argument` on its first read. + transaction.commit()?; + Ok::<_, StorageError>((environment, databases)) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)??; + let (environment, databases) = opened; + + tracing::info!( + path = %environment.path().display(), + "the lmdb store is open for reading" + ); + + Ok(Self { + env: environment.env().clone(), + databases, + settings: None, + }) + } + /// How much of the fixed map the store has used, for an operator's tooling and the log /// line [`open`](Self::open) writes. /// @@ -991,6 +1088,72 @@ mod tests { } } + /// A report the migration carried across must not be allocated again: the batch raises the + /// `reports.next_id` sequence past the largest id it wrote, so the next `create` continues + /// from there instead of colliding with a migrated row. + #[tokio::test] + async fn a_migrated_report_raises_the_sequence_the_next_create_uses() { + use crate::migration::{MigrationRow, MigrationTarget}; + use crate::repositories::{NewReport, Report, ReportRepository, ReportType}; + + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let migrated = Report { + id: 7, + event_id: nostream_core::EventId::from_byte_array([21; 32]), + reporter_pubkey: nostream_core::Pubkey::from_byte_array([22; 32]), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + created_at: chrono::Utc::now(), + }; + MigrationTarget::write_batch(&backend, &[MigrationRow::Report(migrated)]) + .await + .expect("the batch writes"); + + let created = ReportRepository::create( + &backend, + &NewReport { + event_id: nostream_core::EventId::from_byte_array([23; 32]), + reporter_pubkey: nostream_core::Pubkey::from_byte_array([24; 32]), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }, + ) + .await + .expect("the report is created"); + + assert_eq!( + created.id, 8, + "the sequence continues past the migrated report" + ); + } + + /// The store directory holds the whole dataset, so it is the owner's alone: heed writes the + /// files 0600, and a directory the umask left group-writable would both widen the audience + /// and be refused by the migration's own target check on the next run. + #[tokio::test] + async fn the_store_directory_is_the_owners_alone() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path().join("store"); + LmdbBackend::open(&root).await.expect("the store opens"); + let mode = std::fs::metadata(crate::backend::store_dir(&root)) + .expect("the store directory exists") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o700, "group and other have no access"); + } + #[tokio::test] async fn the_store_reports_how_much_of_its_map_it_uses() { // The map never grows (LMDB_MAP_SIZE), so the approach to it is what an operator has to diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index 3c73734c..62264b63 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -152,6 +152,31 @@ pub(crate) fn append( Ok(()) } +/// Writes one delivery-log row exactly as the source carries it. +/// +/// [`append`] mints the id and the timestamp and checks the outbox foreign key; a migrated row +/// arrives with its own id, its own `created_at` and an outbox row that is in the source. The +/// value and the two index keys are the same set `append` writes. +pub(super) fn delivery_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + entry: &DeliveryLogEntry, +) -> Result<(), StorageError> { + let value = rows::encode("notification_delivery_log", entry)?; + databases + .notification_delivery_log + .put(txn, entry.id.as_bytes(), &value)?; + databases + .delivery_log_by_created_at + .put(txn, &delivery_created_key(entry), &[])?; + if let Some(outbox_id) = &entry.outbox_id { + databases + .delivery_log_by_outbox + .put(txn, &delivery_outbox_key(outbox_id, entry)?, &[])?; + } + Ok(()) +} + /// `find_recent`: newest first, optionally filtered by status and event type. pub(crate) fn find_recent( txn: &RoTxn<'_>, @@ -288,7 +313,7 @@ fn outbox_load( } /// Writes a message and its two status indexes, removing the entries of the row it replaces. -fn outbox_write( +pub(super) fn outbox_write( txn: &mut RwTxn<'_>, databases: &Databases, message: &OutboxMessage, diff --git a/crates/nostream-storage/src/migration.rs b/crates/nostream-storage/src/migration.rs new file mode 100644 index 00000000..483bd0e6 --- /dev/null +++ b/crates/nostream-storage/src/migration.rs @@ -0,0 +1,260 @@ +//! The Postgres-to-LMDB migration seam. +//! +//! This module exists for one caller — `nostream-cli migrate pg-to-lmdb` — and is **not** a +//! general read or write surface. Nothing in the relay, the daemon or the repository traits may +//! use it, and a second caller needs a decision record first +//! (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D1). +//! +//! It lives here rather than in the CLI because the pieces a copy needs are in this crate: the +//! Postgres row decoders (`crate::postgres::rows`), the event column list, and the LMDB +//! per-table write paths with their index entries. A CLI-side copy would restate nine column +//! lists and nine index write paths and drift from them. +//! +//! The seam is two traits and the rows they move. [`MigrationSource`] reads a table one keyset +//! page at a time with no filter of any kind; [`MigrationTarget`] writes a page exactly as the +//! source carries it and reads a page back for the verification pass. Neither goes through the +//! nine repository traits: no trait enumerates a table, and every trait write stamps, clears or +//! drops at least one column, so a trait-shaped copy would be both lossy and partial. +//! +//! Every type here compiles with no driver. The two implementations are `#[cfg(feature)]`, and +//! this vocabulary must stay buildable with neither. + +use chrono::{DateTime, Utc}; + +use crate::error::StorageError; +use crate::repositories::{ + DeliveryLogEntry, DvmJob, EventRecord, InviteCode, Invoice, Nip05Verification, OutboxMessage, + Report, StorageFuture, User, +}; + +/// One table the migration moves, in the order the copy walks them. +/// +/// The nine are the tables the store has a repository and a database for. Two of the capture's +/// ten repository tables are not here: `event_tags`, whose LMDB `tags` database the event write +/// path maintains itself, and `user_subscriptions`, which has no trait and no database +/// (`MigratedTable::NOT_MIGRATED` records both). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum MigratedTable { + DvmJobs, + Events, + InviteCodes, + Invoices, + Nip05Verifications, + NotificationDeliveryLog, + NotificationOutbox, + Reports, + Users, +} + +impl MigratedTable { + /// Every table, in the order the copy walks them. + pub const ALL: [Self; 9] = [ + Self::DvmJobs, + Self::Events, + Self::InviteCodes, + Self::Invoices, + Self::Nip05Verifications, + Self::NotificationDeliveryLog, + Self::NotificationOutbox, + Self::Reports, + Self::Users, + ]; + + /// The source tables this tool does not move, which the manifest prints as a stated limit. + pub const NOT_MIGRATED: [&'static str; 2] = ["event_tags", "user_subscriptions"]; + + /// The table's name, spelled as Postgres and the manifest spell it. + pub const fn name(self) -> &'static str { + match self { + Self::DvmJobs => "dvm_jobs", + Self::Events => "events", + Self::InviteCodes => "invite_codes", + Self::Invoices => "invoices", + Self::Nip05Verifications => "nip05_verifications", + Self::NotificationDeliveryLog => "notification_delivery_log", + Self::NotificationOutbox => "notification_outbox", + Self::Reports => "reports", + Self::Users => "users", + } + } +} + +/// A `users` row with the one column the seam's [`User`] omits. +/// +/// `fromDBUser` drops `tos_accepted_at` from the read shape (`src/utils/transform.ts:137-144`), +/// so no trait write can carry an operator's terms-of-service record; the migration does. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct MigratedUser { + pub user: User, + /// `users.tos_accepted_at`. + pub tos_accepted_at: Option>, +} + +/// One row of one table, as the migration moves it. +/// +/// One tuple variant per table, so a row is built as `MigrationRow::Report(Report { .. })` and +/// never as a struct variant; the variant tag is part of the canonical bytes. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub enum MigrationRow { + DvmJob(DvmJob), + Event(EventRecord), + InviteCode(InviteCode), + Invoice(Invoice), + Nip05Verification(Nip05Verification), + DeliveryLogEntry(DeliveryLogEntry), + OutboxMessage(OutboxMessage), + Report(Report), + User(MigratedUser), +} + +impl MigrationRow { + /// The table this row belongs to. + pub const fn table(&self) -> MigratedTable { + match self { + Self::DvmJob(_) => MigratedTable::DvmJobs, + Self::Event(_) => MigratedTable::Events, + Self::InviteCode(_) => MigratedTable::InviteCodes, + Self::Invoice(_) => MigratedTable::Invoices, + Self::Nip05Verification(_) => MigratedTable::Nip05Verifications, + Self::DeliveryLogEntry(_) => MigratedTable::NotificationDeliveryLog, + Self::OutboxMessage(_) => MigratedTable::NotificationOutbox, + Self::Report(_) => MigratedTable::Reports, + Self::User(_) => MigratedTable::Users, + } + } + + /// The row's key, in the byte form the LMDB database it belongs to is written under: the + /// 32 raw bytes of an id or a pubkey, the UTF-8 bytes of a text key, and the flip-signed + /// `i32` of `reports.id`. + pub fn key(&self) -> Vec { + match self { + Self::DvmJob(job) => job.id.as_bytes().to_vec(), + Self::Event(record) => record.event.id.as_bytes().to_vec(), + Self::InviteCode(invite) => invite.code.as_bytes().to_vec(), + Self::Invoice(invoice) => invoice.id.as_bytes().to_vec(), + Self::Nip05Verification(verification) => verification.pubkey.as_bytes().to_vec(), + Self::DeliveryLogEntry(entry) => entry.id.as_bytes().to_vec(), + Self::OutboxMessage(message) => message.id.as_bytes().to_vec(), + Self::Report(report) => report_key(report.id).to_vec(), + Self::User(user) => user.user.pubkey.as_bytes().to_vec(), + } + } + + /// The row's canonical bytes: `serde_json` of the row with the two normalisations applied + /// that make a source row and the same row read back from the store project identically. + /// + /// [`EventRecord::row_id`] is set to `None` — it is the one column that is not stored + /// (`crate::lmdb::rows` sets it to `None` on every read) — and every instant is truncated + /// to the microsecond precision a PostgreSQL `timestamp` column has. + pub fn canonical(&self) -> Result, StorageError> { + serde_json::to_vec(&self.normalised()).map_err(|error| StorageError::InvalidRow { + table: self.table().name(), + column: "value", + value: error.to_string(), + }) + } + + /// The row as the target stores it and as [`canonical`](Self::canonical) digests it. + pub(crate) fn normalised(&self) -> Self { + match self { + Self::DvmJob(job) => Self::DvmJob(DvmJob { + picked_up_at: job.picked_up_at.map(micros), + completed_at: job.completed_at.map(micros), + created_at: micros(job.created_at), + updated_at: micros(job.updated_at), + ..job.clone() + }), + Self::Event(record) => Self::Event(EventRecord { + row_id: None, + first_seen: record.first_seen.map(micros), + deleted_at: record.deleted_at.map(micros), + ..record.clone() + }), + Self::InviteCode(invite) => Self::InviteCode(InviteCode { + expires_at: invite.expires_at.map(micros), + created_at: micros(invite.created_at), + updated_at: micros(invite.updated_at), + ..invite.clone() + }), + Self::Invoice(invoice) => Self::Invoice(Invoice { + confirmed_at: invoice.confirmed_at.map(micros), + expires_at: invoice.expires_at.map(micros), + updated_at: micros(invoice.updated_at), + created_at: micros(invoice.created_at), + ..invoice.clone() + }), + Self::Nip05Verification(verification) => Self::Nip05Verification(Nip05Verification { + last_verified_at: verification.last_verified_at.map(micros), + last_checked_at: micros(verification.last_checked_at), + created_at: micros(verification.created_at), + updated_at: micros(verification.updated_at), + ..verification.clone() + }), + Self::DeliveryLogEntry(entry) => Self::DeliveryLogEntry(DeliveryLogEntry { + created_at: micros(entry.created_at), + ..entry.clone() + }), + Self::OutboxMessage(message) => Self::OutboxMessage(OutboxMessage { + available_at: micros(message.available_at), + delivered_at: message.delivered_at.map(micros), + created_at: micros(message.created_at), + updated_at: micros(message.updated_at), + ..message.clone() + }), + Self::Report(report) => Self::Report(Report { + created_at: micros(report.created_at), + ..report.clone() + }), + Self::User(user) => Self::User(MigratedUser { + user: User { + created_at: micros(user.user.created_at), + updated_at: micros(user.user.updated_at), + ..user.user.clone() + }, + tos_accepted_at: user.tos_accepted_at.map(micros), + }), + } + } +} + +/// Reads one table of the source, a keyset page at a time. +/// +/// The implementation issues no statement but these reads: the migration never writes to the +/// source (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D12). +pub trait MigrationSource: Send + Sync { + /// The rows of `table` after `after`, in the table's key order, at most `limit` of them. + /// + /// `after` is exclusive and `None` starts at the table's first row. A page shorter than + /// `limit` means the table has no more rows. + fn read_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec>; +} + +/// Writes one page of the source's rows into the store, and reads a page back. +pub trait MigrationTarget: Send + Sync { + /// Writes every row in `rows`, which all belong to one table, in one transaction, and + /// answers how many rows it wrote. + fn write_batch<'a>(&'a self, rows: &'a [MigrationRow]) -> StorageFuture<'a, u64>; + + /// The rows of `table` after `after`, in the store's key order, at most `limit` of them. + /// + /// This is the read the verification pass folds: it takes the rows a copy would have + /// written, not the rows a repository read would return. + fn scan_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec>; +} + +use crate::keys::micros; + +/// The `reports.id` key, in the one home the key codecs share (`crate::keys`). +fn report_key(id: i32) -> [u8; 4] { + crate::keys::i32_key(id) +} diff --git a/crates/nostream-storage/src/postgres/invoices.rs b/crates/nostream-storage/src/postgres/invoices.rs index 6a5dc7f7..74c32aff 100644 --- a/crates/nostream-storage/src/postgres/invoices.rs +++ b/crates/nostream-storage/src/postgres/invoices.rs @@ -8,7 +8,7 @@ use super::{Arg, Args, EventQuery}; use crate::error::StorageError; use crate::repositories::{Invoice, InvoiceStatus}; -const INVOICE_COLUMNS: &str = "id, pubkey, bolt11, amount_requested, amount_paid, unit, status, \ +pub(super) const INVOICE_COLUMNS: &str = "id, pubkey, bolt11, amount_requested, amount_paid, unit, status, \ description, confirmed_at, expires_at, updated_at, created_at, verify_url"; /// `findById`: one row by primary key. diff --git a/crates/nostream-storage/src/postgres/metadata.rs b/crates/nostream-storage/src/postgres/metadata.rs index 8a92908a..5e359226 100644 --- a/crates/nostream-storage/src/postgres/metadata.rs +++ b/crates/nostream-storage/src/postgres/metadata.rs @@ -14,16 +14,16 @@ use crate::repositories::{ InviteCode, NewReport, Nip05Verification, Report, }; -const NIP05_COLUMNS: &str = "pubkey, nip05, domain, is_verified, last_verified_at, \ +pub(super) const NIP05_COLUMNS: &str = "pubkey, nip05, domain, is_verified, last_verified_at, \ last_checked_at, failure_count, created_at, updated_at"; -const INVITE_COLUMNS: &str = +pub(super) const INVITE_COLUMNS: &str = "code, created_by, claimed_by, expires_at, remaining_uses, created_at, updated_at"; -const DVM_COLUMNS: &str = "id, requester_pubkey, kind, worker_index, status, result_event_id, \ +pub(super) const DVM_COLUMNS: &str = "id, requester_pubkey, kind, worker_index, status, result_event_id, \ error, picked_up_at, completed_at, created_at, updated_at"; -const REPORT_COLUMNS: &str = "id, event_id, reporter_pubkey, reported_pubkey, \ +pub(super) const REPORT_COLUMNS: &str = "id, event_id, reporter_pubkey, reported_pubkey, \ reported_event_id, report_type, weight, actionable, created_at"; /// `Nip05VerificationRepository::find_by_pubkey`. diff --git a/crates/nostream-storage/src/postgres/migration.rs b/crates/nostream-storage/src/postgres/migration.rs new file mode 100644 index 00000000..63c2df04 --- /dev/null +++ b/crates/nostream-storage/src/postgres/migration.rs @@ -0,0 +1,283 @@ +//! `MigrationSource` against Postgres: one keyset statement per table, decoded by the +//! backend's own row decoders. +//! +//! Every statement reads a whole table. It carries no filter of any kind, because the trait +//! reads cannot see the rows a migration has to move: `find_by_filters` adds +//! `deleted_at IS NULL AND (expires_at IS NULL OR expires_at > $now)`, so a soft-deleted or +//! expired event is unreachable through it, and no trait read enumerates a table at all. +//! +//! Nothing here writes. The migration issues no `INSERT`, `UPDATE`, `DELETE` or DDL against the +//! source, and never runs the embedded migrator: a source that has not been migrated fails with +//! the driver's own error (`docs/rewrite/decisions/0012-pg-to-lmdb-migration.md`, D12). + +use sqlx::Row; +use sqlx::postgres::PgRow; + +use super::invoices::INVOICE_COLUMNS; +use super::metadata::{DVM_COLUMNS, INVITE_COLUMNS, NIP05_COLUMNS, REPORT_COLUMNS}; +use super::notifications::{DELIVERY_LOG_COLUMNS, OUTBOX_COLUMNS}; +use super::sql::EVENT_COLUMNS; +use super::{Arg, Args, EventQuery, PostgresBackend, rows}; +use crate::error::StorageError; +use crate::migration::{MigratedTable, MigratedUser, MigrationRow, MigrationSource}; +use crate::repositories::StorageFuture; + +/// `users` columns: [`super::users::find_by_pubkey`]'s list plus `tos_accepted_at`, which the +/// seam's [`crate::repositories::User`] omits and a migration still has to carry. +const USER_COLUMNS: &str = "pubkey, COALESCE(is_admitted, false) AS is_admitted, \ +COALESCE(is_vanished, false) AS is_vanished, COALESCE(balance, 0) AS balance, created_at, \ +updated_at, tos_accepted_at"; + +impl MigrationSource for PostgresBackend { + fn read_batch<'a>( + &'a self, + table: MigratedTable, + after: Option<&'a [u8]>, + limit: usize, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let (sql, args) = statement(table, after, limit)?; + let rows = EventQuery { sql, args } + .into_query() + .fetch_all(self.master()) + .await?; + decode(table, &rows) + }) + } +} + +/// How a table's unique key is bound as a keyset cursor. +/// +/// Each variant is the *column's* type, read from `migrations/`, not the shape the key looks +/// like: `invoices.id` began as a `uuid` and was changed to `text` +/// (`migrations/20230205004400_change_invoice_id_to_string.sql:1`), so a `::uuid` cast against +/// it is rejected by the server on the second page. The table below is the audit. +/// +/// | table | column | type | +/// |---|---|---| +/// | `dvm_jobs` | `id` | `bytea` | +/// | `events` | `event_id` | `bytea` | +/// | `invite_codes` | `code` | `varchar(64)` | +/// | `invoices` | `id` | `text` | +/// | `nip05_verifications` | `pubkey` | `bytea` | +/// | `notification_delivery_log` | `id` | `uuid` | +/// | `notification_outbox` | `id` | `uuid` | +/// | `reports` | `id` | `serial` (`i32`) | +/// | `users` | `pubkey` | `bytea` | +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Cursor { + /// A `bytea` key: the row's key bytes are the column's value. + Bytes, + /// A `uuid` key: the row's key bytes are its canonical text, cast back to the column's type. + Uuid, + /// A `text`/`varchar` key: the row's key bytes are the column's value. + Text, + /// `reports.id`: the row's key is [`crate::migration::MigrationRow::key`]'s flip-signed + /// `i32`, which the column holds unflipped. + Int32Key, +} + +/// `SELECT FROM [WHERE > $1] ORDER BY LIMIT $n`. +/// +/// The cursor is the table's unique key, which is what makes the paging stable across a resume: +/// the copy's own `after` is that key in exactly this form. +fn statement( + table: MigratedTable, + after: Option<&[u8]>, + limit: usize, +) -> Result<(String, Args), StorageError> { + let (columns, key, cursor) = match table { + MigratedTable::DvmJobs => (DVM_COLUMNS, "id", Cursor::Bytes), + MigratedTable::Events => (EVENT_COLUMNS, "event_id", Cursor::Bytes), + MigratedTable::InviteCodes => (INVITE_COLUMNS, "code", Cursor::Text), + // `text`, not `uuid`: `20230205004400_change_invoice_id_to_string.sql` altered it. + MigratedTable::Invoices => (INVOICE_COLUMNS, "id", Cursor::Text), + MigratedTable::Nip05Verifications => (NIP05_COLUMNS, "pubkey", Cursor::Bytes), + MigratedTable::NotificationDeliveryLog => (DELIVERY_LOG_COLUMNS, "id", Cursor::Uuid), + MigratedTable::NotificationOutbox => (OUTBOX_COLUMNS, "id", Cursor::Uuid), + MigratedTable::Reports => (REPORT_COLUMNS, "id", Cursor::Int32Key), + MigratedTable::Users => (USER_COLUMNS, "pubkey", Cursor::Bytes), + }; + + let mut args = Args::default(); + let filter = match after { + None => String::new(), + Some(after) => { + let comparison = match cursor { + Cursor::Bytes => { + let slot = args.push(Arg::Bytes(after.to_vec())); + format!("{key} > {slot}") + } + Cursor::Uuid => { + let slot = args.push(Arg::Text(text(table, after)?)); + // An explicit cast: the bound value is text, the column is `uuid`. + format!("{key} > {slot}::uuid") + } + Cursor::Text => { + let slot = args.push(Arg::Text(text(table, after)?)); + format!("{key} > {slot}") + } + Cursor::Int32Key => { + let slot = args.push(Arg::Int(int32_key(table, after)?)); + format!("{key} > {slot}") + } + }; + format!(" WHERE {comparison}") + } + }; + let limit_slot = args.push(Arg::BigInt(i64::try_from(limit).unwrap_or(i64::MAX))); + let table_name = table.name(); + + Ok(( + format!("SELECT {columns} FROM {table_name} {filter} ORDER BY {key} LIMIT {limit_slot}"), + args, + )) +} + +/// One cursor as the text a `uuid`, `text` or `varchar` column holds. +fn text(table: MigratedTable, after: &[u8]) -> Result { + std::str::from_utf8(after) + .map(str::to_owned) + .map_err(|_| StorageError::InvalidRow { + table: table.name(), + column: "key", + value: "the resume cursor is not utf-8".to_owned(), + }) +} + +/// One cursor as the `i32` `reports.id` holds, through the shared key codec. +fn int32_key(table: MigratedTable, after: &[u8]) -> Result { + crate::keys::i32_from_key(after).ok_or_else(|| StorageError::InvalidRow { + table: table.name(), + column: "key", + value: format!("{} bytes are not a reports key", after.len()), + }) +} + +/// One page of rows, through the decoder the backend already has for that table. +fn decode(table: MigratedTable, rows: &[PgRow]) -> Result, StorageError> { + match table { + MigratedTable::DvmJobs => rows + .iter() + .map(|row| rows::dvm_job(row).map(MigrationRow::DvmJob)) + .collect(), + MigratedTable::Events => rows + .iter() + .map(|row| rows::event_record(row).map(MigrationRow::Event)) + .collect(), + MigratedTable::InviteCodes => rows + .iter() + .map(|row| rows::invite_code(row).map(MigrationRow::InviteCode)) + .collect(), + MigratedTable::Invoices => rows + .iter() + .map(|row| rows::invoice(row).map(MigrationRow::Invoice)) + .collect(), + MigratedTable::Nip05Verifications => rows + .iter() + .map(|row| rows::nip05_verification(row).map(MigrationRow::Nip05Verification)) + .collect(), + MigratedTable::NotificationDeliveryLog => rows + .iter() + .map(|row| rows::delivery_log_entry(row).map(MigrationRow::DeliveryLogEntry)) + .collect(), + MigratedTable::NotificationOutbox => rows + .iter() + .map(|row| rows::outbox_message(row).map(MigrationRow::OutboxMessage)) + .collect(), + MigratedTable::Reports => rows + .iter() + .map(|row| rows::report(row).map(MigrationRow::Report)) + .collect(), + MigratedTable::Users => rows.iter().map(user).collect(), + } +} + +/// One user row: [`rows::user`] plus the column the seam's read shape drops. +fn user(row: &PgRow) -> Result { + let tos_accepted_at: Option = row.try_get("tos_accepted_at")?; + Ok(MigrationRow::User(MigratedUser { + user: rows::user(row)?, + tos_accepted_at: tos_accepted_at.map(crate::records::utc_from_naive), + })) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + /// The statement one table composes when paging from `after`. + fn sql(table: MigratedTable, after: Option<&[u8]>) -> String { + statement(table, after, 2) + .expect("the statement composes") + .0 + } + + #[test] + fn the_first_page_of_every_table_binds_no_cursor() { + for table in MigratedTable::ALL { + let text = sql(table, None); + assert!( + !text.contains("WHERE"), + "{} pages from the first row: {text}", + table.name() + ); + assert!( + text.ends_with("LIMIT $1"), + "{} binds its limit first: {text}", + table.name() + ); + } + } + + /// `invoices.id` was a `uuid` until `20230205004400_change_invoice_id_to_string.sql` made it + /// `text`; a `::uuid` cast on the cursor makes the second page a server-side type error. + #[test] + fn the_uuid_cursor_is_the_two_tables_the_schema_makes_uuid() { + let uuid = b"00000000-0000-4000-8000-000000000001"; + + let invoices = sql(MigratedTable::Invoices, Some(uuid)); + assert!(invoices.contains("id > $1"), "{invoices}"); + assert!( + !invoices.contains("::uuid"), + "invoices.id is text: {invoices}" + ); + + for table in [ + MigratedTable::NotificationDeliveryLog, + MigratedTable::NotificationOutbox, + ] { + let text = sql(table, Some(uuid)); + assert!( + text.contains("id > $1::uuid"), + "{} pages on a uuid column: {text}", + table.name() + ); + } + } + + #[test] + fn every_cursor_is_the_tables_own_key() { + assert!(sql(MigratedTable::Events, Some(&[7; 32])).contains("event_id > $1")); + assert!(sql(MigratedTable::Users, Some(&[7; 32])).contains("pubkey > $1")); + assert!(sql(MigratedTable::Nip05Verifications, Some(&[7; 32])).contains("pubkey > $1")); + assert!(sql(MigratedTable::DvmJobs, Some(&[0; 32])).contains("id > $1")); + assert!(sql(MigratedTable::InviteCodes, Some(b"code-1")).contains("code > $1")); + assert!(sql(MigratedTable::Reports, Some(&[0x80, 0x00, 0x00, 0x02])).contains("id > $1")); + } + + #[test] + fn a_reports_cursor_is_undone_before_it_is_bound() { + assert_eq!( + int32_key(MigratedTable::Reports, &[0x80, 0x00, 0x00, 0x02]).unwrap(), + 2 + ); + assert_eq!( + int32_key(MigratedTable::Reports, &[0x7f, 0xff, 0xff, 0xfd]).unwrap(), + -3 + ); + assert!(int32_key(MigratedTable::Reports, &[0x80]).is_err()); + } +} diff --git a/crates/nostream-storage/src/postgres/mod.rs b/crates/nostream-storage/src/postgres/mod.rs index ed949010..f1c1fd7b 100644 --- a/crates/nostream-storage/src/postgres/mod.rs +++ b/crates/nostream-storage/src/postgres/mod.rs @@ -10,6 +10,7 @@ mod events; mod invoices; mod metadata; +mod migration; mod notifications; mod repository; mod rows; @@ -21,8 +22,8 @@ use std::sync::Arc; use std::time::Duration; use nostream_config::Settings; -use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions}; -use sqlx::{Postgres, Transaction}; +use sqlx::postgres::{PgConnectOptions, PgConnection, PgPool, PgPoolOptions}; +use sqlx::{Connection, Postgres, Transaction}; use crate::error::StorageError; @@ -204,6 +205,54 @@ impl PostgresBackend { Self::with_pools(utc(master), bounds, replica).await } + /// Builds the master pool only, reading none of the replica settings, and **does not dial**: + /// the first query is what connects. `Ok` says the settings parse, not that the server is + /// reachable. + /// + /// The migration copies from the master and nothing else, so a replica that is offline + /// during a cutover must not be able to stop it: `from_env` dials both and fails when the + /// replica is down, which is right for the relay and wrong for a one-shot copy. The copy's + /// own probe is what waits for the source, and `MIGRATION_CONNECT_CAP_MS` is the default + /// it waits with. + pub fn master_only_from_env() -> Result { + Self::master_only_from_lookup(&|name| env::var(name).ok()) + } + + /// Dials the master once, inside `budget`, and answers the driver's own error. + /// + /// A probe through the pool cannot report the reason: sqlx retries a refused socket until + /// the pool's acquire timeout, so an operator reads `pool timed out while waiting for an + /// open connection` and never learns that the port refused, the password is wrong or the + /// database does not exist. One direct attempt answers with the driver's message, and the + /// budget is what stops a black-holed host becoming a minute of silence. + pub async fn probe_master_from_env(budget: Duration) -> Result<(), StorageError> { + let lookup = |name: &str| env::var(name).ok(); + let options = utc(master_options(&lookup)?); + match tokio::time::timeout(budget, PgConnection::connect_with(&options)).await { + Ok(Ok(_connection)) => Ok(()), + Ok(Err(error)) => Err(error.into()), + Err(_) => Err(StorageError::InvalidArgument { + message: "the source did not answer within the connect budget", + }), + } + } + + fn master_only_from_lookup(lookup: &F) -> Result + where + F: Fn(&str) -> Option + Sync, + { + let master = master_options(lookup)?; + // Nothing dials here. The run's first read is its probe, and the probe is what bounds + // the wait (`MIGRATION_CONNECT_CAP_MS`), so an unreachable source is reported in + // seconds rather than at the pool's sixty-second acquire timeout. + let master = lazy_pool(utc(master), &pool_bounds(lookup, "")); + Ok(Self { + master: master.clone(), + reader: master, + settings: None, + }) + } + /// Opens the master pool and, only when a replica is configured, a second one; otherwise /// the reader is the master pool, as `src/database/client.ts:97-100` returns it. async fn with_pools( @@ -276,6 +325,15 @@ async fn connected_pool( Ok(pool_options(bounds).connect_with(options).await?) } +/// The cap, in milliseconds, on the migration's first direct connection to the source. +/// +/// A ceiling, not a default: the probe waits for the smaller of this and +/// `DB_ACQUIRE_CONNECTION_TIMEOUT`, which the shipped `.env.example` sets to 60 000 - a probe +/// that inherited that would be a minute of silence on a mistyped host. Later batches wait on +/// the pool's own acquire timeout like any other query, so this bounds the probe and nothing +/// else. +pub const MIGRATION_CONNECT_CAP_MS: u64 = 10_000; + /// The master connection options: `DB_URI` when set, else the capture's `DB_*` parts. pub(crate) fn master_options(lookup: &F) -> Result where @@ -455,6 +513,29 @@ mod tests { assert!(matches!(error, StorageError::InvalidArgument { .. })); } + /// A replica that is offline, or misconfigured, must not be able to stop a one-shot copy + /// that reads the master alone. + #[tokio::test] + async fn a_master_only_backend_reads_none_of_the_replica_settings() { + let env = lookup(&[ + ("DB_URI", "postgres://127.0.0.1:1/nostream"), + ("DB_ACQUIRE_CONNECTION_TIMEOUT", "1000"), + ("READ_REPLICA_ENABLED", "true"), + ("READ_REPLICAS", "0"), + ]); + let error = PostgresBackend::from_lookup(&env) + .await + .expect_err("a zero replica count is refused before any dial"); + assert!( + matches!(error, StorageError::InvalidArgument { .. }), + "{error}" + ); + + let backend = PostgresBackend::master_only_from_lookup(&env) + .expect("the replica settings are never read, and nothing dials here"); + drop(backend); + } + #[test] fn the_replica_index_is_the_worker_index_modulo_the_replica_count() { assert_eq!( diff --git a/crates/nostream-storage/src/postgres/notifications.rs b/crates/nostream-storage/src/postgres/notifications.rs index bd017e4a..579e3d81 100644 --- a/crates/nostream-storage/src/postgres/notifications.rs +++ b/crates/nostream-storage/src/postgres/notifications.rs @@ -13,10 +13,10 @@ use crate::repositories::{ NewDeliveryLogEntry, OutboxMessage, OutboxPayload, OutboxStatus, }; -const OUTBOX_COLUMNS: &str = "id::text AS id, event_type, payload, status, attempt_count, \ +pub(super) const OUTBOX_COLUMNS: &str = "id::text AS id, event_type, payload, status, attempt_count, \ available_at, last_error, delivered_at, created_at, updated_at"; -const DELIVERY_LOG_COLUMNS: &str = "id::text AS id, outbox_id::text AS outbox_id, event_type, \ +pub(super) const DELIVERY_LOG_COLUMNS: &str = "id::text AS id, outbox_id::text AS outbox_id, event_type, \ target_id, target_type, status, attempt_number, error_snippet, created_at"; /// `append`: one delivery attempt, timestamped now. diff --git a/crates/nostream-storage/src/postgres/sql.rs b/crates/nostream-storage/src/postgres/sql.rs index 098745c8..563b0c0c 100644 --- a/crates/nostream-storage/src/postgres/sql.rs +++ b/crates/nostream-storage/src/postgres/sql.rs @@ -11,7 +11,7 @@ use super::{DEFAULT_FILTER_LIMIT, DEFAULT_TS_CONFIG}; use crate::error::StorageError; /// The columns one stored event row is read from. -const EVENT_COLUMNS: &str = "events.id::text AS id, events.event_id, events.event_pubkey, \ +pub(super) const EVENT_COLUMNS: &str = "events.id::text AS id, events.event_id, events.event_pubkey, \ events.event_kind, events.event_created_at, events.event_content, events.event_tags, \ events.event_signature, events.first_seen, events.deleted_at, events.event_deduplication, \ host(events.remote_address) AS remote_address, events.expires_at"; diff --git a/crates/nostream-storage/tests/migration_pg.rs b/crates/nostream-storage/tests/migration_pg.rs new file mode 100644 index 00000000..c5e0af6a --- /dev/null +++ b/crates/nostream-storage/tests/migration_pg.rs @@ -0,0 +1,444 @@ +//! The migration against a real Postgres source. +//! +//! `PARITY_ADMIN_URL` names an administrative database; this target creates, migrates and seeds +//! a scratch one, copies the nine tables into a temporary LMDB store through the seam, and +//! compares them row for row. It is `#[ignore]`d for the repository's own reason — a checkout +//! without a database still has to pass `cargo test --workspace` — and CI's `Rust Checks` job +//! runs it with `--ignored`. +//! +//! Two things a copy has to get right are asserted here and nowhere else: the reader sees every +//! row the source holds, compared against `count(*)` rather than a constant, and the canonical +//! bytes of the rows the source read are the canonical bytes of the rows the store returns, +//! which is what an equal checksum in the manifest means. A second pass over the same source +//! writes the same keys and values, which is what a replayed batch and a resumed run rely on. +//! +//! The CLI's manifest, `--verify` and the throughput lines live in `nostream-cli`, which this +//! crate cannot depend on — the dependency runs the other way — so the manifest-driven half is +//! `crates/nostream-cli/tests/migrate_cli.rs`. + +#![cfg(all(feature = "postgres", feature = "lmdb"))] +// This integration-test target may unwrap: a failed assertion here is a failed test, +// which is the point. The allowance names this file rather than exempting test targets +// workspace-wide, so a new test file still has to ask. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use chrono::{DateTime, NaiveDateTime, Utc}; +use sqlx::postgres::PgPool; +use sqlx::{Executor, Row}; + +use nostream_storage::LmdbBackend; +use nostream_storage::migration::{MigratedTable, MigrationRow, MigrationSource, MigrationTarget}; +use nostream_storage::postgres::PostgresBackend; + +const DATABASE: &str = "nostream_migration_rust"; + +/// The paging the copy runs at here, small enough that a table of three is two pages. +const BATCH: usize = 2; + +fn admin_url() -> String { + std::env::var("PARITY_ADMIN_URL").expect( + "PARITY_ADMIN_URL must name an administrative database, e.g. \ + postgres://postgres:postgres@host:5432/postgres", + ) +} + +fn with_database(url: &str, database: &str) -> String { + let (prefix, query) = match url.split_once('?') { + Some((prefix, query)) => (prefix, Some(query)), + None => (url, None), + }; + let base = match prefix.rfind('/') { + Some(index) if index > "postgres://".len() => &prefix[..index], + _ => prefix, + }; + match query { + Some(query) => format!("{base}/{database}?{query}"), + None => format!("{base}/{database}"), + } +} + +/// A naive instant (a `timestamp` column), 2020-01-01T00:00:00Z plus `offset` seconds. +fn naive(offset: i64) -> NaiveDateTime { + timestamptz(offset).naive_utc() +} + +/// A UTC instant (a `timestamptz` column), 2020-01-01T00:00:00Z plus `offset` seconds. +fn timestamptz(offset: i64) -> DateTime { + DateTime::::from_timestamp(1_577_836_800 + offset, 0).expect("a whole second") +} + +fn canonical(row: &MigrationRow) -> Vec { + row.canonical().expect("a row serialises") +} + +/// Pages one table of the store to its end. +async fn scan_all(target: &LmdbBackend, table: MigratedTable) -> Vec { + let mut rows = Vec::new(); + let mut after: Option> = None; + loop { + let page = MigrationTarget::scan_batch(target, table, after.as_deref(), BATCH) + .await + .expect("the store scans"); + let count = page.len(); + if let Some(last) = page.last() { + after = Some(last.key()); + } + rows.extend(page); + if count < BATCH { + return rows; + } + } +} + +/// Copies one table of the source into the store, page by page, and answers the rows it read. +async fn copy_table( + source: &PostgresBackend, + target: &LmdbBackend, + table: MigratedTable, +) -> Vec { + let mut rows = Vec::new(); + let mut after: Option> = None; + loop { + let batch = MigrationSource::read_batch(source, table, after.as_deref(), BATCH) + .await + .expect("the source reads"); + let count = batch.len(); + if let Some(last) = batch.last() { + after = Some(last.key()); + } + let written = MigrationTarget::write_batch(target, &batch) + .await + .expect("the store writes"); + assert_eq!( + written, + count as u64, + "{} writes every row it reads", + table.name() + ); + rows.extend(batch); + if count < BATCH { + return rows; + } + } +} + +/// `count(*)` for one table, so the reader is compared with the source rather than a constant. +async fn source_count(pool: &PgPool, table: MigratedTable) -> i64 { + let sql = match table { + MigratedTable::DvmJobs => "SELECT count(*) FROM dvm_jobs", + MigratedTable::Events => "SELECT count(*) FROM events", + MigratedTable::InviteCodes => "SELECT count(*) FROM invite_codes", + MigratedTable::Invoices => "SELECT count(*) FROM invoices", + MigratedTable::Nip05Verifications => "SELECT count(*) FROM nip05_verifications", + MigratedTable::NotificationDeliveryLog => "SELECT count(*) FROM notification_delivery_log", + MigratedTable::NotificationOutbox => "SELECT count(*) FROM notification_outbox", + MigratedTable::Reports => "SELECT count(*) FROM reports", + MigratedTable::Users => "SELECT count(*) FROM users", + }; + sqlx::query(sqlx::AssertSqlSafe(sql)) + .fetch_one(pool) + .await + .expect("the source counts") + .try_get::(0) + .expect("a count") +} + +/// A fresh scratch database with the schema applied. +/// +/// The targets that use it run with `--test-threads=1` (CI and `scripts/storage-parity.sh`), so +/// two of them never drop the database under each other. +async fn scratch() -> PostgresBackend { + let admin = PgPool::connect(&admin_url()) + .await + .expect("the administrative database is reachable"); + admin + .execute(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {DATABASE} WITH (FORCE)" + ))) + .await + .expect("the scratch database drops"); + admin + .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {DATABASE}"))) + .await + .expect("the scratch database is created"); + drop(admin); + + let source = PostgresBackend::connect(&with_database(&admin_url(), DATABASE)) + .await + .expect("the scratch database is reachable"); + source.migrate().await.expect("the schema migrates"); + source +} + +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] +async fn a_seeded_source_copies_into_an_lmdb_store_that_matches_it() { + let source = scratch().await; + seed(&source).await; + + let directory = tempfile::tempdir().expect("a temporary directory"); + let target = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + + for table in MigratedTable::ALL { + let read = copy_table(&source, &target, table).await; + assert_eq!( + read.len() as i64, + source_count(source.master(), table).await, + "{} reads every row the source holds", + table.name() + ); + + let scanned = scan_all(&target, table).await; + assert_eq!(scanned.len(), read.len(), "{} count", table.name()); + + let mut expected: Vec> = read.iter().map(canonical).collect(); + let mut actual: Vec> = scanned.iter().map(canonical).collect(); + expected.sort(); + actual.sort(); + assert_eq!(expected, actual, "{} canonical bytes", table.name()); + } + + // A second pass writes the same keys and values: the store's content does not move, which is + // what makes a replayed batch and a resumed run harmless. + for table in MigratedTable::ALL { + let again = copy_table(&source, &target, table).await; + assert_eq!( + scan_all(&target, table).await.len(), + again.len(), + "{} holds one row per source row after a second pass", + table.name() + ); + } +} + +/// Three rows at a batch size of two makes the second page real, which is the statement a wrong +/// cursor type rejects: `invoices.id` is `text` since +/// `migrations/20230205004400_change_invoice_id_to_string.sql` (so `id > $1::uuid` is a +/// server-side type error), `invite_codes.code` is a `varchar(64)` that takes the same text bind, +/// and the two notification tables' `id`s are `uuid`. Every page is read through the seam's own +/// reader, so the assertion is on the SQL that reader composes. +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; the Rust Checks job sets it"] +async fn every_text_and_uuid_cursor_pages_over_its_own_column_type() { + let source = scratch().await; + let pool = source.master(); + + for seed in 1..=3u8 { + let outbox_id = format!("40000000-0000-4000-8000-00000000000{seed}"); + sqlx::query( + "INSERT INTO invoices (id, pubkey, bolt11, amount_requested, unit, status) \ + VALUES ($1, $2, $3, 1000, 'msats', 'completed')", + ) + .bind(format!("30000000-0000-4000-8000-00000000000{seed}")) + .bind(vec![4u8; 32]) + .bind(format!("lnbc1page{seed}")) + .execute(pool) + .await + .expect("the invoice inserts"); + + sqlx::query( + "INSERT INTO notification_outbox (id, event_type, payload) \ + VALUES ($1::uuid, 'invoice.paid', $2)", + ) + .bind(&outbox_id) + .bind(serde_json::json!({ "seed": seed })) + .execute(pool) + .await + .expect("the outbox row inserts"); + + sqlx::query( + "INSERT INTO notification_delivery_log (id, outbox_id, event_type, target_id, \ + target_type, status, attempt_number) \ + VALUES ($1::uuid, $2::uuid, 'invoice.paid', 'telegram:7175022', 'telegram', \ + 'success', 1)", + ) + .bind(format!("50000000-0000-4000-8000-00000000000{seed}")) + .bind(&outbox_id) + .execute(pool) + .await + .expect("the delivery-log row inserts"); + + sqlx::query("INSERT INTO invite_codes (code) VALUES ($1)") + .bind(format!("invite-{seed}")) + .execute(pool) + .await + .expect("the invite code inserts"); + } + + for table in [ + MigratedTable::InviteCodes, + MigratedTable::Invoices, + MigratedTable::NotificationDeliveryLog, + MigratedTable::NotificationOutbox, + ] { + let first = MigrationSource::read_batch(&source, table, None, BATCH) + .await + .expect("the first page reads"); + assert_eq!(first.len(), BATCH, "{} has a first page", table.name()); + let cursor = first.last().expect("a row").key(); + + let second = MigrationSource::read_batch(&source, table, Some(&cursor), BATCH) + .await + .expect("the cursor binds as the column's own type"); + assert_eq!( + second.len(), + 1, + "{} pages past its first batch", + table.name() + ); + assert!( + second[0].key() > cursor, + "{} pages in key order", + table.name() + ); + + let after = second.last().expect("a row").key(); + let third = MigrationSource::read_batch(&source, table, Some(&after), BATCH) + .await + .expect("the third page reads"); + assert!(third.is_empty(), "{} is exhausted", table.name()); + } +} + +/// Two rows in each seeded table, except events (three, one per liveness state) and the two +/// tables the manifest does not move. +async fn seed(backend: &PostgresBackend) { + let pool = backend.master(); + + // events: live, soft-deleted, expired; a tag, a deduplication value and a first_seen. + for (seed, deleted_at, expires_at) in [ + (1u8, None, None), + (2u8, Some(naive(500)), None), + (3u8, None, Some(1_577_836_800i32)), + ] { + sqlx::query( + "INSERT INTO events (event_id, event_pubkey, event_kind, event_created_at, \ + event_content, event_signature, event_tags, first_seen, deleted_at, expires_at, \ + event_deduplication, remote_address) \ + VALUES ($1, $2, 1, $3, 'hello', $4, $5, $6, $7, $8, $9, $10::inet)", + ) + .bind(vec![seed; 32]) + .bind(vec![1u8; 32]) + .bind(1_600_000_000i32 + i32::from(seed)) + .bind(vec![9u8; 64]) + .bind(serde_json::json!([["t", "x"]])) + .bind(naive(0)) + .bind(deleted_at) + .bind(expires_at) + .bind(serde_json::json!(["abc"])) + .bind(None::) + .execute(pool) + .await + .expect("the event inserts"); + } + + for seed in 1..=2u8 { + sqlx::query( + "INSERT INTO users (pubkey, is_admitted, is_vanished, balance, tos_accepted_at, \ + created_at, updated_at) VALUES ($1, true, false, $2, $3, $4, $4)", + ) + .bind(vec![30 + seed; 32]) + .bind(if seed == 1 { 42i64 } else { 0 }) + .bind((seed == 1).then(|| naive(60))) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the user inserts"); + + sqlx::query( + "INSERT INTO invoices (id, pubkey, bolt11, amount_requested, amount_paid, unit, \ + status, description, confirmed_at, expires_at, created_at, updated_at) \ + VALUES ($1::uuid, $2, $3, 1000, $4, 'msats', $5, 'an invoice', $6, $7, $8, $8)", + ) + .bind(format!("00000000-0000-4000-8000-00000000000{seed}")) + .bind(vec![4u8; 32]) + .bind(format!("lnbc1invoice{seed}")) + .bind((seed == 1).then_some(1_000i64)) + .bind(if seed == 1 { "completed" } else { "pending" }) + .bind((seed == 1).then(|| naive(50))) + .bind(naive(60)) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the invoice inserts"); + + sqlx::query( + "INSERT INTO reports (event_id, reporter_pubkey, reported_pubkey, reported_event_id, \ + report_type, weight, actionable, created_at) \ + VALUES ($1, $2, $3, NULL, 'spam', 1.5, true, $4)", + ) + .bind(vec![40 + seed; 32]) + .bind(vec![5u8; 32]) + .bind(vec![6u8; 32]) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the report inserts"); + + sqlx::query( + "INSERT INTO dvm_jobs (id, requester_pubkey, kind, worker_index, status, \ + result_event_id, error, picked_up_at, completed_at, created_at, updated_at) \ + VALUES ($1, $2, 5100, 1, 'completed', $3, NULL, $4, $5, $6, $6)", + ) + .bind(vec![seed; 32]) + .bind(vec![2u8; 32]) + .bind(vec![200 + seed; 32]) + .bind(timestamptz(10)) + .bind(timestamptz(20)) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the job inserts"); + } + + sqlx::query( + "INSERT INTO nip05_verifications (pubkey, nip05, domain, is_verified, last_verified_at, \ + last_checked_at, failure_count, created_at, updated_at) \ + VALUES ($1, 'user@example.com', 'example.com', true, $2, $3, 1, $3, $2)", + ) + .bind(vec![20u8; 32]) + .bind(timestamptz(80)) + .bind(timestamptz(90)) + .execute(pool) + .await + .expect("the verification inserts"); + + sqlx::query( + "INSERT INTO invite_codes (code, created_by, claimed_by, expires_at, remaining_uses, \ + created_at, updated_at) VALUES ('code-1', $1, NULL, $2, 2, $3, $3)", + ) + .bind(vec![3u8; 32]) + .bind(timestamptz(100)) + .bind(timestamptz(0)) + .execute(pool) + .await + .expect("the invite code inserts"); + + sqlx::query( + "INSERT INTO notification_outbox (id, event_type, payload, status, attempt_count, \ + available_at, last_error, delivered_at, created_at, updated_at) \ + VALUES ($1::uuid, 'invoice.paid', $2, 'delivered', 1, $3, NULL, $4, $3, $4)", + ) + .bind("10000000-0000-4000-8000-000000000001") + .bind(serde_json::json!({ "amount": 1_000 })) + .bind(timestamptz(120)) + .bind(timestamptz(130)) + .execute(pool) + .await + .expect("the outbox row inserts"); + + sqlx::query( + "INSERT INTO notification_delivery_log (id, outbox_id, event_type, target_id, \ + target_type, status, attempt_number, error_snippet, created_at) \ + VALUES ($1::uuid, $2::uuid, 'invoice.paid', 'telegram:7175022', 'telegram', 'success', \ + 1, NULL, $3)", + ) + .bind("20000000-0000-4000-8000-000000000001") + .bind("10000000-0000-4000-8000-000000000001") + .bind(timestamptz(110)) + .execute(pool) + .await + .expect("the delivery-log row inserts"); +} diff --git a/crates/nostream-storage/tests/migration_seam.rs b/crates/nostream-storage/tests/migration_seam.rs new file mode 100644 index 00000000..f1417510 --- /dev/null +++ b/crates/nostream-storage/tests/migration_seam.rs @@ -0,0 +1,130 @@ +//! The migration seam's canonical projection and key bytes, with no driver compiled in. +//! +//! This target is also the proof that the seam builds without either backend: it runs under +//! `cargo test -p nostream-storage --no-default-features --test migration_seam`. +//! +//! These targets may unwrap and assert by panicking: a failed assertion is a failed test. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use chrono::{DateTime, TimeZone, Utc}; +use nostream_core::{Event, EventId, Kind, Pubkey, Signature}; +use nostream_storage::migration::MigrationRow; +use nostream_storage::repositories::{EventRecord, Report, ReportType}; + +fn instant(text: &str) -> DateTime { + DateTime::parse_from_rfc3339(text) + .expect("a fixture instant") + .with_timezone(&Utc) +} + +/// An event record with the given row id, content and `first_seen`. +fn record(row_id: Option<&str>, content: &str, first_seen: DateTime) -> EventRecord { + EventRecord { + row_id: row_id.map(str::to_owned), + event: Event { + id: EventId::from_byte_array([7; 32]), + pubkey: Pubkey::from_byte_array([1; 32]), + created_at: 1_577_836_800, + kind: Kind::TEXT_NOTE, + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([9; 64]), + }, + first_seen: Some(first_seen), + deleted_at: None, + expires_at: None, + event_deduplication: None, + remote_address: None, + } +} + +fn canonical(row: &MigrationRow) -> Vec { + row.canonical().expect("a row serialises") +} + +#[test] +fn the_canonical_projection_drops_the_row_id_and_truncates_instants() { + let epoch = instant("2020-01-01T00:00:00Z"); + + // `events.id` is the one column the store does not keep, so it is not in the bytes. + let with_row_id = canonical(&MigrationRow::Event(record( + Some("6f1e7c1a-0000-4000-8000-000000000000"), + "a", + epoch, + ))); + assert_eq!( + with_row_id, + canonical(&MigrationRow::Event(record(None, "a", epoch))) + ); + assert!(!with_row_id.is_empty()); + + // Every instant is truncated to the microsecond precision a `timestamp` column has. + let nanos = Utc + .timestamp_opt(1_577_836_800, 123_456_789) + .single() + .expect("a nanosecond instant"); + let micros = Utc + .timestamp_opt(1_577_836_800, 123_456_000) + .single() + .expect("the same instant at microseconds"); + assert_eq!( + canonical(&MigrationRow::Event(record(None, "a", nanos))), + canonical(&MigrationRow::Event(record(None, "a", micros))) + ); + + // A projection that returns one constant fails both of these. + assert_ne!( + canonical(&MigrationRow::Event(record(None, "a", epoch))), + canonical(&MigrationRow::Event(record(None, "b", epoch))), + "one wire field apart is different bytes" + ); + // `timestamp_opt`'s second argument is nanoseconds, so one microsecond is a thousand. + let one_microsecond_later = Utc + .timestamp_opt(1_577_836_800, 1_000) + .single() + .expect("one microsecond later"); + assert_ne!( + canonical(&MigrationRow::Event(record(None, "a", epoch))), + canonical(&MigrationRow::Event(record( + None, + "a", + one_microsecond_later + ))), + "a projection that truncates to whole seconds fails here" + ); + + // The whole literal, so an empty, truncated or re-ordered projection fails. + let report = MigrationRow::Report(Report { + id: 1, + event_id: EventId::from_byte_array([0; 32]), + reporter_pubkey: Pubkey::from_byte_array([0; 32]), + reported_pubkey: None, + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + created_at: epoch, + }); + let zeros = "0".repeat(64); + let expected = format!( + concat!( + "{{\"Report\":{{\"id\":1,\"event_id\":\"{}\",\"reporter_pubkey\":\"{}\",", + "\"reported_pubkey\":null,\"reported_event_id\":null,\"report_type\":\"spam\",", + "\"weight\":1.0,\"actionable\":true,\"created_at\":\"2020-01-01T00:00:00Z\"}}}}" + ), + zeros, zeros + ); + assert_eq!( + String::from_utf8(canonical(&report)).expect("the projection is utf-8"), + expected + ); + + // The key bytes are the store's: the raw id and the flip-signed `reports.id`. + let event = record(None, "a", epoch); + assert_eq!( + MigrationRow::Event(event.clone()).key(), + event.event.id.as_bytes().to_vec() + ); + assert_eq!(report.key(), vec![0x80, 0x00, 0x00, 0x01]); +} diff --git a/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md new file mode 100644 index 00000000..c14e19ad --- /dev/null +++ b/docs/rewrite/decisions/0012-pg-to-lmdb-migration.md @@ -0,0 +1,273 @@ +# ADR 0012: The Postgres-to-LMDB migration (job 029) + +- **Status:** accepted (job 029, Phase 2) +- **Deciders:** the rewrite team, on the evidence in job 029's recon and spec +- **Date:** 2026-11-11 + +## Context + +The rewrite's store becomes LMDB, and the relay that is running holds its data in the +Postgres schema ADR 0003 defines. Cutover therefore needs one operation the repository did +not have: a copy of every table from the Postgres store into an LMDB store, resumable after +a crash, with a record of what it moved so the result can be checked before anything is +pointed at it. + +The nine repository traits cannot carry that copy. No trait enumerates a table — every read +is a filtered query — and every write is shaped like "a new row now": `create` stamps +`first_seen` with the clock, the invoice insert clears `amount_paid` and `confirmed_at`, +`UserPatch` carries neither `balance` nor `created_at`, and the outbox and delivery-log +inserts mint fresh ids. A trait-only copy would be both lossy and partial, which is the one +thing a cutover tool must not be. + +This document records the twelve decisions the tool rests on, each with the reason it was +taken. + +## The copy goes through a migration-only seam (D1) + +`nostream_storage::migration` is a new module with two traits — `MigrationSource` and +`MigrationTarget` — and the rows they move. It exists for `nostream-cli migrate pg-to-lmdb` +and for no other caller; its doc comment says so, and a second caller needs a record of its +own first. + +The alternative was to implement the copy in the CLI, and it loses on three counts. The +Postgres row decoders are in-crate, the event column list is private to the crate +(`postgres/sql.rs`), and eight of the nine LMDB per-table write paths already exist in +`lmdb/`. A CLI-side copy would restate nine column lists and nine index write paths and +drift from the ones the relay uses. The seam is also the only place where "write this row +exactly" can sit beside the write paths it has to agree with. + +The seam compiles with no driver: the two implementations are `#[cfg(feature)]` and the +vocabulary is not, so `cargo check -p nostream-storage --no-default-features --all-targets` +still passes and a build that links neither backend can still name a `MigrationRow`. + +The cost is a public surface in a crate that documents what it does not own. The crate's +`lib.rs` says which half it owns — the seam, not the tool — and the doc comment on the +module repeats it where an implementor meets it. + +## Nine tables, two stated limits (D2) + +`MigratedTable` has nine variants: `dvm_jobs`, `events`, `invite_codes`, `invoices`, +`nip05_verifications`, `notification_delivery_log`, `notification_outbox`, `reports`, +`users`. They are the tables the store has a repository and a database for. + +Two of the capture's tables are not moved, and both are stated limits rather than omissions: + +- `event_tags` is a trigger-maintained index in Postgres. LMDB's `tags` database is written + from the event row itself by the event write path, so copying the table would duplicate + what the event write already produces. +- `user_subscriptions` has a TypeScript repository and no Rust trait, and no LMDB database + (ADR 0011). It is that repository's slice to build, not this tool's to invent. + +`MigratedTable::NOT_MIGRATED` carries both names and the manifest prints them, so an +operator reading the result sees the two tables the store does not hold instead of having to +know. + +## Every column is preserved, through migration-only writers (D3) + +Each row goes to LMDB through a writer that puts the source's own key and value and every +index entry that table's existing write path writes, and stamps, clears and defaults +nothing. `events` is the one table whose write set had to be composed: the value, the +created-at index, the author index, the tag index and the expiring index are the set +`write_event` writes, the `replaceable` pointer is what `write_upserted` writes, and an +`events_deleted` entry is what `soft_delete` writes — so a migrated soft-deleted or expired +row, which never passed through `soft_delete`, gets both tail entries from the migration +writer itself. + +`users.tos_accepted_at` is carried, which no trait write can do: the seam's `User` omits the +column deliberately (`fromDBUser` drops it), so the seam carries the pair +`MigratedUser { user, tos_accepted_at }`. + +One field is lost, and it is lost on purpose. `events.id` is the Postgres v4 uuid, no LMDB +reader uses it, and the store does not keep it. It is dropped from the canonical projection +so a source row and the same row read back from the store stay comparable. + +## The canonical projection and its checksum (D4) + +`MigrationRow::canonical` is `serde_json` of the row — so the enum's variant tag is part of +the bytes — after exactly two normalisations: `EventRecord::row_id` is set to `None`, and +every `DateTime` is truncated to the microsecond precision a PostgreSQL `timestamp` +column has. Both are what makes a source row and its stored copy project identically. + +The checksum is the 256-bit sum of per-row `SHA-256(len32 ‖ key ‖ canonical)`, added +modulo 2^256. Three properties made that shape the choice: the sum does not depend on the +order rows are read in, it needs constant memory over a table of any size, and it composes +across a resume (`sum(A ∪ B) = sum(A) + sum(B)`), so an interrupted run continues the number +it had rather than starting again. + +It is a corruption and completeness check — any added, removed or altered row moves it — and +not a cryptographic commitment, and it is not a defence against a deliberate collision. +Nothing here needs one. + +## Resume is per table, and a replay is one batch (D5) + +The manifest holds, per table, a keyset cursor `after`, the accumulated `rows` and +`checksum`, and `complete`. A run resumes each incomplete table at its cursor and seeds the +count and the sum from the entry. + +The store commit and the manifest write cannot share a transaction through the public +surface, so the batch is committed first and the manifest is replaced atomically second — a +temporary file, synced before the rename, and the directory synced after it. Both syncs are +load-bearing: a rename is durable only when the directory entry is on disk, and a manifest +that survives as a name pointing at a truncated file is worse than a missing one, because +`read_manifest` refuses it and the target refusal below then blocks the store it describes. +The manifest is written 0600 and through an exclusively created temporary file: it records the +last key written per table, and for `invite_codes` that key is the invite code itself — a bearer +credential NIP-43 accepts — so a world-readable manifest would hand one out, and a planted link +at the temporary path must not be written through and renamed into place. +A crash between the two therefore replays at most one batch, +which is harmless because every write in it is a put of the source's own key and value: the +same rows produce the same count and the same sum, so the merged entry is unchanged. + +For the first batch that replay has nothing to fall back on, so the run writes the manifest +fresh before its first read: every table `rows: 0`, the zero checksum, `after: ""`, +`complete: false`. A store this tool has begun to fill therefore always carries a manifest, and +the refusal of a store that is not this tool's stays what it should be — a judgement about rows +that appeared without one, not about a crash. + +Two refusals guard the target, and they are the same judgement from two sides. A store that holds +rows and carries no manifest is not this tool's: copying over it would leave the displaced rows' +index entries behind, and verify reads the primary databases only, so the run refuses before it +reads anything. A manifest that records progress — a row counted, a cursor moved, a table +finished — is only meaningful beside the store that progress went into, so the run requires +`data.mdb` to be a file there and refuses a missing one instead of creating it; a copy over an +empty store would skip the tables already counted and exit 0 against a store missing every row +the manifest claims. The file test cannot tell a store the run wrote from a truncated one that +still has a `data.mdb`, so a second refusal covers that shape: a table the manifest counts rows +for must hold at least one row, or the store beside the manifest is not the one the run wrote +to. A manifest with no progress cannot have written a row, so a fresh store beside +it is honest and the run starts one. + +## The command surface (D6) + +The source is named before it is dialled. With nothing configured, `PostgresBackend::from_env` +falls back to `postgres@localhost:5432/postgres`, so the run refuses at once and names `DB_URI` +and the `DB_*` parts rather than copying whatever a local server happens to hold; a source that +cannot be reached is reported as the database it could not read — host and database, password +dropped — instead of as the pool's acquire timeout a minute later. + +A target that is a symlink, writable by group or other, or owned by another user, is refused +before anything is opened: the copy writes the dataset into `/lmdb`, so a writable +target lets another local user pre-place the store and read what lands in it, an owner can +replace the directory between the check and the open whatever its bits say, and a link sends +the copy somewhere the operator did not name. The owner is compared with the process's +effective uid, read from `/proc/self/status`, because `std::fs` exposes the file's owner but +not the caller's and the alternative was a dependency for one integer. The target the run +writes into is printed with the summary. + +`nostream-cli migrate pg-to-lmdb` takes `--target `, `--batch-size ` and `--verify`, +and nothing else. `--target` is required with no default: a default of the working directory +would let a mistyped run write a store where none was meant, and verify needs it too because +the manifest lives under it. `--batch-size` defaults to 1 000, the import's default, and its +minimum is 1: a batch of zero would never advance a cursor. `--verify` copies nothing and +reads only the target. + +Verify creates no store and no manifest: it requires `/lmdb/data.mdb` to be a file +before it opens anything, and opens through `LmdbBackend::open_existing`: a read-only +environment, no `create_dir_all`, every database opened by name rather than created. That +opener commits the read transaction it opens those handles in, because heed publishes a +database handle only once its opening transaction has committed: a dropped read transaction +leaves the handles unusable, and the next transaction on them answers `Invalid argument` — +which a fresh process, having no earlier committed open to inherit them from, hits on its +first read. Every in-process test misses it, because the store was opened writable first. +LMDB's read-only open is not entirely a read: it recreates `lock.mdb` beside an existing +`data.mdb` (measured), and a target directory it cannot create that file in is refused. The +ordinary opener would answer a verify of a store that is not there with a fresh empty +environment and nine short tables — a check that reports corruption it caused. heed's +process-global registry answers `BadOpenOptions` for a second open of one path with different +options, and it holds the environment process-wide, so dropping a handle does not free the +path. That is why the resume check asks whether `data.mdb` is there instead of opening the +store read-only first: the writable open that follows would otherwise refuse the one case the +check exists to allow. + +A re-run resumes wherever the manifest says the last one stopped. Deleting +`/migration-manifest.json` does not restart a half-filled store — a store that holds rows +and no manifest is refused — so starting over takes a fresh `--target`, or the manifest and the +store removed together; the command's help and its refusals say so. + +Exit codes are the CLI's own, from `CLI.md`: 0 on success, 2 for a usage error, 1 for every +other failure, which is written to stderr as `Migration failed: {message}`. That message +needs a `CliError` variant of its own — reusing `Import` would report `Import failed:` for a +migration — so `CliError::Migrate` exists and `main` prints it with its own prefix. + +The help text states the one precondition nothing can check (see below) and names the source: +Postgres from `DB_URI`, else the `DB_*` parts. + +## The manifest (D7) + +The manifest is `/migration-manifest.json`: `format: 1`, `not_migrated` with the two +table names, and one entry per table carrying `rows`, `checksum`, `after` and `complete`. + +It carries no timestamp, no rate and no path. That is what makes two runs over the same +source produce byte-identical manifests, and what lets an interrupted-then-resumed run be +compared with a clean one as bytes rather than as a parse. It also keeps a source URL out of +a file that may be copied around. + +The checksum string is sixty-four `0` characters for an empty table, so an entry is always +present and always well formed. + +## Throughput is the run's own measurement (D8) + +The copy measures the wall clock from just before its first read to just after its last +write and prints two lines: + +``` +Migrated {total} rows in {ms} ms +Events throughput: {rate} events/s +``` + +`total` is the rows this invocation wrote, and the rate is `events * 1000 / max(ms, 1)` over +the `events` table only. Both numbers are plain digits so a reader can recompute the rate +exactly, and the rate comes from the same integers that were printed. + +**No floor is asserted.** The board's 50 000 events/s target belongs to the benchmark rig, +which does not exist; a CI runner's rate is not that number, so this tool reports what it +measured and claims nothing about it. + +## The relay must be stopped (D9) + +Both stores are documented preconditions of a run, printed in the command's help: the relay +must be stopped on both. + +The reason is that nothing in this repository proves a second process may hold the LMDB +environment. LMDB's lock file would let one in, and heed opens with no flags beyond the +verify opener's `READ_ONLY`, so the locking protocol is active — but the daemon opens no backend of its own today, so +sharing is untested rather than known-safe. The tool takes no lock of its own: a lock that +nothing else honours is worse than a documented precondition, because an operator would read +it as protection. + +## A refused row stops the run (D10) + +A row the store refuses — a tag value at the 469-byte key bound, a NUL in a text key — fails +the batch and stops the run. The message names the table, not a batch ordinal, so an +operator knows which row stopped the copy, and the manifest keeps the previous batch's +cursor, count and sum, so the next run resumes at the refused row. + +Nothing is skipped and no table is marked complete. Skipping would leave the target missing +a row while the manifest's checksum still counted it, which turns a row an operator can look +at into a verify failure after a full copy. + +## The report sequence moves with the rows (D11) + +`reports.id` is carried across, and after each batch `meta`'s `reports.next_id` is raised to +the largest migrated id in that batch — never lowered, and left alone by a batch with no +report. A later `ReportRepository::create` then allocates past the migrated rows instead of +colliding with them. + +## The source is read-only (D12) + +The migration issues no `INSERT`, `UPDATE`, `DELETE` or DDL against Postgres, does not run +the embedded migrator, and runs no statement but the nine paged reads. Its schema contract +is the migrations' own, so a source that has not been migrated fails with the driver's +error rather than being repaired in place. + +A read that repaired its source would be a second writer on a live relay's database and +would make a failed run something an operator has to undo. + +## What this record does not decide + +- No throughput floor: the benchmark rig owns that number. +- No cutover, and no judgement that the migrated store is fit to serve. +- No migration of `user_subscriptions` or `event_tags`. +- No reverse migration, and no migration between two Postgres databases. +- No change to the nine traits, the LMDB encodings, the database list, the Postgres schema, + `LMDB_MAP_SIZE`, `LMDB_MAX_DBS` or the maximum key length. diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index 3e8291e4..494efe0e 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -1,6 +1,7 @@ #!/bin/sh -# The storage-parity oracle: the 37 sqlx migrations, the differential 51-method replay and the -# unmodified TypeScript relay's Cucumber suite, all against databases this script creates. +# The storage-parity oracle: the 37 sqlx migrations, the differential 51-method replay, the +# Postgres-to-LMDB migration and the unmodified TypeScript relay's Cucumber suite, all against +# databases this script creates. # # Usage: scripts/storage-parity.sh # @@ -285,6 +286,11 @@ echo "== behavioural parity" cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \ || fail "the parity target failed" +echo "== the Postgres-to-LMDB migration" +./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" \ + cargo test -p nostream-storage --test migration_pg -- --ignored --test-threads=1 \ + || fail "the migration target failed" + echo "== lmdb behavioural parity (no database)" ./scripts/rust.sh cargo test -p nostream-storage --test parity_lmdb -- --nocapture \ || fail "the lmdb parity target failed" @@ -317,4 +323,4 @@ NODE_OPTIONS=--no-experimental-strip-types \ "$CUCUMBER_BIN" --format summary \ || fail "the cucumber suite failed" -echo "== storage-parity: all four steps passed" +echo "== storage-parity: all steps passed"