From 51abc7a9da3888d41a1fcc3ac953011d4d184230 Mon Sep 17 00:00:00 2001 From: Priyanshubhartistm Date: Sun, 15 Nov 2026 01:41:15 +0530 Subject: [PATCH 1/3] docs(storage): state the count cap cost as a decode per key past it Signed-off-by: Priyanshubhartistm --- .changeset/storage-count-cap-cost-docs.md | 14 +++++++++++ crates/nostream-storage/src/lmdb/events.rs | 15 ++++++------ docs/rewrite/decisions/0011-lmdb-backend.md | 27 ++++++++++++--------- 3 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 .changeset/storage-count-cap-cost-docs.md diff --git a/.changeset/storage-count-cap-cost-docs.md b/.changeset/storage-count-cap-cost-docs.md new file mode 100644 index 00000000..5a309906 --- /dev/null +++ b/.changeset/storage-count-cap-cost-docs.md @@ -0,0 +1,14 @@ +--- +--- + +docs(storage): state the count cap's cost as a decode per key past it, not per repeated key (#1081) + +ADR 0011 and the comments in `lmdb/events.rs` said the cap on a tag count's id set costs +"a body decode per repeated key past it". `counted_earlier` reads the row's body for every +key the set does not answer past the cap, unless the range is the family's first exact +criterion, so the cost is not confined to repeated keys: a `{"#t":["zz","a"]}` filter, +one key a row and no repeats, costs 5.7× its uncapped walk at 20,000 rows. The record +and its copies now say so, and qualify "below the cap the walk reads no body" to a walk +over a single tag family, since a second family decodes there too. + +Comment and prose only: no code changes and no behaviour change. diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index c1281b4f..c35c91d6 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -722,7 +722,7 @@ struct Plan<'a> { /// `tag_ranges` builds one range per criterion that fits a key, carrying that criterion. A /// key in these ranges has already proved that family, so the row's body is decoded only for /// the filter's other tag families, and a `tags` count holds one id per row its matcher - /// accepts until the set is full, then reads a repeated key's body instead. + /// accepts until the set is full, then reads the body of every key the set does not answer. tag: Option<(&'a str, &'a [String])>, } @@ -902,9 +902,10 @@ fn count_matches_bounded( // A single exact criterion names one key per row, so only a second criterion or a `#g` // prefix can walk one row twice, and only then is the id set paid for. // - // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; a repeated key past that reads the - // row's body to ask whether an earlier key of this walk already answered for it, against a - // slot per repeated row. Upgrade when a count past the cap must decode nothing, which needs + // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; every key past that the set does + // not answer reads the row's body to ask whether an earlier key of this walk already answered + // for it, not only a repeated one, against a slot per repeated row. Upgrade when a count + // past the cap must decode nothing, which needs // an index order that visits a row once. let mut counted_ids = HashSet::new(); let dedupe = plan.tag.is_some_and(|(name, criteria)| { @@ -3196,9 +3197,9 @@ mod tests { ); } - /// A count's total does not depend on how many ids its walk may hold: past the cap a - /// repeated key reads the row's body to tell an earlier key from a first one, and this runs - /// that walk and the one the set alone answers over one corpus. + /// A count's total does not depend on how many ids its walk may hold: past the cap a key the + /// set does not answer reads the row's body to tell an earlier key from a first one, and this + /// runs that walk and the one the set alone answers over one corpus. #[test] fn a_count_past_its_id_cap_totals_what_the_unbounded_walk_totals() { let _serial = counting(); diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md index c8f34cf4..15204d09 100644 --- a/docs/rewrite/decisions/0011-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -197,15 +197,18 @@ as the capture's does. One read keeps a second thing resident, and it is a count's rather than a page's: a `tags` count whose criteria can select one row twice holds an id per row its matcher accepts, and that set is -bounded by `MAX_PAGE_KEYS` — the same 5 000. Below the cap the walk reads no body; at the cap it -stops inserting and decides each further key with a bounded-memory check instead: a repeated key -reads the row's body and asks whether an earlier key of the walk already answered for it, an -earlier criterion that matches one of its tags or, within one `#g` prefix range, a smaller value -the prefix matches. The total is therefore the same at every corpus size, and the set is a -constant 270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, -against 1 081 360 B for the same 20 000-row corpus with the set unbounded). The price of the -cap is a body decode per repeated key past it; an index order that visits each row once is what -would remove that too. +bounded by `MAX_PAGE_KEYS` — the same 5 000. Below the cap a walk over a single tag family reads +no body (a second family decodes there too, as below); at the cap it stops inserting and decides +each further key with a bounded-memory check instead: a key the set does not answer reads the +row's body and asks whether an earlier key of the walk already answered for it, an earlier +criterion that matches one of its tags or, within one `#g` prefix range, a smaller value the +prefix matches. The total is therefore the same at every corpus size, and the set is a constant +270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, +against 1 081 360 B for the same 20 000-row corpus with the set unbounded). Past the cap every key +the set does not answer reads the row's body — a later criterion's key to ask whether an earlier +criterion matched the row, a `#g` prefix's key to ask whether a smaller value did — not only a +repeated one. The walk costs 5.3× to 5.7× its uncapped time at 20,000 rows for that reason. An +index order that visits each row once is what would remove that cost. The index is chosen from the filter, and which ranges can stop early follows from the key: @@ -254,7 +257,8 @@ Two consequences of the same bound: row two criteria — or two values of a `#g` prefix — both select is counted at the first key the walk reaches and skipped at the rest without a body decode, while the set has room: the ranges run in the criteria's order, so that key is not necessarily the row's smallest matching - one. Once the set is full the walk reads the body of a repeated key instead and asks whether + one. Once the set is full the walk reads the body of every key the set does not answer + instead and asks whether an earlier key of the walk already answered for it — an earlier criterion, or a smaller value of the same prefix — so the count stays exact and its resident set stays a constant. Only those two shapes hold the set, whose population is one id per row its matcher accepts: a single exact @@ -357,7 +361,8 @@ its op is one of the five above — so the list cannot grow without the diff sho - A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range; a count materialises no result set, and the ids it holds for a tag walk that can select a row twice are bounded by the same `MAX_PAGE_KEYS` — 270 352 B of resident set once full, however large - the corpus, with a body decode per repeated key past the cap. So a client cannot choose how + the corpus, with a body decode per key past the cap that the set does not answer, not only per + repeated one. So a client cannot choose how much a request materialises. The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page without kinds reads the created-at index, and so does an author × kind filter whose From 6cb26b6ad524a8cae7adeead9ce35f8f0407fa99 Mon Sep 17 00:00:00 2001 From: Priyanshubhartistm Date: Sun, 15 Nov 2026 12:35:23 +0530 Subject: [PATCH 2/3] docs(storage): limit the per-key decode to keys past the first criterion Signed-off-by: Priyanshubhartistm --- crates/nostream-storage/src/lmdb/events.rs | 17 ++++++----- docs/rewrite/decisions/0011-lmdb-backend.md | 33 +++++++++++---------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index c35c91d6..d75354e6 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -722,7 +722,8 @@ struct Plan<'a> { /// `tag_ranges` builds one range per criterion that fits a key, carrying that criterion. A /// key in these ranges has already proved that family, so the row's body is decoded only for /// the filter's other tag families, and a `tags` count holds one id per row its matcher - /// accepts until the set is full, then reads the body of every key the set does not answer. + /// accepts until the set is full, then reads the body of every key outside the family's + /// first exact criterion. tag: Option<(&'a str, &'a [String])>, } @@ -902,10 +903,10 @@ fn count_matches_bounded( // A single exact criterion names one key per row, so only a second criterion or a `#g` // prefix can walk one row twice, and only then is the id set paid for. // - // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; every key past that the set does - // not answer reads the row's body to ask whether an earlier key of this walk already answered - // for it, not only a repeated one, against a slot per repeated row. Upgrade when a count - // past the cap must decode nothing, which needs + // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; every key past that outside the + // family's first exact criterion reads the row's body to ask whether an earlier key of this + // walk already answered for it, not only a repeated one, against a slot per repeated row. + // Upgrade when a count past the cap must decode nothing, which needs // an index order that visits a row once. let mut counted_ids = HashSet::new(); let dedupe = plan.tag.is_some_and(|(name, criteria)| { @@ -3197,9 +3198,9 @@ mod tests { ); } - /// A count's total does not depend on how many ids its walk may hold: past the cap a key the - /// set does not answer reads the row's body to tell an earlier key from a first one, and this - /// runs that walk and the one the set alone answers over one corpus. + /// A count's total does not depend on how many ids its walk may hold: past the cap a key + /// outside the family's first exact criterion reads the row's body to tell an earlier key from + /// a first one, and this runs that walk and the one the set alone answers over one corpus. #[test] fn a_count_past_its_id_cap_totals_what_the_unbounded_walk_totals() { let _serial = counting(); diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md index 15204d09..4381bb39 100644 --- a/docs/rewrite/decisions/0011-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -199,16 +199,17 @@ One read keeps a second thing resident, and it is a count's rather than a page's whose criteria can select one row twice holds an id per row its matcher accepts, and that set is bounded by `MAX_PAGE_KEYS` — the same 5 000. Below the cap a walk over a single tag family reads no body (a second family decodes there too, as below); at the cap it stops inserting and decides -each further key with a bounded-memory check instead: a key the set does not answer reads the -row's body and asks whether an earlier key of the walk already answered for it, an earlier -criterion that matches one of its tags or, within one `#g` prefix range, a smaller value the -prefix matches. The total is therefore the same at every corpus size, and the set is a constant -270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, +each further key with a bounded-memory check instead: a key outside the family's first exact +criterion reads the row's body and asks whether an earlier key of the walk already answered for +it, an earlier criterion that matches one of its tags or, within one `#g` prefix range, a smaller +value the prefix matches. The total is therefore the same at every corpus size, and the set is a +constant 270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, against 1 081 360 B for the same 20 000-row corpus with the set unbounded). Past the cap every key -the set does not answer reads the row's body — a later criterion's key to ask whether an earlier -criterion matched the row, a `#g` prefix's key to ask whether a smaller value did — not only a -repeated one. The walk costs 5.3× to 5.7× its uncapped time at 20,000 rows for that reason. An -index order that visits each row once is what would remove that cost. +outside the family's first exact criterion reads the row's body — a later criterion's key to ask +whether an earlier criterion matched the row, a `#g` prefix's key to ask whether a smaller value +did — not only a repeated one, while the first criterion's own keys decode nothing. The walk costs +5.3× to 5.7× its uncapped time at 20,000 rows for that reason. An index order that visits each row +once is what would remove that cost. The index is chosen from the filter, and which ranges can stop early follows from the key: @@ -257,10 +258,10 @@ Two consequences of the same bound: row two criteria — or two values of a `#g` prefix — both select is counted at the first key the walk reaches and skipped at the rest without a body decode, while the set has room: the ranges run in the criteria's order, so that key is not necessarily the row's smallest matching - one. Once the set is full the walk reads the body of every key the set does not answer - instead and asks whether - an earlier key of the walk already answered for it — an earlier criterion, or a smaller value - of the same prefix — so the count stays exact and its resident set stays a constant. Only those + one. Once the set is full the walk reads the body of every key outside the family's first + exact criterion instead and asks whether an earlier key of the walk already answered for it — + an earlier criterion, or a smaller value of the same prefix — so the count stays exact and its + resident set stays a constant. Only those two shapes hold the set, whose population is one id per row its matcher accepts: a single exact criterion names one key per row. The tags index has no order that visits each row once, which is why the cap trades a slot past it for a body decode. @@ -361,9 +362,9 @@ its op is one of the five above — so the list cannot grow without the diff sho - A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range; a count materialises no result set, and the ids it holds for a tag walk that can select a row twice are bounded by the same `MAX_PAGE_KEYS` — 270 352 B of resident set once full, however large - the corpus, with a body decode per key past the cap that the set does not answer, not only per - repeated one. So a client cannot choose how - much a request materialises. + the corpus, with a body decode per key past the cap outside the family's first exact + criterion, not only per repeated one. So a client cannot choose how much a request + materialises. The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page without kinds reads the created-at index, and so does an author × kind filter whose composition passes `MAX_PAGE_KEYS` ranges; an `ids` read whose criteria are exact is bounded From 5da05b09c00efa79ba8a5f395771ece7d8443e1b Mon Sep 17 00:00:00 2001 From: Priyanshubhartistm Date: Sun, 15 Nov 2026 13:53:33 +0530 Subject: [PATCH 3/3] docs(storage): keep the saved-id exception beside the first-criterion one Signed-off-by: Priyanshubhartistm --- crates/nostream-storage/src/lmdb/events.rs | 16 +++++---- docs/rewrite/decisions/0011-lmdb-backend.md | 37 +++++++++++---------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index d75354e6..35f56cc2 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -722,8 +722,8 @@ struct Plan<'a> { /// `tag_ranges` builds one range per criterion that fits a key, carrying that criterion. A /// key in these ranges has already proved that family, so the row's body is decoded only for /// the filter's other tag families, and a `tags` count holds one id per row its matcher - /// accepts until the set is full, then reads the body of every key outside the family's - /// first exact criterion. + /// accepts until the set is full, then reads the body of every key the set does not answer + /// outside the family's first exact criterion. tag: Option<(&'a str, &'a [String])>, } @@ -903,9 +903,10 @@ fn count_matches_bounded( // A single exact criterion names one key per row, so only a second criterion or a `#g` // prefix can walk one row twice, and only then is the id set paid for. // - // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; every key past that outside the - // family's first exact criterion reads the row's body to ask whether an earlier key of this - // walk already answered for it, not only a repeated one, against a slot per repeated row. + // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; every key past that the set does + // not answer, outside the family's first exact criterion, reads the row's body to ask whether + // an earlier key of this walk already answered for it, not only a repeated one, against a slot + // per repeated row. // Upgrade when a count past the cap must decode nothing, which needs // an index order that visits a row once. let mut counted_ids = HashSet::new(); @@ -3199,8 +3200,9 @@ mod tests { } /// A count's total does not depend on how many ids its walk may hold: past the cap a key - /// outside the family's first exact criterion reads the row's body to tell an earlier key from - /// a first one, and this runs that walk and the one the set alone answers over one corpus. + /// the set does not answer, outside the family's first exact criterion, reads the row's body + /// to tell an earlier key from a first one, and this runs that walk and the one the set alone + /// answers over one corpus. #[test] fn a_count_past_its_id_cap_totals_what_the_unbounded_walk_totals() { let _serial = counting(); diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md index 4381bb39..ceba6a9a 100644 --- a/docs/rewrite/decisions/0011-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -199,17 +199,18 @@ One read keeps a second thing resident, and it is a count's rather than a page's whose criteria can select one row twice holds an id per row its matcher accepts, and that set is bounded by `MAX_PAGE_KEYS` — the same 5 000. Below the cap a walk over a single tag family reads no body (a second family decodes there too, as below); at the cap it stops inserting and decides -each further key with a bounded-memory check instead: a key outside the family's first exact -criterion reads the row's body and asks whether an earlier key of the walk already answered for -it, an earlier criterion that matches one of its tags or, within one `#g` prefix range, a smaller -value the prefix matches. The total is therefore the same at every corpus size, and the set is a -constant 270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, -against 1 081 360 B for the same 20 000-row corpus with the set unbounded). Past the cap every key -outside the family's first exact criterion reads the row's body — a later criterion's key to ask -whether an earlier criterion matched the row, a `#g` prefix's key to ask whether a smaller value -did — not only a repeated one, while the first criterion's own keys decode nothing. The walk costs -5.3× to 5.7× its uncapped time at 20,000 rows for that reason. An index order that visits each row -once is what would remove that cost. +each further key with a bounded-memory check instead: a key the set does not answer, outside the +family's first exact criterion, reads the row's body and asks whether an earlier key of the walk +already answered for it, an earlier criterion that matches one of its tags or, within one `#g` +prefix range, a smaller value the prefix matches. The total is therefore the same at every corpus +size, and the set is a constant 270 352 B once full (measured live: 5 000 ids at a `HashSet` +capacity of 7 168 slots, against 1 081 360 B for the same 20 000-row corpus with the set +unbounded). Past the cap every key the set does not answer, outside the family's first exact +criterion, reads the row's body — a later criterion's key to ask whether an earlier criterion +matched the row, a `#g` prefix's key to ask whether a smaller value did — not only a repeated one, +while the first criterion's own keys decode nothing. The walk costs 5.3× to 5.7× its uncapped time +at 20,000 rows for that reason. An index order that visits each row once is what would remove that +cost. The index is chosen from the filter, and which ranges can stop early follows from the key: @@ -258,10 +259,10 @@ Two consequences of the same bound: row two criteria — or two values of a `#g` prefix — both select is counted at the first key the walk reaches and skipped at the rest without a body decode, while the set has room: the ranges run in the criteria's order, so that key is not necessarily the row's smallest matching - one. Once the set is full the walk reads the body of every key outside the family's first - exact criterion instead and asks whether an earlier key of the walk already answered for it — - an earlier criterion, or a smaller value of the same prefix — so the count stays exact and its - resident set stays a constant. Only those + one. Once the set is full the walk reads the body of every key the set does not answer, + outside the family's first exact criterion, instead and asks whether an earlier key of the walk + already answered for it — an earlier criterion, or a smaller value of the same prefix — so the + count stays exact and its resident set stays a constant. Only those two shapes hold the set, whose population is one id per row its matcher accepts: a single exact criterion names one key per row. The tags index has no order that visits each row once, which is why the cap trades a slot past it for a body decode. @@ -362,9 +363,9 @@ its op is one of the five above — so the list cannot grow without the diff sho - A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range; a count materialises no result set, and the ids it holds for a tag walk that can select a row twice are bounded by the same `MAX_PAGE_KEYS` — 270 352 B of resident set once full, however large - the corpus, with a body decode per key past the cap outside the family's first exact - criterion, not only per repeated one. So a client cannot choose how much a request - materialises. + the corpus, with a body decode per key past the cap that the set does not answer and + that lies outside the family's first exact criterion, not only per repeated one. So a client + cannot choose how much a request materialises. The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page without kinds reads the created-at index, and so does an author × kind filter whose composition passes `MAX_PAGE_KEYS` ranges; an `ids` read whose criteria are exact is bounded