From 7315816ae2fa88170f37aff8fbcfd5380cd80bbc Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:58:09 +0000 Subject: [PATCH 1/8] yeast: Require context for template lowering Pass the generated or caller-spanned `ctx` identifier through every template lowering path instead of falling back to an implicitly constructed identifier. This is needed primarily due to macro hygiene. Eventually we will be introducing macros that define macros, and here it's problematic that we rely on an implicit `ctx` variable to be present. So instead, we pass a suitable `ctx` identifier through the entire lowering, and interpolate that (using `#ctx`) instead. --- shared/yeast-macros/src/lower.rs | 63 +++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/shared/yeast-macros/src/lower.rs b/shared/yeast-macros/src/lower.rs index 8fa4662374e2..32ef0cd8b967 100644 --- a/shared/yeast-macros/src/lower.rs +++ b/shared/yeast-macros/src/lower.rs @@ -7,9 +7,10 @@ use crate::ast::{ Alternative, Cardinality, Content, Field, Node, Pattern, Replacement, Rule, Template, }; -#[derive(Clone, Copy, Default)] +#[derive(Clone, Copy)] struct LoweringContext<'a> { choice_alternative_scope: Option>, + ctx: &'a Ident, } #[derive(Clone, Copy)] @@ -69,12 +70,12 @@ impl Rule { quote! { yeast::OutputId }, &translated_captures_ident, ); - let replacement = self.replacement.lower(); + let ctx = Ident::new("ctx", Span::call_site()); + let replacement = self.replacement.lower(&ctx); let guard = self .guard .clone() .unwrap_or_else(|| syn::parse_quote!(true)); - let ctx = Ident::new("ctx", Span::call_site()); quote! { { @@ -105,16 +106,16 @@ impl Rule { } impl Replacement { - fn lower(&self) -> TokenStream { + fn lower(&self, ctx: &Ident) -> TokenStream { match self { - Replacement::Templates(templates) => Template::lower_list(templates), + Replacement::Templates(templates) => Template::lower_list(templates, ctx), Replacement::Rust { body, .. } => { let body = rust_block(body); quote! { let __value = ::std::iter::IntoIterator::into_iter(#body) .collect::>(); let mut __ids: Vec = Vec::new(); - ctx.extend_field_ids(__value, &mut __ids); + #ctx.extend_field_ids(__value, &mut __ids); __ids } } @@ -307,15 +308,23 @@ fn capture_bindings<'a>( } impl Template { - pub(crate) fn lower_root(&self) -> TokenStream { + pub(crate) fn lower_root(&self, ctx: &Ident) -> TokenStream { + let context = LoweringContext { + choice_alternative_scope: None, + ctx, + }; match self { - Template::Choice(alternatives) => Self::lower_choice(alternatives), - _ => self.lower_single(LoweringContext::default()), + Template::Choice(alternatives) => Self::lower_choice(alternatives, context), + _ => self.lower_single(context), } } - pub(crate) fn lower_list(templates: &[Self]) -> TokenStream { - let context = LoweringContext::default(); + pub(crate) fn lower_list(templates: &[Self], ctx: &Ident) -> TokenStream { + let context = LoweringContext { + choice_alternative_scope: None, + ctx, + }; + let ctx = context.ctx; let items = templates .iter() .map(|template| match template { @@ -328,15 +337,16 @@ impl Template { let block = rust_block(block); isolated_interpolation( uses_ctx, + ctx, quote! { let __value = ::std::iter::IntoIterator::into_iter(#block) .collect::>(); - ctx.extend_field_ids(__value, &mut __nodes); + #ctx.extend_field_ids(__value, &mut __nodes); }, ) } Template::Choice(alternatives) => { - let pick = Self::lower_choice(alternatives); + let pick = Self::lower_choice(alternatives, context); quote! { if let ::std::option::Option::Some(__id) = #pick { __nodes.push(__id); @@ -356,6 +366,7 @@ impl Template { } fn lower_single(&self, context: LoweringContext<'_>) -> TokenStream { + let ctx = context.ctx; match self { Template::Node(node) => node.lower(context), Template::Splice(block) => { @@ -363,6 +374,7 @@ impl Template { let block = rust_block(block); isolated_interpolation( uses_ctx, + ctx, quote! { ::std::convert::Into::::into(#block) }, @@ -374,22 +386,27 @@ impl Template { } } - fn lower_choice(alternatives: &[Alternative]) -> TokenStream { + fn lower_choice(alternatives: &[Alternative], context: LoweringContext<'_>) -> TokenStream { let pick = alternatives.iter().rev().fold( quote! { ::std::option::Option::None }, - |fallback, alternative| alternative.lower_candidate(fallback), + |fallback, alternative| alternative.lower_candidate(fallback, context), ); quote! {{ #pick }} } } impl Alternative { - fn lower_candidate(&self, fallback: TokenStream) -> TokenStream { + fn lower_candidate( + &self, + fallback: TokenStream, + parent_context: LoweringContext<'_>, + ) -> TokenStream { let interpolations = RefCell::new(Vec::new()); let context = LoweringContext { choice_alternative_scope: Some(ChoiceAlternativeScope { interpolations: &interpolations, }), + ctx: parent_context.ctx, }; let value = self.template.lower_single(context); let interpolations = interpolations.into_inner(); @@ -456,7 +473,7 @@ impl Alternative { impl Node { fn lower(&self, context: LoweringContext<'_>) -> TokenStream { - let ctx = Ident::new("ctx", Span::call_site()); + let ctx = context.ctx; let kind_str = self.kind.to_string(); let node_init = if let Some(content) = &self.content { @@ -470,6 +487,7 @@ impl Node { let source_range = fresh_prepared_ident("source_range"); let expression = isolated_interpolation( uses_ctx, + ctx, quote! { let __expr = #block; yeast::MaybeYeastValue::maybe_yeast_value(&__expr).map( @@ -506,6 +524,7 @@ impl Node { } else { isolated_interpolation( uses_ctx, + ctx, quote! { let __expr = #block; let __value = @@ -542,6 +561,7 @@ impl Node { let block = rust_block(block); stmts.push(isolated_interpolation( uses_ctx, + ctx, quote! { let __value = ::std::iter::IntoIterator::into_iter(#block) .collect::>(); @@ -570,6 +590,7 @@ impl Node { let block = rust_block(block); let interpolation = isolated_interpolation( uses_ctx, + ctx, quote! { let __value = ::std::iter::IntoIterator::into_iter(#block) .collect::>(); @@ -587,7 +608,7 @@ impl Node { }); } Template::Choice(alternatives) => { - let pick = Template::lower_choice(alternatives); + let pick = Template::lower_choice(alternatives, context); stmts.push(quote! { let #temp: ::std::option::Option = #pick; }); @@ -642,12 +663,12 @@ fn rust_block(block: &syn::Block) -> TokenStream { quote!({ #(#statements)* }) } -fn isolated_interpolation(uses_ctx: bool, body: TokenStream) -> TokenStream { +fn isolated_interpolation(uses_ctx: bool, ctx: &Ident, body: TokenStream) -> TokenStream { if uses_ctx { quote! {{ - let __saved_user_ctx = (*ctx.user_ctx).clone(); + let __saved_user_ctx = (*#ctx.user_ctx).clone(); let __result = { #body }; - *ctx.user_ctx = __saved_user_ctx; + *#ctx.user_ctx = __saved_user_ctx; __result }} } else { From bbe8ad10654460f48a4d29b5a29f0e3b9ba43dbb Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:59:19 +0000 Subject: [PATCH 2/8] yeast: Add schema-bound tree entry points Add hidden `tree!` and `trees!` variants that inject a schema dependency and preserve access to the caller's `ctx` binding. Essentially, these inner variants (`tree_with_schema!` etc.) will be used internally with an interpolated reference to the output schema. Thus, when we write `tree!(...)` this turns into `tree_with_schema(SCHEMA, ...)`, which enables that local macro call to (eventually) check the tree template against that schema. --- shared/yeast-macros/src/lib.rs | 20 ++++++++++++++ shared/yeast-macros/src/parse.rs | 45 +++++++++++++++++++++++++++++++- shared/yeast/src/lib.rs | 4 +++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 868ebe15cdff..2cf24edfc798 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -118,6 +118,16 @@ pub fn tree(input: TokenStream) -> TokenStream { } } +#[doc(hidden)] +#[proc_macro] +pub fn tree_with_schema(input: TokenStream) -> TokenStream { + let input2: TokenStream2 = input.into(); + match parse::parse_tree_with_schema_top(input2) { + Ok(output) => output.into(), + Err(err) => err.to_compile_error().into(), + } +} + /// Build a list of AST nodes from a template, returning `Vec`. /// /// Like `tree!` but returns `Vec` and supports multiple top-level @@ -137,6 +147,16 @@ pub fn trees(input: TokenStream) -> TokenStream { } } +#[doc(hidden)] +#[proc_macro] +pub fn trees_with_schema(input: TokenStream) -> TokenStream { + let input2: TokenStream2 = input.into(); + match parse::parse_trees_with_schema_top(input2) { + Ok(output) => output.into(), + Err(err) => err.to_compile_error().into(), + } +} + /// Define a desugaring rule with query and transform in one declaration. /// /// ```text diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index ccc8b036ff4e..8870f739d5b0 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -1,4 +1,4 @@ -use proc_macro2::{Span, TokenStream, TokenTree}; +use proc_macro2::{Ident, Span, TokenStream, TokenTree}; use quote::quote; type Result = std::result::Result; @@ -19,11 +19,47 @@ pub fn parse_tree_top(input: TokenStream) -> Result { Ok(quote!({ #template })) } +pub fn parse_tree_with_schema_top(input: TokenStream) -> Result { + let mut tokens = input.into_iter().peekable(); + let output_path = parse_named_string_arg(&mut tokens, "output")?; + expect_punct(&mut tokens, ',', "expected `,` after output schema path")?; + let template_tokens: TokenStream = tokens.collect(); + let ctx = Ident::new( + "ctx", + first_leaf_span(template_tokens.clone()).unwrap_or_else(Span::call_site), + ); + let template = crate::template_parse::parse_template(template_tokens)?; + let template = template.lower_root(&ctx); + let output_lit = proc_macro2::Literal::string(&output_path.value); + Ok(quote!({ + const _: &::core::primitive::str = ::core::include_str!(#output_lit); + #template + })) +} + pub fn parse_trees_top(input: TokenStream) -> Result { let templates = crate::template_parse::parse_templates(input)?; Ok(crate::ast::Template::lower_list(&templates)) } +pub fn parse_trees_with_schema_top(input: TokenStream) -> Result { + let mut tokens = input.into_iter().peekable(); + let output_path = parse_named_string_arg(&mut tokens, "output")?; + expect_punct(&mut tokens, ',', "expected `,` after output schema path")?; + let template_tokens: TokenStream = tokens.collect(); + let ctx = Ident::new( + "ctx", + first_leaf_span(template_tokens.clone()).unwrap_or_else(Span::call_site), + ); + let templates = crate::template_parse::parse_templates(template_tokens)?; + let templates = crate::ast::Template::lower_list(&templates, &ctx); + let output_lit = proc_macro2::Literal::string(&output_path.value); + Ok(quote!({ + const _: &::core::primitive::str = ::core::include_str!(#output_lit); + #templates + })) +} + pub fn parse_rule_top(input: TokenStream) -> Result { Ok(crate::rule_parse::parse_rule(input)?.lower()) } @@ -113,6 +149,13 @@ struct NamedString { literal_span: Span, } +fn first_leaf_span(stream: TokenStream) -> Option { + stream.into_iter().find_map(|token| match token { + TokenTree::Group(group) => first_leaf_span(group.stream()), + token => Some(token.span()), + }) +} + fn parse_named_string_arg( tokens: &mut std::iter::Peekable, expected_name: &str, diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index e34fa1490d79..35643a1b7949 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -15,6 +15,10 @@ pub mod schema; mod visitor; pub use range::{Point, Range}; +#[doc(hidden)] +pub use yeast_macros::tree_with_schema as __tree_with_schema; +#[doc(hidden)] +pub use yeast_macros::trees_with_schema as __trees_with_schema; pub use yeast_macros::{query, rule, rules, tree, trees}; use captures::{Captures, TranslatedCaptures}; From e520a2ec29ca94120fe5db9cc428e74c67610dbe Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:59:29 +0000 Subject: [PATCH 3/8] yeast: Add tree schema namespaces Adds a new macro, `tree_schema!` which takes two arguments: an identifier and the name of an output schema. When the macro is executed, it then creates a local module with the same name as the specified identifier, which exports `tree` and `trees` macros and also the schema. These can then be imported in order to get access to these macros. --- shared/yeast-macros/src/lib.rs | 20 +++++++++ shared/yeast-macros/src/parse.rs | 69 +++++++++++++++++++++++++++++++- shared/yeast/src/lib.rs | 2 +- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 2cf24edfc798..c76bd881af38 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -157,6 +157,26 @@ pub fn trees_with_schema(input: TokenStream) -> TokenStream { } } +/// Declare an output-schema namespace with importable `tree!` and `trees!` +/// wrappers and the schema contents. +/// +/// ```ignore +/// yeast::tree_schema! { +/// swift_tree, +/// "ast_types.yml", +/// } +/// +/// use swift_tree::{tree, trees, SCHEMA}; +/// ``` +#[proc_macro] +pub fn tree_schema(input: TokenStream) -> TokenStream { + let input2: TokenStream2 = input.into(); + match parse::parse_tree_schema_top(input2) { + Ok(output) => output.into(), + Err(err) => err.to_compile_error().into(), + } +} + /// Define a desugaring rule with query and transform in one declaration. /// /// ```text diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 8870f739d5b0..4d80202da647 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -60,6 +60,51 @@ pub fn parse_trees_with_schema_top(input: TokenStream) -> Result { })) } +pub fn parse_tree_schema_top(input: TokenStream) -> Result { + let mut tokens = input.into_iter().peekable(); + let module = expect_ident(&mut tokens, "expected schema module name")?; + expect_punct(&mut tokens, ',', "expected `,` after schema module name")?; + let output_path = parse_string_arg(&mut tokens, "schema")?; + if tokens.peek().is_some() { + expect_punct(&mut tokens, ',', "expected trailing `,`")?; + } + if let Some(token) = tokens.next() { + return Err(syn::Error::new_spanned( + token, + "unexpected token after tree schema declaration", + )); + } + + let output_abs = resolve_manifest_path(&output_path, "tree_schema")?; + let output_lit = proc_macro2::Literal::string(&output_abs.to_string_lossy()); + Ok(quote! { + pub(crate) mod #module { + #[allow(dead_code)] + pub const SCHEMA: &str = ::core::include_str!(#output_lit); + + macro_rules! tree { + ($($tokens:tt)*) => { + yeast::__tree_with_schema! { + output: #output_lit, + $($tokens)* + } + }; + } + pub(crate) use tree; + + macro_rules! trees { + ($($tokens:tt)*) => { + yeast::__trees_with_schema! { + output: #output_lit, + $($tokens)* + } + }; + } + pub(crate) use trees; + } + }) +} + pub fn parse_rule_top(input: TokenStream) -> Result { Ok(crate::rule_parse::parse_rule(input)?.lower()) } @@ -149,6 +194,21 @@ struct NamedString { literal_span: Span, } +fn resolve_manifest_path(path: &NamedString, macro_name: &str) -> Result { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").map_err(|_| { + syn::Error::new( + Span::call_site(), + format!("{macro_name}!: CARGO_MANIFEST_DIR is not set; cannot resolve schema path"), + ) + })?; + let path = std::path::PathBuf::from(&path.value); + Ok(if path.is_absolute() { + path + } else { + std::path::PathBuf::from(manifest_dir).join(path) + }) +} + fn first_leaf_span(stream: TokenStream) -> Option { stream.into_iter().find_map(|token| match token { TokenTree::Group(group) => first_leaf_span(group.stream()), @@ -172,11 +232,18 @@ fn parse_named_string_arg( ':', &format!("expected `:` after `{expected_name}`"), )?; + parse_string_arg(tokens, expected_name) +} + +fn parse_string_arg( + tokens: &mut std::iter::Peekable, + description: &str, +) -> Result { let literal = expect_literal(tokens)?; let value = string_literal_value(&literal).ok_or_else(|| { syn::Error::new( literal.span(), - format!("`{expected_name}` must be a string literal path"), + format!("`{description}` must be a string literal path"), ) })?; Ok(NamedString { diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 35643a1b7949..01b7241fcdc0 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -19,7 +19,7 @@ pub use range::{Point, Range}; pub use yeast_macros::tree_with_schema as __tree_with_schema; #[doc(hidden)] pub use yeast_macros::trees_with_schema as __trees_with_schema; -pub use yeast_macros::{query, rule, rules, tree, trees}; +pub use yeast_macros::{query, rule, rules, tree, tree_schema, trees}; use captures::{Captures, TranslatedCaptures}; use query::QueryNode; From 342e0a973c6c56aa1a62c9c3062cd59e904499dc Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:58:16 +0000 Subject: [PATCH 4/8] yeast: Use schema-bound builders in tests --- shared/yeast/tests/test.rs | 89 ++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 02b0c59f7c43..4e9d89150c87 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -7,6 +7,13 @@ use yeast::*; const OUTPUT_SCHEMA_YAML: &str = include_str!("node-types.yml"); +yeast::tree_schema! { + test_tree, + "tests/node-types.yml", +} + +use test_tree::{tree, trees}; + /// Helper: parse Ruby source with no rules, return dump. fn parse_and_dump(input: &str) -> String { let ast = parse_input(input); @@ -769,9 +776,9 @@ fn test_tree_builder() { // Swap left and right let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); - let left = yeast::tree!((identifier "x")); - let right = yeast::tree!((integer "1")); - let new_id = yeast::tree!( + let left = tree!((identifier "x")); + let right = tree!((integer "1")); + let new_id = tree!( (program child: (assignment left: {right} @@ -792,6 +799,32 @@ fn test_tree_builder() { ); } +#[test] +fn test_tree_schema_namespace() { + assert_eq!(test_tree::SCHEMA, OUTPUT_SCHEMA_YAML); + + let input_ast = parse_input("x = 1"); + let mut output = empty_output_ast("x = 1"); + let mut user_ctx = (); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut output, &mut user_ctx); + let assignment = tree!((assignment + left: (identifier "x") + right: (integer "1"))); + assert_eq!( + ctx.output.get_node(assignment).unwrap().kind_name(), + "assignment" + ); + + let nodes = trees!((identifier "x") (integer "1")); + assert_eq!(nodes.len(), 2); + + let bare_leaf = tree!((identifier)); + assert_eq!( + ctx.output.get_node(bare_leaf).unwrap().kind_name(), + "identifier" + ); +} + /// Builds `(assignment left: … right: [(integer #{value})])`, where a `None` /// `value` leaves `right` unset rather than producing an `integer` with no /// content. @@ -802,8 +835,8 @@ fn build_choice_right( ) -> (yeast::OutputId, yeast::OutputId) { let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(input_ast, ast, &mut user_ctx); - let left = yeast::tree!((identifier "x")); - let root = yeast::tree!( + let left = tree!((identifier "x")); + let root = tree!( (assignment left: {left} right: [(integer #{value})] @@ -859,7 +892,7 @@ fn test_choice_uses_first_constructible_alternative() { let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); let absent: Option = None; - let root = yeast::tree!( + let root = tree!( (assignment left: [ (identifier #{absent}) @@ -886,7 +919,7 @@ fn test_choice_guard_skips_an_alternative() { let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); let use_first = false; - let root = yeast::tree!( + let root = tree!( (assignment left: [ (identifier "first") if { use_first } @@ -913,11 +946,11 @@ fn test_choice_fallback_can_reuse_consumed_splice() { let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); let items = vec![ - yeast::tree!((identifier "x")), - yeast::tree!((identifier "y")), + tree!((identifier "x")), + tree!((identifier "y")), ]; let use_first = false; - let root = yeast::tree!( + let root = tree!( (assignment left: [ (left_assignment_list @@ -955,9 +988,9 @@ fn test_output_template_interpolations_clone_user_context() { let mut ast = empty_output_ast(input); let mut user_ctx = InterpolationContext::default(); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); - let source = yeast::tree!((identifier "source")); + let source = tree!((identifier "source")); - let nodes = yeast::trees!( + let nodes = trees!( (call location: { ctx.counter += 1; @@ -1005,9 +1038,9 @@ fn test_interpolation_preserves_build_ctx_method_borrowing() { let mut ast = empty_output_ast(input); let mut user_ctx = InterpolationContext::default(); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); - let source = yeast::tree!((identifier "source")); + let source = tree!((identifier "source")); - let node = yeast::tree!((call + let node = tree!((call method: { ctx.literal("identifier", &ctx.source_text(source)) } @@ -1030,7 +1063,7 @@ fn test_interpolation_preserves_user_context_assignment() { let mut user_ctx = InterpolationContext::default(); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); - let node = yeast::tree!((identifier #{ + let node = tree!((identifier #{ *ctx = InterpolationContext { counter: 1 }; ctx.counter })); @@ -1049,7 +1082,7 @@ fn test_interpolation_accepts_borrowed_build_ctx() { fn build_identifier( ctx: &mut yeast::build::BuildCtx<'_, InterpolationContext>, ) -> yeast::OutputId { - yeast::tree!((identifier #{ + tree!((identifier #{ ctx.counter += 1; ctx.counter })) @@ -1083,11 +1116,11 @@ fn test_interpolation_accepts_value_borrowed_from_context() { let mut ast = empty_output_ast(input); let mut user_ctx = BorrowedInterpolationContext::default(); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); - let location = yeast::tree!((identifier "location")); + let location = tree!((identifier "location")); ctx.locations.push(location); let locations = ctx.locations.iter(); - yeast::tree!((call location: {locations})); + tree!((call location: {locations})); } #[test] @@ -1098,10 +1131,10 @@ fn test_location_interpolation_materializes_context_iterator() { let mut user_ctx = BorrowedInterpolationContext::default(); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); let location = - yeast::tree!((identifier "location" location: {input_ast.get_root()})); + tree!((identifier "location" location: {input_ast.get_root()})); ctx.locations.push(location); - let call = yeast::tree!((call location: {ctx.locations.drain(..)})); + let call = tree!((call location: {ctx.locations.drain(..)})); assert_eq!(ctx.locations, vec![location]); assert_eq!(ctx.output.get_node(call).unwrap().byte_range(), 0..5); @@ -1117,11 +1150,11 @@ fn test_root_choice_returns_an_optional_node() { let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); let absent: Option = None; - let missing = yeast::tree!([(identifier #{absent})]); + let missing = tree!([(identifier #{absent})]); assert_eq!(missing, None); assert_eq!(ctx.output.nodes().len(), node_count); - let root = yeast::tree!([ + let root = tree!([ (identifier #{absent}) (identifier "fallback") ]) @@ -1134,7 +1167,7 @@ fn test_root_choice_returns_an_optional_node() { ); let source = ctx.ast().get_root(); - let located = yeast::tree!([ + let located = tree!([ (identifier #{absent}) (identifier "located fallback" location: {source}) ]) @@ -1155,7 +1188,7 @@ fn test_choice_preflights_dependencies_before_constructing() { let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); let absent: Option = None; - let root = yeast::tree!( + let root = tree!( (assignment left: [ (left_assignment_list @@ -1188,8 +1221,8 @@ fn test_choice_propagates_through_nested_nodes() { // The absent value sits two levels below the choice, so the whole // `left_assignment_list` subtree is abandoned along with it. let absent: Option = None; - let right = yeast::tree!((integer "1")); - let root = yeast::tree!( + let right = tree!((integer "1")); + let root = tree!( (assignment left: [(left_assignment_list child: (identifier #{absent}))] right: {right} @@ -1216,7 +1249,7 @@ fn test_innermost_choice_catches_first() { // The inner choice catches, so only `child` is dropped; `left` survives. let absent: Option = None; - let root = yeast::tree!( + let root = tree!( (assignment left: [ (left_assignment_list child: [(identifier #{absent})]) @@ -1252,7 +1285,7 @@ fn ruby_rules() -> Vec { right: {right} ) {left.iter().enumerate().map(|(i, &lhs)| - yeast::tree!( + tree!( (assignment left: {lhs} right: (element_reference From eff504ae353edd4b59536a51f5ec335043ce5300 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:58:20 +0000 Subject: [PATCH 5/8] unified: Use schema-bound YEAST builders Declare the shared AST schema once and import its `tree!` wrapper for Swift lowering helpers and rule bodies. --- unified/extractor/src/languages/swift/swift.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index ac2fa2173f1e..96b1a57b5cf8 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -1,5 +1,12 @@ use codeql_extractor::extractor::desugaring; -use yeast::{tree, ConcreteDesugarer, DesugaringConfig, Rule}; +use yeast::{ConcreteDesugarer, DesugaringConfig, Rule}; + +yeast::tree_schema! { + swift_tree, + "ast_types.yml", +} + +use swift_tree::tree; /// User context propagated from outer rules down to the inner rules that /// emit the corresponding output declarations, so that each emitted node From dd101116c7925bef01d3be17d2a2be63dffe4c7c Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:58:23 +0000 Subject: [PATCH 6/8] yeast: Document schema-bound builders --- shared/yeast/doc/yeast.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 3a67f3041c43..8373eb616e04 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -184,6 +184,20 @@ Templates construct new AST nodes using the `tree!` and `trees!` macros. All children in a template must be in named fields — output AST nodes are always fully fielded. +Declare the output schema and import its builders: + +```rust +yeast::tree_schema! { + swift_tree, + "ast_types.yml", +} + +use swift_tree::{tree, trees, SCHEMA as OUTPUT_SCHEMA}; +``` + +The generated namespace exposes the schema contents as `SCHEMA` for runtime +configuration. + The macros use a `BuildCtx` binding named `ctx` from the surrounding scope. `rule!` provides this binding automatically; standalone uses must create it: @@ -198,7 +212,7 @@ yeast::rule!( // Standalone — create a binding named ctx let mut user_ctx = (); let mut ctx = BuildCtx::new(ast, &captures, &mut user_ctx); -let id = yeast::tree!( +let id = tree!( (assignment left: {ctx.capture("lhs")} right: {ctx.capture("rhs")} @@ -211,7 +225,7 @@ let id = yeast::tree!( `tree!(...)` returns a single node `Id`: ```rust -yeast::tree!( +tree!( (assignment left: {ctx.capture("lhs")} right: {ctx.capture("rhs")} @@ -224,7 +238,7 @@ yeast::tree!( `trees!(...)` returns `Vec`: ```rust -yeast::trees!( +trees!( (assignment left: {tmp} right: {right}) {body} ) @@ -420,7 +434,7 @@ options uniformly: right: {rhs} // a captured value (inside rule!) ) -yeast::trees!( +trees!( (assignment left: {tmp} right: {right}) {extra_nodes} // splices a Vec ) From 65b401ac418a5d9b3d5e3870e3fd96811fcd9e4e Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 12:59:48 +0000 Subject: [PATCH 7/8] yeast: Remove schema-less tree builders I think in general we'll be happier if all uses of `tree!`/`trees!` have an (implicitly) associated schema. Thus, we don't need the generic schema-less versions anymore. --- shared/yeast-macros/src/lib.rs | 33 -------------------------------- shared/yeast-macros/src/parse.rs | 11 ----------- shared/yeast/src/lib.rs | 2 +- 3 files changed, 1 insertion(+), 45 deletions(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index c76bd881af38..3b1dc0aea57d 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -104,20 +104,6 @@ pub fn query(input: TokenStream) -> TokenStream { /// top-level choice passed to `tree!` returns `Option`; ordinary /// templates continue to return `OutputId`. /// -/// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: -/// -/// ```text -/// tree!((kind ...)) -/// ``` -#[proc_macro] -pub fn tree(input: TokenStream) -> TokenStream { - let input2: TokenStream2 = input.into(); - match parse::parse_tree_top(input2) { - Ok(output) => output.into(), - Err(err) => err.to_compile_error().into(), - } -} - #[doc(hidden)] #[proc_macro] pub fn tree_with_schema(input: TokenStream) -> TokenStream { @@ -128,25 +114,6 @@ pub fn tree_with_schema(input: TokenStream) -> TokenStream { } } -/// Build a list of AST nodes from a template, returning `Vec`. -/// -/// Like `tree!` but returns `Vec` and supports multiple top-level -/// elements. All syntax from `tree!` is available. -/// -/// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: -/// -/// ```text -/// trees!((node1 ...) (node2 ...)) -/// ``` -#[proc_macro] -pub fn trees(input: TokenStream) -> TokenStream { - let input2: TokenStream2 = input.into(); - match parse::parse_trees_top(input2) { - Ok(output) => output.into(), - Err(err) => err.to_compile_error().into(), - } -} - #[doc(hidden)] #[proc_macro] pub fn trees_with_schema(input: TokenStream) -> TokenStream { diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 4d80202da647..05677a10a49c 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -13,12 +13,6 @@ pub fn parse_query_top(input: TokenStream) -> Result { }) } -pub fn parse_tree_top(input: TokenStream) -> Result { - let template = crate::template_parse::parse_template(input)?; - let template = template.lower_root(); - Ok(quote!({ #template })) -} - pub fn parse_tree_with_schema_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); let output_path = parse_named_string_arg(&mut tokens, "output")?; @@ -37,11 +31,6 @@ pub fn parse_tree_with_schema_top(input: TokenStream) -> Result { })) } -pub fn parse_trees_top(input: TokenStream) -> Result { - let templates = crate::template_parse::parse_templates(input)?; - Ok(crate::ast::Template::lower_list(&templates)) -} - pub fn parse_trees_with_schema_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); let output_path = parse_named_string_arg(&mut tokens, "output")?; diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 01b7241fcdc0..c8caa20ef0f6 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -19,7 +19,7 @@ pub use range::{Point, Range}; pub use yeast_macros::tree_with_schema as __tree_with_schema; #[doc(hidden)] pub use yeast_macros::trees_with_schema as __trees_with_schema; -pub use yeast_macros::{query, rule, rules, tree, tree_schema, trees}; +pub use yeast_macros::{query, rule, rules, tree_schema}; use captures::{Captures, TranslatedCaptures}; use query::QueryNode; From a9329b0c860494670017c8d530c4a0d8d59c3f0b Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 12 Nov 2026 13:56:10 +0000 Subject: [PATCH 8/8] yeast: Add explicit-context tree builders Provide `tree_with_context!` and `trees_with_context!` for macro-generated templates whose forwarded tokens cannot reliably identify the local `ctx` binding. This is essentially only needed if you're doing really peculiar stuff with macros, but Copilot insisted, and it seemed harmless to add this. --- shared/yeast-macros/src/lib.rs | 9 +++-- shared/yeast-macros/src/parse.rs | 58 +++++++++++++++++++++++++++----- shared/yeast/doc/yeast.md | 19 ++++++++++- shared/yeast/tests/test.rs | 53 ++++++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 13 deletions(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 3b1dc0aea57d..682e60eda2b6 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -124,8 +124,9 @@ pub fn trees_with_schema(input: TokenStream) -> TokenStream { } } -/// Declare an output-schema namespace with importable `tree!` and `trees!` -/// wrappers and the schema contents. +/// Declare an output-schema namespace with importable `tree!`, `trees!`, +/// `tree_with_context!`, and `trees_with_context!` wrappers and the schema +/// contents. /// /// ```ignore /// yeast::tree_schema! { @@ -133,7 +134,9 @@ pub fn trees_with_schema(input: TokenStream) -> TokenStream { /// "ast_types.yml", /// } /// -/// use swift_tree::{tree, trees, SCHEMA}; +/// use swift_tree::{ +/// tree, tree_with_context, trees, trees_with_context, SCHEMA, +/// }; /// ``` #[proc_macro] pub fn tree_schema(input: TokenStream) -> TokenStream { diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 05677a10a49c..37b9ed156a17 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -17,11 +17,14 @@ pub fn parse_tree_with_schema_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); let output_path = parse_named_string_arg(&mut tokens, "output")?; expect_punct(&mut tokens, ',', "expected `,` after output schema path")?; + let explicit_ctx = parse_context_arg(&mut tokens)?; let template_tokens: TokenStream = tokens.collect(); - let ctx = Ident::new( - "ctx", - first_leaf_span(template_tokens.clone()).unwrap_or_else(Span::call_site), - ); + let ctx = explicit_ctx.unwrap_or_else(|| { + Ident::new( + "ctx", + first_leaf_span(template_tokens.clone()).unwrap_or_else(Span::call_site), + ) + }); let template = crate::template_parse::parse_template(template_tokens)?; let template = template.lower_root(&ctx); let output_lit = proc_macro2::Literal::string(&output_path.value); @@ -35,11 +38,14 @@ pub fn parse_trees_with_schema_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); let output_path = parse_named_string_arg(&mut tokens, "output")?; expect_punct(&mut tokens, ',', "expected `,` after output schema path")?; + let explicit_ctx = parse_context_arg(&mut tokens)?; let template_tokens: TokenStream = tokens.collect(); - let ctx = Ident::new( - "ctx", - first_leaf_span(template_tokens.clone()).unwrap_or_else(Span::call_site), - ); + let ctx = explicit_ctx.unwrap_or_else(|| { + Ident::new( + "ctx", + first_leaf_span(template_tokens.clone()).unwrap_or_else(Span::call_site), + ) + }); let templates = crate::template_parse::parse_templates(template_tokens)?; let templates = crate::ast::Template::lower_list(&templates, &ctx); let output_lit = proc_macro2::Literal::string(&output_path.value); @@ -81,6 +87,17 @@ pub fn parse_tree_schema_top(input: TokenStream) -> Result { } pub(crate) use tree; + macro_rules! tree_with_context { + ($ctx:ident, $($tokens:tt)*) => { + yeast::__tree_with_schema! { + output: #output_lit, + context: $ctx, + $($tokens)* + } + }; + } + pub(crate) use tree_with_context; + macro_rules! trees { ($($tokens:tt)*) => { yeast::__trees_with_schema! { @@ -90,6 +107,17 @@ pub fn parse_tree_schema_top(input: TokenStream) -> Result { }; } pub(crate) use trees; + + macro_rules! trees_with_context { + ($ctx:ident, $($tokens:tt)*) => { + yeast::__trees_with_schema! { + output: #output_lit, + context: $ctx, + $($tokens)* + } + }; + } + pub(crate) use trees_with_context; } }) } @@ -205,6 +233,20 @@ fn first_leaf_span(stream: TokenStream) -> Option { }) } +fn parse_context_arg(tokens: &mut Tokens) -> Result> { + let Some(TokenTree::Ident(name)) = tokens.peek() else { + return Ok(None); + }; + if name != "context" { + return Ok(None); + } + tokens.next(); + expect_punct(tokens, ':', "expected `:` after `context`")?; + let ctx = expect_ident(tokens, "expected context identifier")?; + expect_punct(tokens, ',', "expected `,` after context identifier")?; + Ok(Some(ctx)) +} + fn parse_named_string_arg( tokens: &mut std::iter::Peekable, expected_name: &str, diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 8373eb616e04..306fe55a187a 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -192,12 +192,29 @@ yeast::tree_schema! { "ast_types.yml", } -use swift_tree::{tree, trees, SCHEMA as OUTPUT_SCHEMA}; +use swift_tree::{ + tree, tree_with_context, trees, trees_with_context, SCHEMA as OUTPUT_SCHEMA, +}; ``` The generated namespace exposes the schema contents as `SCHEMA` for runtime configuration. +The default builders infer the hygiene of the ambient `ctx` binding from the +template tokens. A macro that forwards any of those tokens must use +`tree_with_context!` or `trees_with_context!` to pass its local context +explicitly. Otherwise the generated code may fail to resolve `ctx`, or silently +use an unrelated `ctx` from the scope where the forwarded tokens originated: + +```rust +macro_rules! build_kind { + ($kind:ident, $context:expr) => {{ + let mut ctx = $context; + tree_with_context!(ctx, ($kind "generated")) + }}; +} +``` + The macros use a `BuildCtx` binding named `ctx` from the surrounding scope. `rule!` provides this binding automatically; standalone uses must create it: diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 4e9d89150c87..d8b7624bbdf5 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -12,7 +12,28 @@ yeast::tree_schema! { "tests/node-types.yml", } -use test_tree::{tree, trees}; +use test_tree::{tree, tree_with_context, trees, trees_with_context}; + +macro_rules! build_tree_with_kind { + ($kind:ident, $context:expr) => {{ + let mut ctx = $context; + tree_with_context!(ctx, ($kind "x")) + }}; +} + +macro_rules! build_trees_with_kind { + ($kind:ident, $context:expr) => {{ + let mut ctx = $context; + trees_with_context!(ctx, ($kind "x")) + }}; +} + +macro_rules! build_tree_from_template { + ($template:tt, $context:expr) => {{ + let mut ctx = $context; + tree_with_context!(ctx, $template) + }}; +} /// Helper: parse Ruby source with no rules, return dump. fn parse_and_dump(input: &str) -> String { @@ -825,6 +846,36 @@ fn test_tree_schema_namespace() { ); } +#[test] +fn test_explicit_context_builders_support_forwarded_templates() { + let input_ast = parse_input("x = 1"); + + let mut output = empty_output_ast("x = 1"); + let mut user_ctx = (); + let node = build_tree_with_kind!( + identifier, + yeast::build::BuildCtx::new(&input_ast, &mut output, &mut user_ctx) + ); + assert_eq!(output.get_node(node).unwrap().kind_name(), "identifier"); + + let mut output = empty_output_ast("x = 1"); + let mut user_ctx = (); + let nodes = build_trees_with_kind!( + identifier, + yeast::build::BuildCtx::new(&input_ast, &mut output, &mut user_ctx) + ); + assert_eq!(nodes.len(), 1); + assert_eq!(output.get_node(nodes[0]).unwrap().kind_name(), "identifier"); + + let mut output = empty_output_ast("x = 1"); + let mut user_ctx = (); + let node = build_tree_from_template!( + (identifier "x"), + yeast::build::BuildCtx::new(&input_ast, &mut output, &mut user_ctx) + ); + assert_eq!(output.get_node(node).unwrap().kind_name(), "identifier"); +} + /// Builds `(assignment left: … right: [(integer #{value})])`, where a `None` /// `value` leaves `right` unset rather than producing an `integer` with no /// content.