From 98ed7c62afb118c181008c1ec6903a57489c1ff0 Mon Sep 17 00:00:00 2001 From: Priyanshu Dangare Date: Mon, 23 Mar 2026 19:26:18 +0530 Subject: [PATCH 1/4] feat: add STRINGIFY() builtin macro STRINGIFY(expr) turns any macro argument into a string token containing the space-joined text of the argument tokens. Modelled after substitute_concat; handles nested parentheses. Example: %define X 42 say STRINGIFY(X + 1); // => "X + 1" --- src/pre_processor.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/pre_processor.rs b/src/pre_processor.rs index e73f0290..b94194b5 100644 --- a/src/pre_processor.rs +++ b/src/pre_processor.rs @@ -73,6 +73,10 @@ impl<'a> PreProcessor<'a, '_> { dirty = true; continue; } + if self.substitute_stringify(span)? { + dirty = true; + continue; + } *self.i += 1; } if dirty { @@ -333,6 +337,65 @@ impl<'a> PreProcessor<'a, '_> { Ok(true) } + fn substitute_stringify(&mut self, span: &mut Span) -> Result { + let Token::Name(macro_name) = get_token(&self.tokens[*self.i]) else { + return Ok(false); + }; + let macro_name_span = get_span(&self.tokens[*self.i]); + if macro_name != "STRINGIFY" { + return Ok(false); + } + if self + .tokens + .get(*self.i + 1) + .is_none_or(|token| get_token(token) != &Token::LParen) + { + return Ok(false); + } + // Remove STRINGIFY and '(' + self.remove_token(span); + self.remove_token(span); + self.expect_no_eof()?; + // Collect all tokens until the matching ')' + let mut parts: Vec = vec![]; + let mut parens: i32 = 0; + loop { + let token = get_token(&self.tokens[*self.i]).clone(); + match &token { + Token::LParen => { + parens += 1; + parts.push(token.to_string()); + self.remove_token(span); + } + Token::RParen => { + if parens == 0 { + self.remove_token(span); + break; + } + parens -= 1; + parts.push(token.to_string()); + self.remove_token(span); + } + _ => { + parts.push(token.to_string()); + self.remove_token(span); + } + } + self.expect_no_eof()?; + } + let stringified: SmolStr = parts.join(" ").into(); + self.tokens.insert( + *self.i, + ( + macro_name_span.start, + Token::Str(stringified), + macro_name_span.end, + ), + ); + span.end += 1; + Ok(true) + } + fn substitute_concat(&mut self, span: &mut Span) -> Result { let Token::Name(macro_name) = get_token(&self.tokens[*self.i]) else { return Ok(false); From dff1670c21fcb448c8b5e1fe10f8bb2323985bdc Mon Sep 17 00:00:00 2001 From: Priyanshu Dangare Date: Tue, 24 Mar 2026 03:44:45 +0530 Subject: [PATCH 2/4] docs: document STRINGIFY built-in macro --- docs/language/macros.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/language/macros.md b/docs/language/macros.md index 0a2c180a..20e73b30 100644 --- a/docs/language/macros.md +++ b/docs/language/macros.md @@ -111,3 +111,29 @@ all overloads for that name at once. ```goboscript CONCAT(prefix, suffix) # becomes prefixsuffix ``` + +## Stringify Tokens + +`STRINGIFY` is a built-in macro that converts its argument tokens into a string literal. +All tokens inside the parentheses are joined with spaces and produced as a single string value. + +```goboscript +STRINGIFY(hello world) # becomes "hello world" +``` + +This is useful when you need to turn a macro expansion or a sequence of tokens into a +string at compile time: + +```goboscript +%define VERSION 1 2 3 + +onflag { + say STRINGIFY(VERSION); # says "1 2 3" +} +``` + +Nested parentheses are supported and are included verbatim in the resulting string: + +```goboscript +STRINGIFY(foo(bar, baz)) # becomes "foo ( bar , baz )" +``` From 000cc8251c19c6759f230201cddb6e82dfbda8ce Mon Sep 17 00:00:00 2001 From: Priyanshu Dangare Date: Tue, 24 Mar 2026 04:10:22 +0530 Subject: [PATCH 3/4] refactor: simplify token to string conversions --- src/lexer/token.rs | 14 +++++++------- src/pre_processor.rs | 29 ++++++++++------------------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/lexer/token.rs b/src/lexer/token.rs index de1967ab..3fee5a0d 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -241,18 +241,18 @@ pub enum Token { impl Display for Token { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Token::Name(name) => write!(f, "name{}", name), + Token::Name(name) => write!(f, "{}", name), Token::Define => write!(f, "%define"), Token::Undef => write!(f, "%undef"), Token::Newline => writeln!(f), Token::Backslash => write!(f, "\\"), Token::Arg(name) => write!(f, "${}", name), - Token::Bin(value) => write!(f, "bin{}", value), - Token::Oct(value) => write!(f, "oct{}", value), - Token::Int(value) => write!(f, "int{}", value), - Token::Hex(value) => write!(f, "hex{}", value), - Token::Float(value) => write!(f, "float{}", value), - Token::Str(value) => write!(f, "str{}", value), + Token::Bin(value) => write!(f, "{}", value), + Token::Oct(value) => write!(f, "{}", value), + Token::Int(value) => write!(f, "{}", value), + Token::Hex(value) => write!(f, "{}", value), + Token::Float(value) => write!(f, "{}", value), + Token::Str(value) => write!(f, "\"{}\"", value), Token::Costumes => write!(f, "costumes"), Token::Sounds => write!(f, "sounds"), Token::Local => write!(f, "local"), diff --git a/src/pre_processor.rs b/src/pre_processor.rs index b94194b5..daf68a7e 100644 --- a/src/pre_processor.rs +++ b/src/pre_processor.rs @@ -361,26 +361,17 @@ impl<'a> PreProcessor<'a, '_> { let mut parens: i32 = 0; loop { let token = get_token(&self.tokens[*self.i]).clone(); - match &token { - Token::LParen => { - parens += 1; - parts.push(token.to_string()); - self.remove_token(span); - } - Token::RParen => { - if parens == 0 { - self.remove_token(span); - break; - } - parens -= 1; - parts.push(token.to_string()); - self.remove_token(span); - } - _ => { - parts.push(token.to_string()); - self.remove_token(span); - } + if token == Token::RParen && parens == 0 { + self.remove_token(span); + break; } + match token { + Token::LParen => parens += 1, + Token::RParen => parens -= 1, + _ => {} + } + parts.push(token.to_string()); + self.remove_token(span); self.expect_no_eof()?; } let stringified: SmolStr = parts.join(" ").into(); From ab3872a025af39da43ae188574dbd15fe4bc1858 Mon Sep 17 00:00:00 2001 From: Priyanshu Dangare Date: Tue, 26 May 2026 23:23:11 +0530 Subject: [PATCH 4/4] fix: concat and stringify --- src/lexer/literal.rs | 4 +- src/pre_processor.rs | 390 ++++++++++++++++++++++++++++++++++--------- src/wasm.rs | 6 +- 3 files changed, 317 insertions(+), 83 deletions(-) diff --git a/src/lexer/literal.rs b/src/lexer/literal.rs index d2244eb4..a6fa729e 100644 --- a/src/lexer/literal.rs +++ b/src/lexer/literal.rs @@ -8,7 +8,9 @@ pub fn name(lex: &mut Lexer) -> SmolStr { } pub fn string(lex: &mut Lexer) -> Option { - serde_json::from_str::(lex.slice()).ok().map(SmolStr::from) + serde_json::from_str::(lex.slice()) + .ok() + .map(SmolStr::from) } pub fn arg(lex: &mut Lexer) -> SmolStr { diff --git a/src/pre_processor.rs b/src/pre_processor.rs index eb6e1bf4..54877ea1 100644 --- a/src/pre_processor.rs +++ b/src/pre_processor.rs @@ -70,7 +70,7 @@ impl<'a> PreProcessor<'a, '_> { if self.substitute_function_define(span, suppress)? { continue; } - if self.substitute_concat(span)? { + if self.substitute_concat(span, suppress)? { dirty = true; continue; } @@ -97,9 +97,14 @@ impl<'a> PreProcessor<'a, '_> { fn expect_no_eof(&self) -> Result<(), Diagnostic> { if *self.i >= self.tokens.len() { + let span = if *self.i > 0 { + get_span(&self.tokens[*self.i - 1]) + } else { + 0..0 + }; return Err(Diagnostic { kind: DiagnosticKind::UnrecognizedEof(vec![]), - span: get_span(&self.tokens[*self.i - 1]), + span, }); } Ok(()) @@ -208,6 +213,11 @@ impl<'a> PreProcessor<'a, '_> { if suppress.contains(&*macro_name) { return Ok(false); } + if simple_define_body.is_empty() { + self.tokens.remove(*self.i); + span.end = span.end.saturating_sub(1); + return Ok(true); + } self.tokens.splice( *self.i..(*self.i + 1), simple_define_body @@ -240,48 +250,15 @@ impl<'a> PreProcessor<'a, '_> { if suppress.contains(&*macro_name) { return Ok(false); } - if get_token(&self.tokens[*self.i + 1]) != &Token::LParen { + if self + .tokens + .get(*self.i + 1) + .is_none_or(|token| get_token(token) != &Token::LParen) + { return Ok(false); } - // Parse the call arguments first so we know the arity before looking up. - self.remove_token(span); // remove macro name - self.remove_token(span); // remove '(' - self.expect_no_eof()?; - let mut token = get_token(&self.tokens[*self.i]).clone(); - let mut args: Vec> = vec![]; - let mut arg: Vec = vec![]; - if token != Token::RParen { - let mut parens = 0; - while parens >= 0 { - match token { - Token::LParen => { - parens += 1; - arg.push(token); - } - Token::RParen => { - parens -= 1; - if parens < 0 { - args.push(arg); - arg = vec![]; - } else { - arg.push(token); - } - } - Token::Comma if parens == 0 => { - args.push(arg); - arg = vec![]; - } - _ => { - arg.push(token); - } - } - self.remove_token(span); - token = get_token(&self.tokens[*self.i]).clone(); - } - } else { - self.remove_token(span); - } + let (_, args) = self.parse_macro_call_args(span)?; let arity = args.len(); let Some((function_define_params, function_define_body)) = overloads.get(&arity).cloned() @@ -388,56 +365,309 @@ impl<'a> PreProcessor<'a, '_> { Ok(true) } - fn substitute_concat(&mut self, span: &mut Span) -> Result { + fn parse_macro_call_args( + &mut self, + span: &mut Span, + ) -> Result<(Span, Vec>), Diagnostic> { + self.remove_token(span); + + if get_token(&self.tokens[*self.i]) != &Token::LParen { + return Err(Diagnostic { + kind: DiagnosticKind::UnrecognizedToken( + get_token(&self.tokens[*self.i]).clone(), + vec!["(".to_string()], + ), + span: get_span(&self.tokens[*self.i]), + }); + } + + self.remove_token(span); + self.expect_no_eof()?; + + let args_start = *self.i; + + let mut token = get_token(&self.tokens[*self.i]).clone(); + + let mut args: Vec> = vec![]; + let mut arg: Vec = vec![]; + + if token != Token::RParen { + let mut parens = 0; + + while parens >= 0 { + match token { + Token::LParen => { + parens += 1; + arg.push(token.clone()); + } + + Token::RParen => { + parens -= 1; + + if parens < 0 { + args.push(arg); + arg = vec![]; + } else { + arg.push(token.clone()); + } + } + + Token::Comma if parens == 0 => { + args.push(arg); + arg = vec![]; + } + + _ => { + arg.push(token.clone()); + } + } + + self.remove_token(span); + if parens >= 0 { + self.expect_no_eof()?; + token = get_token(&self.tokens[*self.i]).clone(); + } + } + } else { + self.remove_token(span); + } + + let args_end = *self.i; + + Ok((args_start..args_end, args)) + } + + fn expand_token_list( + &mut self, + tokens: Vec, + suppress: &FxHashSet, + ) -> Result, Diagnostic> { + let mut spanned: Vec = + tokens.into_iter().map(|token| (0, token, 0)).collect(); + + let length = spanned.len(); + + if length > 0 { + let mut i = 0; + + let mut pre_processor = PreProcessor { + tokens: &mut spanned, + i: &mut i, + function_defines: self.function_defines.clone(), + simple_defines: self.simple_defines.clone(), + }; + + pre_processor.process(&mut (0..length), suppress)?; + pre_processor.remove_marker_tokens(); + } + + Ok(spanned.into_iter().map(|(_, token, _)| token).collect()) + } + + fn peek_macro_call_args(&self) -> Result>>, Diagnostic> { + let mut i = *self.i + 1; + if i >= self.tokens.len() || get_token(&self.tokens[i]) != &Token::LParen { + return Ok(None); + } + i += 1; + + if i >= self.tokens.len() { + return Err(Diagnostic { + kind: DiagnosticKind::UnrecognizedEof(vec![]), + span: get_span(&self.tokens[i - 1]), + }); + } + + let mut args: Vec> = vec![]; + let mut arg: Vec = vec![]; + let mut token = get_token(&self.tokens[i]).clone(); + + if token != Token::RParen { + let mut parens = 0; + + while parens >= 0 { + match token { + Token::LParen => { + parens += 1; + arg.push(token.clone()); + } + + Token::RParen => { + parens -= 1; + + if parens < 0 { + args.push(arg); + arg = vec![]; + } else { + arg.push(token.clone()); + } + } + + Token::Comma if parens == 0 => { + args.push(arg); + arg = vec![]; + } + + _ => { + arg.push(token.clone()); + } + } + + i += 1; + if parens >= 0 { + if i >= self.tokens.len() { + return Err(Diagnostic { + kind: DiagnosticKind::UnrecognizedEof(vec![]), + span: get_span(&self.tokens[i - 1]), + }); + } + token = get_token(&self.tokens[i]).clone(); + } + } + } + + Ok(Some(args)) + } + + fn concat_tokens(&self, left: &Token, right: &Token, span: Span) -> Result { + let pasted = format!("{left}{right}"); + let mut lexer = crate::lexer::adaptor::Lexer::new(&pasted); + + let Some(first) = lexer.next() else { + return Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span, + }); + }; + + let Ok((_, token, end)) = first else { + return Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span, + }); + }; + + if end != pasted.len() { + return Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span, + }); + } + + match lexer.next() { + None => Ok(token), + Some(Ok(_)) => Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span, + }), + Some(Err(_)) => Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span, + }), + } + } + + fn substitute_concat( + &mut self, + span: &mut Span, + suppress: &FxHashSet, + ) -> Result { let Token::Name(macro_name) = get_token(&self.tokens[*self.i]) else { return Ok(false); }; - let macro_name_span = get_span(&self.tokens[*self.i]); + if macro_name != "CONCAT" { return Ok(false); } - if self - .tokens - .get(*self.i + 1) - .is_none_or(|token| get_token(token) != &Token::LParen) - { - return Ok(false); - } - if self - .tokens - .get(*self.i + 3) - .is_none_or(|token| get_token(token) != &Token::Comma) - { - return Ok(false); - } - if self - .tokens - .get(*self.i + 5) - .is_none_or(|token| get_token(token) != &Token::RParen) - { + + let macro_name_span = get_span(&self.tokens[*self.i]); + + let Some(args) = self.peek_macro_call_args()? else { return Ok(false); + }; + + if args.len() != 2 { + return Err(Diagnostic { + kind: DiagnosticKind::MacroArgsCountMismatch { + expected: 2, + given: args.len(), + }, + span: macro_name_span, + }); } - let Some(Token::Name(left)) = self.tokens.get(*self.i + 2).map(get_token).cloned() else { - return Ok(false); + + let left = self.expand_token_list(args[0].clone(), suppress)?; + let right = self.expand_token_list(args[1].clone(), suppress)?; + + let [left] = &left[..] else { + return Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span: macro_name_span, + }); }; - let Some(Token::Name(right)) = self.tokens.get(*self.i + 4).map(get_token).cloned() else { - return Ok(false); + + let [right] = &right[..] else { + return Err(Diagnostic { + kind: DiagnosticKind::InvalidToken, + span: macro_name_span, + }); }; - self.remove_token(span); - self.remove_token(span); - self.remove_token(span); - self.remove_token(span); - self.remove_token(span); - self.remove_token(span); - self.tokens.insert( - *self.i, - ( - macro_name_span.start, - Token::Name(format!("{}{}", left, right).into()), - macro_name_span.end, - ), - ); + + let token = self.concat_tokens(left, right, macro_name_span.clone())?; + + let (_, _) = self.parse_macro_call_args(span)?; + + self.tokens + .insert(*self.i, (macro_name_span.start, token, macro_name_span.end)); + span.end += 1; + Ok(true) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::lexer::adaptor::Lexer; + + fn preprocess(source: &str) -> Result, Diagnostic> { + let mut tokens: Vec<_> = Lexer::new(source).collect::>()?; + PreProcessor::apply(&mut tokens)?; + Ok(tokens.into_iter().map(|(_, token, _)| token).collect()) + } + + #[test] + fn concat_expands_function_macro_arguments_before_pasting() { + let tokens = + preprocess("%define SUFFIX bar\n%define C(x) CONCAT(x, SUFFIX)\nC(foo)\n").unwrap(); + + assert_eq!(tokens, vec![Token::Name("foobar".into())]); + } + + #[test] + fn function_macro_call_at_end_of_expansion_does_not_panic() { + let tokens = preprocess("%define C(x) x\n%define WRAPPED C(foo)\nWRAPPED\n").unwrap(); + + assert_eq!(tokens, vec![Token::Name("foo".into())]); + } + + #[test] + fn concat_reprocesses_the_pasted_token() { + let tokens = preprocess("%define foobar 1\n%define C(x) CONCAT(x, bar)\nC(foo)\n").unwrap(); + + assert_eq!(tokens, vec![Token::Int(1)]); + } + + #[test] + fn concat_errors_without_removing_tokens_for_invalid_pastes() { + let mut tokens: Vec<_> = Lexer::new("%define TWO foo bar\nCONCAT(TWO, baz)\n") + .collect::>() + .unwrap(); + + let result = PreProcessor::apply(&mut tokens); + + assert!(result.is_err()); + assert!(!tokens.is_empty()); + } +} diff --git a/src/wasm.rs b/src/wasm.rs index 16c908b5..dd8e8ea2 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -17,12 +17,14 @@ use wasm_bindgen::{ use crate::{ ast::Sprite, - codegen::sb3::Sb3, + codegen::{ + build::build_impl, + sb3::Sb3, + }, diagnostic::{ Artifact, Diagnostic, }, - codegen::build::build_impl, misc::base64, standard_library::StandardLibrary, vfs::MemFS,