diff --git a/docs/issues/162-plan.md b/docs/issues/162-plan.md new file mode 100644 index 0000000..a28fddd --- /dev/null +++ b/docs/issues/162-plan.md @@ -0,0 +1,36 @@ +# Issue 162: correct existing syntax highlighting + +## Goal + +Restore ordinary TypeScript/TSX syntax categories and keep Markdown formatting outside code blocks. +Do not change text editing, add grammars, or alter the highlighter cache policy. + +## Evidence and approach + +The new TypeScript regression fails on `const` before the fix because only the TypeScript supplemental query is configured. +Combine JavaScript captures with TypeScript captures and add JSX captures for TSX. +Tree-sitter Highlight 0.26.11 gives later matching patterns precedence on the same node, so the base query comes first and the language-specific queries follow it. +Keep JavaScript injection support for template literals. + +The new Markdown regressions fail because a second inline pass interprets fenced and indented source as prose. +Removing that pass alone loses valid inline markup: the upstream Markdown injection query excludes child nodes by default. +Use Markdown injection queries that include child content for inline, pipe-table-cell, and fenced-code nodes. +Keep the other existing HTML and frontmatter injection rules. +The block parser now supplies the code/prose boundary and multiline inline context; remove the redundant line-based parser and its unused helpers. + +## Acceptance and proof + +- Assert exact TypeScript keyword, type, string, comment, function, and number categories. +- Assert TSX keyword, type, tag, attribute, string, and property categories. +- Check backtick, tilde, and quoted Rust fences retain string styles when source contains Markdown-looking emphasis. +- Check unlabelled, unsupported-language, and indented code do not acquire inline Markdown formatting. +- Preserve multiline emphasis and existing prose links, headings, inline code, and table-cell formatting. +- Verify the actual retained renderer cell uses the Rust string style inside a Markdown fence. +- Run all highlighter and renderer tests, formatting, Clippy, and the complete suite. +- Run a release-build terminal smoke with temporary TS, TSX, and Markdown fixtures, checking emitted colours and shell restoration. + +## Delivery + +Update roadmap state for the already merged direction task and superseded layout tickets. +Open one PR for #162, obtain independent review, pass CI, and merge under the owner's batch authorization. +Performance redesign and additional language support remain separate tasks. diff --git a/docs/roadmap.md b/docs/roadmap.md index e79caaf..2e4ba5b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -134,7 +134,7 @@ Parent: [#160 Deliver a minimalist daily coding editor](https://github.com/owain This extends the v0.3 editing work; individual ticket criteria define completion. Delivery order: -- [open] [#161 Align the product direction](https://github.com/owainlewis/cortex/issues/161) +- [closed] [#161 Align the product direction](https://github.com/owainlewis/cortex/issues/161) - [open] [#162 Correct TypeScript and fenced Markdown colours](https://github.com/owainlewis/cortex/issues/162) - [open] [#46 Add command registry and M-x](https://github.com/owainlewis/cortex/issues/46) - [open] [#164 Add typing and region indentation](https://github.com/owainlewis/cortex/issues/164) @@ -159,7 +159,7 @@ Publishing a release is separate from this implementation batch. Manual reload, the dirty reload guard, and the disk-changed indicator are implemented through [#48](https://github.com/owainlewis/cortex/issues/48). Idle disk-change notification and a verified tmux workflow are included in #172. Internal split layouts (#31), tabs (#32), and an embedded terminal pane (#49) are superseded by the external-terminal direction in #161. -Their original tickets retain the design history and will be closed as not planned after that decision merges. +Those tickets are closed as not planned and retain their original design history. Revisit internal views only for a demonstrated need to show the same unsaved buffer in two places. ## Release and Install diff --git a/src/highlighter.rs b/src/highlighter.rs index d036c99..868c584 100644 --- a/src/highlighter.rs +++ b/src/highlighter.rs @@ -1143,10 +1143,6 @@ impl SyntaxHighlighter { } } - if is_markdown_path(path) { - self.highlight_markdown_inline_lines(&lines, &mut highlighted_lines); - } - highlighted_lines } @@ -1156,66 +1152,6 @@ impl SyntaxHighlighter { .iter() .position(|language| language.extensions.contains(&extension.as_str())) } - - fn language_idx_for_name(&self, name: &str) -> Option { - self.languages - .iter() - .position(|language| language.config.language_name == name) - } - - fn highlight_markdown_inline_lines( - &mut self, - lines: &[String], - highlighted_lines: &mut [Vec], - ) { - let Some(language_idx) = self.language_idx_for_name("markdown_inline") else { - return; - }; - - let languages = &self.languages; - let language = &languages[language_idx]; - - for (line_idx, line) in lines.iter().enumerate() { - if line.is_empty() { - continue; - } - - let events = - self.highlighter - .highlight(&language.config, line.as_bytes(), None, |name| { - language_config_for_name(languages, name) - }); - let Ok(events) = events else { - continue; - }; - - let mut highlight_stack = Vec::new(); - for event in events { - let Ok(event) = event else { - break; - }; - - match event { - HighlightEvent::Source { start, end } => { - if start < end { - if let Some(kind) = - highlight_stack.last().copied().and_then(highlight_kind) - { - highlighted_lines[line_idx].push(HighlightSpan { - range: start..end, - kind, - }); - } - } - } - HighlightEvent::HighlightStart(highlight) => highlight_stack.push(highlight.0), - HighlightEvent::HighlightEnd => { - highlight_stack.pop(); - } - } - } - } - } } fn overflowed_rust_comment_lines( @@ -1278,13 +1214,45 @@ fn rust_definition() -> Option { ) } +// Markdown's inline and code-content nodes contain children that belong to the +// injected language. Include them so emphasis and string punctuation are parsed +// in context instead of applying a second inline pass to every source line. +const MARKDOWN_INJECTIONS: &str = r#" +(fenced_code_block + (info_string (language) @injection.language) + (code_fence_content) @injection.content + (#set! injection.include-children)) + +([(inline) (pipe_table_cell)] @injection.content + (#set! injection.language "markdown_inline") + (#set! injection.include-children)) + +((html_block) @injection.content + (#set! injection.language "html")) + +(document + . + (section + . + (thematic_break) + (_) @injection.content + (thematic_break)) + (#set! injection.language "yaml")) + +((minus_metadata) @injection.content + (#set! injection.language "yaml")) + +((plus_metadata) @injection.content + (#set! injection.language "toml")) +"#; + fn markdown_definition() -> Option { language_definition( MARKDOWN_EXTENSIONS, tree_sitter_md::LANGUAGE.into(), "markdown", tree_sitter_md::HIGHLIGHT_QUERY_BLOCK, - tree_sitter_md::INJECTION_QUERY_BLOCK, + MARKDOWN_INJECTIONS, ) } @@ -1345,22 +1313,33 @@ fn javascript_definition() -> Option { } fn typescript_definition() -> Option { + let highlights = format!( + "{}\n{}", + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY + ); language_definition( TYPESCRIPT_EXTENSIONS, tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), "typescript", - tree_sitter_typescript::HIGHLIGHTS_QUERY, - "", + &highlights, + tree_sitter_javascript::INJECTIONS_QUERY, ) } fn typescript_tsx_definition() -> Option { + let highlights = format!( + "{}\n{}\n{}", + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY + ); language_definition( TYPESCRIPT_TSX_EXTENSIONS, tree_sitter_typescript::LANGUAGE_TSX.into(), "tsx", - tree_sitter_typescript::HIGHLIGHTS_QUERY, - "", + &highlights, + tree_sitter_javascript::INJECTIONS_QUERY, ) } @@ -1456,13 +1435,6 @@ fn language_config_for_name<'a>( .map(|language| &language.config) } -fn is_markdown_path(path: &Path) -> bool { - path.extension() - .and_then(|extension| extension.to_str()) - .map(|extension| matches!(extension.to_ascii_lowercase().as_str(), "md" | "markdown")) - .unwrap_or(false) -} - fn document_lines(source: &str) -> Vec { source .split('\n') @@ -1668,6 +1640,130 @@ mod tests { } } + #[test] + fn typescript_highlights_javascript_tokens_and_types() { + let mut highlighter = SyntaxHighlighter::new(); + let source = "const message: string = \"hello\"; // greeting\nfunction answer(): number { return 42; }"; + let highlighted = highlighter.highlight_document(Path::new("app.ts"), source); + for (line, token, kind) in [ + (0, "const", HighlightKind::Keyword), + (0, "string", HighlightKind::Type), + (0, "hello", HighlightKind::String), + (0, "greeting", HighlightKind::Comment), + (1, "function", HighlightKind::Keyword), + (1, "answer", HighlightKind::Function), + (1, "number", HighlightKind::Type), + (1, "return", HighlightKind::Keyword), + (1, "42", HighlightKind::Number), + ] { + let offset = source.lines().nth(line).unwrap().find(token).unwrap(); + assert_eq!( + highlighted[line] + .iter() + .rev() + .find(|span| span.range.contains(&offset)) + .map(|span| span.kind), + Some(kind), + "expected {kind:?} for {token}" + ); + } + } + + #[test] + fn tsx_highlights_jsx_tags_attributes_and_typescript() { + let mut highlighter = SyntaxHighlighter::new(); + let source = + "const Button = (props: Props) => ;"; + let highlighted = highlighter.highlight_document(Path::new("button.tsx"), source); + for (token, kind) in [ + ("const", HighlightKind::Keyword), + ("Props", HighlightKind::Type), + ("button", HighlightKind::Tag), + ("title", HighlightKind::Attribute), + ("save", HighlightKind::String), + ("label", HighlightKind::Property), + ] { + let offset = source.find(token).unwrap(); + assert_eq!( + highlighted[0] + .iter() + .rev() + .find(|span| span.range.contains(&offset)) + .map(|span| span.kind), + Some(kind), + "expected {kind:?} for {token}" + ); + } + } + + #[test] + fn markdown_inline_markup_does_not_override_fenced_source() { + let mut highlighter = SyntaxHighlighter::new(); + for source in [ + "```rust\nlet s = \"**hello**\";\n```\n**outside**", + "~~~rust\nlet s = \"**hello**\";\n~~~\n**outside**", + "> ```rust\n> let s = \"**hello**\";\n> ```\n\n**outside**", + ] { + let highlighted = highlighter.highlight_document(Path::new("notes.md"), source); + let offset = source.lines().nth(1).unwrap().find("hello").unwrap(); + assert_eq!( + highlighted[1] + .iter() + .rev() + .find(|span| span.range.contains(&offset)) + .map(|span| span.kind), + Some(HighlightKind::String), + "fenced source must retain its string style: {source}" + ); + assert!(line_has_kind( + highlighted.last().unwrap(), + HighlightKind::MarkupBold + )); + } + } + + #[test] + fn markdown_inline_emphasis_keeps_multiline_context() { + let mut highlighter = SyntaxHighlighter::new(); + let source = "A **bold\ncontinued** phrase."; + let highlighted = highlighter.highlight_document(Path::new("notes.md"), source); + assert!(line_has_kind(&highlighted[0], HighlightKind::MarkupBold)); + assert!(line_has_kind(&highlighted[1], HighlightKind::MarkupBold)); + } + + #[test] + fn markdown_table_cells_keep_inline_formatting() { + let mut highlighter = SyntaxHighlighter::new(); + let source = "| **Title** | `Code` |\n| --- | --- |\n| *body* | [link](uri) |"; + let highlighted = highlighter.highlight_document(Path::new("notes.md"), source); + assert!(line_has_kind(&highlighted[0], HighlightKind::MarkupBold)); + assert!(line_has_kind(&highlighted[0], HighlightKind::MarkupRaw)); + assert!(line_has_kind(&highlighted[2], HighlightKind::MarkupItalic)); + assert!(line_has_kind(&highlighted[2], HighlightKind::MarkupLink)); + assert!(line_has_kind(&highlighted[2], HighlightKind::MarkupLinkUrl)); + } + + #[test] + fn markdown_code_without_a_grammar_does_not_gain_inline_markup() { + let mut highlighter = SyntaxHighlighter::new(); + for source in [ + "```\n**literal** and [link](uri)\n```", + "```unknown\n**literal** and [link](uri)\n```", + " **literal** and [link](uri)", + ] { + let highlighted = highlighter.highlight_document(Path::new("notes.md"), source); + assert!( + !highlighted.iter().flatten().any(|span| matches!( + span.kind, + HighlightKind::MarkupBold + | HighlightKind::MarkupItalic + | HighlightKind::MarkupLink + )), + "code must not be interpreted as prose: {source}" + ); + } + } + #[test] fn highlights_markdown_document_structure() { let mut highlighter = SyntaxHighlighter::new(); diff --git a/src/renderer.rs b/src/renderer.rs index 9150cee..2c786ab 100644 --- a/src/renderer.rs +++ b/src/renderer.rs @@ -2350,6 +2350,33 @@ mod tests { assert!(output.contains("\x1b[38;2;137;180;250m")); } + #[test] + fn fenced_rust_string_retains_its_rendered_style() { + let code = "let s = \"**hello**\";"; + let buffer = buffer_with_text("notes.md", &format!("```rust\n{code}\n```\n")); + let renderer = super::Renderer::new(); + let size = TerminalSize { cols: 80, rows: 5 }; + renderer + .render( + &mut Vec::new(), + &buffer, + &View::new(), + size, + None, + None, + None, + None, + None, + ) + .unwrap(); + let column = super::editor_gutter_width(&buffer, 80) + code.find("hello").unwrap(); + let frame = renderer.last_frame.borrow(); + assert_eq!( + frame.as_ref().unwrap().cells[80 + column].style, + super::highlight_style(crate::highlighter::HighlightKind::String) + ); + } + #[test] fn render_emits_markdown_document_styles() { let buffer = buffer_with_text(