diff --git a/AGENTS.md b/AGENTS.md index f4c80c7d..c750569a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -517,7 +517,7 @@ describe('functionUnderTest', () => { ```typescript const result = await parseMarkdown(markdownContent, { autoUnwrap: true, // Remove

wrappers from single-paragraph containers - autoClose: true, // Auto-close incomplete syntax; also accepts (markdown) => string + autoClose: 'streaming', // Heal incomplete syntax while streaming; `true` heals every parse, also accepts (markdown) => string unwrap: 'p', // Strip top-level wrapper tags (MDC unwrap); merges paragraphs registerDefaultPlugins: true, // frontmatter, html, alert, task-list, components, attributes; false to disable }) diff --git a/docs/content/3.rendering/2.html.md b/docs/content/3.rendering/2.html.md index 54e90ab7..8f4d8027 100644 --- a/docs/content/3.rendering/2.html.md +++ b/docs/content/3.rendering/2.html.md @@ -80,7 +80,7 @@ This is a **bold** statement with a [link](https://example.com). | [`plugins`](#render-options-plugins) | `ComarkPlugin[]` | `[]` | Array of plugins | | [`components`](#render-options-components) | `Record` | `{}` | Custom component renderers | | [`data`](#render-options-data) | `Record` | `undefined` | Data passed to component renderers | -| `autoClose` | `boolean` | `true` | Close incomplete Markdown and components before parsing | +| `autoClose` | `boolean \| 'streaming' \| fn` | `'streaming'` | Close incomplete Markdown and components before parsing. `'streaming'` only closes on a streaming parse, so a plain render leaves input as written | | `autoUnwrap` | `boolean` | `true` | Remove a single paragraph wrapper inside components | | `linkify` | `boolean` | `true` | Convert URL-like text into links | | `registerDefaultPlugins` | `boolean` | `true` | Register default plugins (`frontmatter`, `html`, `alert`, `task-list`, `components`, `attributes`) | diff --git a/docs/content/3.rendering/3.vue.md b/docs/content/3.rendering/3.vue.md index c8426cd9..83889404 100644 --- a/docs/content/3.rendering/3.vue.md +++ b/docs/content/3.rendering/3.vue.md @@ -347,7 +347,7 @@ import { AppMarkdown } from './markdown' | [`extends`](#code-markdown-code-definemarkdowncomponent-code-extends) | `ReturnType` | `undefined` | Inherit plugins and components from another component | | `name` | `string` | `undefined` | Component name for debugging | | `autoUnwrap` | `boolean` | `true` | Automatically unwrap single block elements | -| `autoClose` | `boolean` | `true` | Auto-close incomplete markdown syntax | +| `autoClose` | `boolean \| 'streaming' \| fn` | `'streaming'` | Auto-close incomplete markdown syntax. Only closes on a streaming parse by default | | `linkify` | `boolean` | `true` | Auto-convert URL-like text into links | | `registerDefaultPlugins` | `boolean` | `true` | Register default plugins (`frontmatter`, `html`, `alert`, `task-list`, `components`, `attributes`) | | [`plugins`](#code-markdown-props-code-plugins) | `ComarkPlugin[]` | `[]` | Array of plugins | @@ -744,7 +744,7 @@ async function askAI(prompt: string) { ``` ::callout{icon="i-lucide-info" color="info"} -`autoClose` is enabled by default: incomplete syntax like `**bold text` is automatically closed on every parse. Disable with `:options="{ autoClose: false }"`. +`autoClose` defaults to `'streaming'`: incomplete syntax like `**bold text` is closed when the `streaming` prop is set, and left as written otherwise. Set `:options="{ autoClose: true }"` to close incomplete syntax on every parse, or `:options="{ autoClose: false }"` to never close it. :: ### Caret diff --git a/docs/content/3.rendering/4.nuxt.md b/docs/content/3.rendering/4.nuxt.md index c86cb635..a9bcc047 100644 --- a/docs/content/3.rendering/4.nuxt.md +++ b/docs/content/3.rendering/4.nuxt.md @@ -456,7 +456,7 @@ async function askAI(prompt: string) { ``` ::callout{icon="i-lucide-info" color="info"} -`autoClose` is enabled by default: incomplete syntax like `**bold text` is automatically closed on every parse. Disable with `:options="{ autoClose: false }"`. +`autoClose` defaults to `'streaming'`: incomplete syntax like `**bold text` is closed when the `streaming` prop is set, and left as written otherwise. Set `:options="{ autoClose: true }"` to close incomplete syntax on every parse, or `:options="{ autoClose: false }"` to never close it. :: ### Caret diff --git a/docs/content/3.rendering/5.react.md b/docs/content/3.rendering/5.react.md index 9ec2deb1..58d133fc 100644 --- a/docs/content/3.rendering/5.react.md +++ b/docs/content/3.rendering/5.react.md @@ -308,7 +308,7 @@ export default function App() { | [`extends`](#code-markdown-code-definemarkdowncomponent-code-extends) | `ReturnType` | `undefined` | Inherit plugins and components from another component | | `name` | `string` | `undefined` | Component name for debugging | | `autoUnwrap` | `boolean` | `true` | Automatically unwrap single block elements | -| `autoClose` | `boolean` | `true` | Auto-close incomplete markdown syntax | +| `autoClose` | `boolean \| 'streaming' \| fn` | `'streaming'` | Auto-close incomplete markdown syntax. Only closes on a streaming parse by default | | `linkify` | `boolean` | `true` | Auto-convert URL-like text into links | | `registerDefaultPlugins` | `boolean` | `true` | Register default plugins (`frontmatter`, `html`, `alert`, `task-list`, `components`, `attributes`) | | [`plugins`](#code-markdown-props-code-plugins) | `ComarkPlugin[]` | `[]` | Array of plugins | @@ -677,7 +677,7 @@ export default function AiChat() { ``` ::callout{icon="i-lucide-info" color="info"} -`autoClose` is enabled by default: incomplete syntax like `**bold text` is automatically closed on every parse. Disable with `options={{ autoClose: false }}`. +`autoClose` defaults to `'streaming'`: incomplete syntax like `**bold text` is closed when the `streaming` prop is set, and left as written otherwise. Set `options={{ autoClose: true }}` to close incomplete syntax on every parse, or `options={{ autoClose: false }}` to never close it. :: ### Caret diff --git a/docs/content/3.rendering/6.svelte.md b/docs/content/3.rendering/6.svelte.md index a09a9102..486ecb40 100644 --- a/docs/content/3.rendering/6.svelte.md +++ b/docs/content/3.rendering/6.svelte.md @@ -573,7 +573,7 @@ Set `streaming` to `true` while content is being received, then `false` when don ``` ::callout{icon="i-lucide-info" color="info"} -`autoClose` is enabled by default: incomplete syntax like `**bold text` is automatically closed on every parse. Disable with `options={{ autoClose: false }}`. +`autoClose` defaults to `'streaming'`: incomplete syntax like `**bold text` is closed when the `streaming` prop is set, and left as written otherwise. Set `options={{ autoClose: true }}` to close incomplete syntax on every parse, or `options={{ autoClose: false }}` to never close it. :: ### Caret diff --git a/docs/content/3.rendering/7.angular.md b/docs/content/3.rendering/7.angular.md index 5ffa3d4e..17d4e434 100644 --- a/docs/content/3.rendering/7.angular.md +++ b/docs/content/3.rendering/7.angular.md @@ -433,7 +433,7 @@ export class AiChatComponent { ``` ::callout{icon="i-lucide-info" color="info"} -`autoClose` is enabled by default: incomplete syntax like `**bold text` is automatically closed on every parse. Disable with `[options]="{ autoClose: false }"`. +`autoClose` defaults to `'streaming'`: incomplete syntax like `**bold text` is closed when the `streaming` prop is set, and left as written otherwise. Set `[options]="{ autoClose: true }"` to close incomplete syntax on every parse, or `[options]="{ autoClose: false }"` to never close it. :: ### Caret diff --git a/docs/content/3.rendering/8.ansi.md b/docs/content/3.rendering/8.ansi.md index 2b63de4c..a9f89b00 100644 --- a/docs/content/3.rendering/8.ansi.md +++ b/docs/content/3.rendering/8.ansi.md @@ -80,7 +80,7 @@ This is a bold statement with a link (https://example.com). | [`data`](#render-options-data) | `Record` | `undefined` | Data passed to component renderers | | `colors` | `boolean` | `true`* | Emit ANSI escape codes | | `width` | `number` | `80` | Terminal width for HR and code block headers | -| `autoClose` | `boolean` | `true` | Close incomplete Markdown and components before parsing | +| `autoClose` | `boolean \| 'streaming' \| fn` | `'streaming'` | Close incomplete Markdown and components before parsing. `'streaming'` only closes on a streaming parse, so a plain render leaves input as written | | `autoUnwrap` | `boolean` | `true` | Remove a single paragraph wrapper inside components | | `linkify` | `boolean` | `true` | Convert URL-like text into links | | `registerDefaultPlugins` | `boolean` | `true` | Register default plugins (`frontmatter`, `html`, `alert`, `task-list`, `components`, `attributes`) | diff --git a/docs/content/5.reference/1.parse.md b/docs/content/5.reference/1.parse.md index c1b4bb6e..9ddcece2 100644 --- a/docs/content/5.reference/1.parse.md +++ b/docs/content/5.reference/1.parse.md @@ -14,7 +14,7 @@ links: variant: soft --- -## `parseMarkdown(source, options?)`{lang="ts"} +## `parseMarkdown(source, options?, parseOptions?)`{lang="ts"} Parses Markdown from a string and returns a complete `MarkdownDocument`. Default plugins add frontmatter, alerts, task lists, HTML, components, and attributes to the standard Markdown parser. @@ -22,6 +22,7 @@ Parses Markdown from a string and returns a complete `MarkdownDocument`. Default - `source` - The Markdown content as a string - `options?` - Parser options including plugins +- `parseOptions?` - Per-call options, `{ streaming?: boolean }`. Pass `{ streaming: true }` for a chunk of a stream so the default `autoClose` heals incomplete syntax. **Returns:** `MarkdownDocument` object containing: @@ -224,7 +225,7 @@ import toc from 'comark/plugins/toc' // Create a parser with specific configuration const parse = createMarkdownParser({ autoUnwrap: true, - autoClose: true, + autoClose: 'streaming', plugins: [ shiki({ themes: { light: githubLight, dark: githubDark } @@ -360,7 +361,7 @@ Both `parseMarkdown()` and `createMarkdownParser()` accept the same `ParserOptio | Option | Type | Default | Description | |--------|------|---------|-------------| | `autoUnwrap` | `boolean` | `true` | Remove unnecessary `

` wrappers from single-element containers | -| `autoClose` | `boolean \| (markdown: string) => string` | `true` | Auto-close incomplete markdown syntax, or use a custom completion function | +| `autoClose` | `boolean \| 'streaming' \| (markdown: string) => string` | `'streaming'` | Auto-close incomplete markdown syntax. `'streaming'` only heals a parse called with `{ streaming: true }`; `true` heals every parse | | `unwrap` | `boolean \| string \| string[]` | `false` | Remove wrapper tags from the tree, hoisting their children (MDC `unwrap` behaviour). `true` unwraps `p`; a comma/whitespace-separated string or array unwraps the listed tags; `'*'` matches any tag. Tags apply sequentially (each descends one level), and adjacent text is merged into a single string. | | `html` | `boolean` | `true` | **Deprecated** (warns). Prefer `registerDefaultPlugins: false` and register `html()` explicitly. `html: false` still skips the default html plugin. | | `linkify` | `boolean` | `true` | Auto-convert URL-like text into links. Set `false` to disable | diff --git a/docs/content/5.reference/2.auto-close.md b/docs/content/5.reference/2.auto-close.md index cea804c1..14c0ed89 100644 --- a/docs/content/5.reference/2.auto-close.md +++ b/docs/content/5.reference/2.auto-close.md @@ -57,19 +57,36 @@ autoCloseMarkdown('hello *', { dropTrailingOpeners: true }) :: ::tip -`autoCloseMarkdown` is also available as a parse option: set `autoClose: true` (default) in `parseMarkdown()` or `createMarkdownParser()` to apply it automatically. +`autoCloseMarkdown` is also available as a parse option: `autoClose` in `parseMarkdown()` and `createMarkdownParser()` applies it for you while streaming. :: ### Parser integration -`autoClose` is enabled by default in `parseMarkdown()` and `createMarkdownParser()`. You can disable it or provide a custom completion function: +`autoClose` defaults to `'streaming'`, so healing runs when you parse with `{ streaming: true }` and a plain parse follows CommonMark: + +```typescript +import { createMarkdownParser, parseMarkdown } from 'comark' + +await parseMarkdown('a _b') +// 'a _b' stays literal text + +const parse = createMarkdownParser() +await parse('a _b', { streaming: true }) +// ['em', {}, 'b'] +``` + +::warning +Before this default, healing ran on every parse. If you parse a stored response that may have been cut off, set `autoClose: true` to keep closing it. +:: + +You can force it on, turn it off, or provide a custom completion function: ::code-group -```typescript [Enabled (default)] +```typescript [Always] import { parseMarkdown } from 'comark' const result = await parseMarkdown(content, { - autoClose: true // default + autoClose: true }) ``` diff --git a/docs/content/5.reference/3.reference.md b/docs/content/5.reference/3.reference.md index 7132dcae..1c912fe4 100644 --- a/docs/content/5.reference/3.reference.md +++ b/docs/content/5.reference/3.reference.md @@ -26,7 +26,7 @@ import shiki from 'comark/plugins/shiki' const result = await parseMarkdown(source, { autoUnwrap: true, // Remove

wrappers from single-paragraph containers - autoClose: true, // Auto-close incomplete syntax + autoClose: 'streaming', // Auto-close incomplete syntax while streaming plugins: [shiki()] // HTML parsing is on by default via the html plugin }) @@ -296,7 +296,7 @@ type Node = ```typescript interface ParserOptions { autoUnwrap?: boolean // default: true - autoClose?: boolean | AutoCloseFunction // default: true + autoClose?: boolean | 'streaming' | AutoCloseFunction // default: 'streaming' unwrap?: boolean | string | string[] // strip top-level wrapper tags, e.g. 'p' (default: false) /** @deprecated Prefer registerDefaultPlugins: false */ html?: boolean // default: true diff --git a/docs/content/7.kb/2.migration-from-mdc.md b/docs/content/7.kb/2.migration-from-mdc.md index 273a504c..d42d9356 100644 --- a/docs/content/7.kb/2.migration-from-mdc.md +++ b/docs/content/7.kb/2.migration-from-mdc.md @@ -327,7 +327,7 @@ const document = await parseMarkdown(md, { plugins: [emoji()] }) { plugins: ComarkPlugin[], // ordered array, not a record autoUnwrap: true, // removes

from single-paragraph containers - autoClose: true, // completes incomplete syntax (useful for streaming) + autoClose: 'streaming', // completes incomplete syntax while streaming // HTML, components, attributes, alerts, task-list, frontmatter are on by default } ``` @@ -687,4 +687,4 @@ These features may be added in a future release. If your project relies on bindi | Prose components | `components/prose/Prose*.vue` | `components/prose/Prose*.vue` | | Render slot | `` | `` | | Unwrap slot | `` or `` | `` | -| Streaming | Not supported | `streaming` prop + `autoClose` | +| Streaming | Not supported | `streaming` prop, which also turns on `autoClose` healing | diff --git a/docs/skills/comark/AGENTS.md b/docs/skills/comark/AGENTS.md index 48ae692a..5b673ebc 100644 --- a/docs/skills/comark/AGENTS.md +++ b/docs/skills/comark/AGENTS.md @@ -6,7 +6,7 @@ A guide for using Comark in AI agent and LLM-powered applications where markdown LLMs stream markdown token-by-token. Standard markdown parsers expect complete input. They fail or produce broken output on partial streams. Comark was built to handle exactly this: -- **`autoClose`** (default: `true`): incomplete syntax like `**bold text` is automatically closed on every parse, so partial tokens always render correctly +- **`autoClose`** (default: `'streaming'`): incomplete syntax like `**bold text` is closed when you parse with `{ streaming: true }`, so partial tokens always render correctly. A plain parse leaves it as written; set `autoClose: true` to close on every parse - **Streaming mode**: re-renders efficiently as content arrives - **Caret indicator**: shows a live cursor during generation - **ANSI rendering**: styled terminal output for CLI agents @@ -297,7 +297,7 @@ export const ChatMarkdown = defineMarkdownComponent({ shiki({ themes: { light: githubDark, dark: githubDark } }), ], components: { Math, alert: Alert }, - autoClose: true, + autoClose: 'streaming', }) ``` diff --git a/docs/skills/comark/references/parsing-ast.md b/docs/skills/comark/references/parsing-ast.md index fa14e681..0a39df91 100644 --- a/docs/skills/comark/references/parsing-ast.md +++ b/docs/skills/comark/references/parsing-ast.md @@ -53,7 +53,7 @@ interface MarkdownDocument { ```typescript interface ParserOptions { autoUnwrap?: boolean // Remove unnecessary

wrappers (default: true) - autoClose?: boolean // Auto-close unclosed syntax (default: true) + autoClose?: boolean | 'streaming' | ((markdown: string) => string) // Auto-close unclosed syntax (default: 'streaming') plugins?: ComarkPlugin[] // Enable plugins (e.g., highlight, emoji, toc) } ``` diff --git a/docs/skills/comark/references/rendering-svelte.md b/docs/skills/comark/references/rendering-svelte.md index 2db2470e..ac561411 100644 --- a/docs/skills/comark/references/rendering-svelte.md +++ b/docs/skills/comark/references/rendering-svelte.md @@ -330,7 +330,7 @@ The `caret` prop appends a blinking cursor indicator to the last text node durin ``` -`autoClose` is enabled by default: incomplete syntax like `**bold text` is automatically closed on every parse. +`autoClose` defaults to `'streaming'`: incomplete syntax like `**bold text` is closed when the `streaming` prop is set, and left as written otherwise. --- diff --git a/docs/skills/migrate-mdc-to-comark/SKILL.md b/docs/skills/migrate-mdc-to-comark/SKILL.md index 62b3bcb0..67884606 100644 --- a/docs/skills/migrate-mdc-to-comark/SKILL.md +++ b/docs/skills/migrate-mdc-to-comark/SKILL.md @@ -61,7 +61,7 @@ The migration has two parts: **Core Package** (programmatic API) and **Nuxt Modu { plugins: ComarkPlugin[], // ordered array, not a record autoUnwrap: true, // removes

from single-paragraph containers - autoClose: true, // completes incomplete syntax (useful for streaming) + autoClose: 'streaming', // completes incomplete syntax while streaming // HTML, components, attributes, alerts, task-list, frontmatter are on by default } ``` diff --git a/packages/comark-ansi/test/index.test.ts b/packages/comark-ansi/test/index.test.ts index 0439a89d..deac9995 100644 --- a/packages/comark-ansi/test/index.test.ts +++ b/packages/comark-ansi/test/index.test.ts @@ -437,8 +437,16 @@ describe('renderAnsi', () => { }) it('passes parser and renderer options through', async () => { - const output = await renderAnsi('**bold', { autoClose: false, colors: false }) + // `autoClose: false` matches the default for a non-streaming render, so force + // healing on to prove the option reaches the parser. + const output = await renderAnsi('**bold', { autoClose: true, colors: false }) expect(output).not.toContain('\x1B[') + expect(output).toContain('bold') + expect(output).not.toContain('\\*\\*bold') + }) + + it('leaves incomplete markdown alone by default', async () => { + const output = await renderAnsi('**bold', { colors: false }) expect(output).toContain('\\*\\*bold') }) diff --git a/packages/comark-html/test/index.test.ts b/packages/comark-html/test/index.test.ts index 7cdabd26..886a30e8 100644 --- a/packages/comark-html/test/index.test.ts +++ b/packages/comark-html/test/index.test.ts @@ -68,7 +68,14 @@ describe('renderHtml', () => { }) it('passes parser options through', async () => { - const html = await renderHtml('**bold', { autoClose: false }) + // `autoClose: false` matches the default for a non-streaming render, so force + // healing on to prove the option reaches the parser. + const html = await renderHtml('**bold', { autoClose: true }) + expect(html).toContain('') + }) + + it('leaves incomplete markdown alone by default', async () => { + const html = await renderHtml('**bold') expect(html).toContain('**bold') expect(html).not.toContain('') }) diff --git a/packages/comark-react/src/components/MarkdownClient.tsx b/packages/comark-react/src/components/MarkdownClient.tsx index f3d4a9ad..76073d3f 100644 --- a/packages/comark-react/src/components/MarkdownClient.tsx +++ b/packages/comark-react/src/components/MarkdownClient.tsx @@ -36,6 +36,8 @@ function MarkdownContent({ } export function MarkdownClient({ children, value, options = {}, plugins = [], ...rest }: MarkdownProps) { + const streaming = rest.streaming ?? false + const content = isMarkdownDocument(value) ? value : children @@ -46,8 +48,13 @@ export function MarkdownClient({ children, value, options = {}, plugins = [], .. // Note: options/plugins should be stable references (defined outside render or memoized). // Pre-parsed documents resolve immediately without calling parseMarkdown(). const parsePromise = useMemo( - () => (isMarkdownDocument(content) ? Promise.resolve(content) : parseMarkdown(content, { ...options, plugins })), - [content] + () => + isMarkdownDocument(content) + ? Promise.resolve(content) + : // `streaming` must reach the parser, not just the renderer: it drives + // auto-close healing and incremental node reuse. + parseMarkdown(content, { ...options, plugins }, { streaming }), + [content, streaming] ) // Keep showing the previous parsed result while a new parse is pending — diff --git a/packages/comark-react/test/streaming.test.tsx b/packages/comark-react/test/streaming.test.tsx new file mode 100644 index 00000000..31679335 --- /dev/null +++ b/packages/comark-react/test/streaming.test.tsx @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import React from 'react' +import { renderToReadableStream } from 'react-dom/server' +import { Markdown } from '../src/index' + +async function renderAsync(element: React.ReactElement): Promise { + const stream = await renderToReadableStream(element) + await stream.allReady + return new Response(stream).text() +} + +// The `streaming` prop drives the renderer (caret, stream components) but it also +// has to reach the parser: auto-close heals only on a streaming parse. +describe('', () => { + it('heals incomplete markdown while streaming', async () => { + const html = await renderAsync( + + ) + expect(html).toContain('') + expect(html).toContain('wor') + }) + + it('leaves incomplete markdown alone when not streaming', async () => { + const html = await renderAsync() + expect(html).not.toContain('') + expect(html).toContain('**wor') + }) +}) diff --git a/packages/comark-svelte/src/async/MarkdownAsync.svelte b/packages/comark-svelte/src/async/MarkdownAsync.svelte index 090dcdd5..d4577406 100644 --- a/packages/comark-svelte/src/async/MarkdownAsync.svelte +++ b/packages/comark-svelte/src/async/MarkdownAsync.svelte @@ -65,7 +65,7 @@ and wrap this component in a `` for pending/error states. : // `parse` directly mutates `plugins` which creates an infinite effect loop // so we copy it before passing it in so it gets a regular JS array and we get to still // track dependencies from an external perspective - await parseMarkdown(content, { ...options, ...(unwrap ? { unwrap } : {}), plugins: [...plugins] }), + await parseMarkdown(content, { ...options, ...(unwrap ? { unwrap } : {}), plugins: [...plugins] }, { streaming }), ) diff --git a/packages/comark-svelte/src/components/Markdown.svelte b/packages/comark-svelte/src/components/Markdown.svelte index fdcffadf..cfac7623 100644 --- a/packages/comark-svelte/src/components/Markdown.svelte +++ b/packages/comark-svelte/src/components/Markdown.svelte @@ -65,7 +65,7 @@ This is an alert component // `parse` directly mutates `plugins` which creates an infinite effect loop // so we copy it before passing it in so it gets a regular JS array and we get to still // track dependencies from an external perspective - parseMarkdown(content, { ...options, ...(unwrap ? { unwrap } : {}), plugins: [...plugins] }).then((result) => { + parseMarkdown(content, { ...options, ...(unwrap ? { unwrap } : {}), plugins: [...plugins] }, { streaming }).then((result) => { if (currentVersion > appliedVersion) { appliedVersion = currentVersion parsed = result diff --git a/packages/comark-svelte/test/streaming.svelte.test.ts b/packages/comark-svelte/test/streaming.svelte.test.ts index a4867572..0dcd1082 100644 --- a/packages/comark-svelte/test/streaming.svelte.test.ts +++ b/packages/comark-svelte/test/streaming.svelte.test.ts @@ -89,6 +89,29 @@ describe('streaming mode', () => { await expect.element(screen.getByText('Hello world')).toBeInTheDocument() }) + // The `streaming` prop has to reach the parser, not just the renderer: auto-close + // heals only on a streaming parse, so these pass no `options.autoClose`. + // Asserted on markup rather than text, because 'wor' is a substring of '**wor'. + it('heals incomplete bold from the streaming prop alone', async () => { + const screen = await render(Markdown, { + value: 'Hello **wor', + streaming: true, + }) + + await expect.poll(() => screen.container.innerHTML).toContain('') + expect(screen.container.innerHTML).not.toContain('**wor') + }) + + it('leaves incomplete bold alone when not streaming', async () => { + const screen = await render(Markdown, { + value: 'Hello **wor', + streaming: false, + }) + + await expect.poll(() => screen.container.innerHTML).toContain('**wor') + expect(screen.container.innerHTML).not.toContain('') + }) + it('handles incomplete heading during streaming', async () => { const screen = await render(Markdown, { value: '# Hell', diff --git a/packages/comark/SPEC/auto-close.md b/packages/comark/SPEC/auto-close.md index 10dd4ac7..04472ac2 100644 --- a/packages/comark/SPEC/auto-close.md +++ b/packages/comark/SPEC/auto-close.md @@ -14,6 +14,10 @@ options: - math: auto-close inline `$…$` and block `$$…$$` (default: `false`) - dropTrailingOpeners: drop a trailing opener after whitespace at EOF (`hello *` → `hello`) so half-typed markers do not flash (default: `false`; enabled when parsing with `streaming: true`) +The parse-level `autoClose` option defaults to `'streaming'`, so `parseMarkdown()` and +`createMarkdownParser()` heal only a parse called with `{ streaming: true }`. `autoClose: true` +heals every parse. Calling `autoCloseMarkdown` directly always heals. + --- @@ -183,6 +187,73 @@ Leaves finished inline code alone: + Text with `inline code` ``` +### Delimiter runs + +A span closes on a backtick run of the same length as its opener. A shorter or longer +run inside the span is literal content, so a finished multi-backtick span is left alone +even when text follows it. + +```diff +- a ``x`` b ++ a ``x`` b +``` + +```diff +- `` a _b `` ++ `` a _b `` +``` + +```diff +- ``{ modelValue: _Number }`` and more ++ ``{ modelValue: _Number }`` and more +``` + +```diff +- ``Use `code` in your Markdown file.`` ++ ``Use `code` in your Markdown file.`` +``` + +An unclosed run is closed with a run of its own length, and markers opened inside it +still close inside: + +```diff +- use ``a _b`` then _c ++ use ``a _b`` then _c_ +``` + +```diff +- `a`` b ++ `a`` b` +``` + +```diff +- ``a` b ++ ``a` b`` +``` + +A trailing run merges with the closer instead of being appended to, so only the +backticks the run still needs are added: + +```diff +- Use ``code` ++ Use ``code`` +``` + +A trailing run longer than the opener is left alone. No closer can be appended next to +it that would read as a run of the opener's length: + +```diff +- `a`` ++ `a`` +``` + +A run of three or more mid-line is fence-shaped, not an inline span, and stays literal: + +```diff +- a ```x b ++ a ```x b +``` + --- ## Strikethrough @@ -728,6 +799,68 @@ Space-flanked `*` is treated as multiply, not italic: --- +## Multiple openers on one line + +Two closers for the same marker emitted back-to-back would merge into a different +marker, so each run of same-marker closers collapses to one. `a _b and _c` used to +heal to `a _b and _c__`, which nests an em inside an em. This covers `_`, `__`, `~~` +and `*`. A pair of `**` openers follows the balanced-overlap rule below instead. + +```diff +- a _b and _c ++ a _b and _c_ +``` + +```diff +- a __b and __c ++ a __b and __c__ +``` + +```diff +- a ~~b and ~~c ++ a ~~b and ~~c~~ +``` + +```diff +- *a *b *c ++ *a *b *c* +``` + +Markers of different families still nest, so non-adjacent repeats are left alone: + +```diff +- _a **b _c ++ _a **b _c_**_ +``` + +```diff +- a _b and __c ++ a _b and __c___ +``` + +The collapse only applies to closers that came from different openers. One run of four +underscores opens `__` twice at the same spot, and both of those need closing: + +```diff +- ____a ++ ____a____ +``` + +```diff +- a ______b ++ a ______b______ +``` + +A `**` pair follows the balanced-overlap rule: an even number of `**` runs reads as +already balanced, so nothing is appended. + +```diff +- a **b and **c ++ a **b and **c +``` + +--- + ## Math protects inner markers Underscores and asterisks inside math are not italic/bold. diff --git a/packages/comark/SPEC/common-mark/paragraph-code-double-tick-trailing.md b/packages/comark/SPEC/common-mark/paragraph-code-double-tick-trailing.md new file mode 100644 index 00000000..cbbca08f --- /dev/null +++ b/packages/comark/SPEC/common-mark/paragraph-code-double-tick-trailing.md @@ -0,0 +1,39 @@ +## Input + +```md +a ``x`` b +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "a ", + [ + "code", + {}, + "x" + ], + " b" + ] + ] +} +``` + +## HTML + +```html +

a x b

+``` + +## Markdown + +```md +a `x` b +``` diff --git a/packages/comark/SPEC/common-mark/paragraph-unmatched-emphasis.md b/packages/comark/SPEC/common-mark/paragraph-unmatched-emphasis.md new file mode 100644 index 00000000..74270799 --- /dev/null +++ b/packages/comark/SPEC/common-mark/paragraph-unmatched-emphasis.md @@ -0,0 +1,33 @@ +## Input + +```md +a _b and *c +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "a _b and *c" + ] + ] +} +``` + +## HTML + +```html +

a _b and *c

+``` + +## Markdown + +```md +a \_b and \*c +``` diff --git a/packages/comark/src/internal/parse/auto-close/index.ts b/packages/comark/src/internal/parse/auto-close/index.ts index 3fb0ee22..16d4c600 100644 --- a/packages/comark/src/internal/parse/auto-close/index.ts +++ b/packages/comark/src/internal/parse/auto-close/index.ts @@ -31,7 +31,7 @@ export interface AutoCloseOptions { /** * Drop a trailing opener (`* _ $ : [ { !`) after whitespace at EOF so a * half-typed marker does not flash (`hello *` → `hello`). Default false. - * Enabled automatically when `parseMarkdown(..., { streaming: true })`. + * Enabled automatically when `parseMarkdown(md, {}, { streaming: true })`. */ dropTrailingOpeners?: boolean } @@ -320,10 +320,20 @@ function healInline(text: string, opts: HealOpts): string { // 2) Build mutated string for escapes while collecting open markers const len = text.length const out: string[] = [] - let stack: Marker[] = [] + const stack: Marker[] = [] + // Source index of the delimiter run that pushed each marker, kept in lockstep + // with `stack`. One run of four underscores pushes `__` twice, and both closers + // have to be emitted, so `closeOpenStack` needs to tell those apart from two + // markers opened at different places. + const stackPos: number[] = [] let fence = false let inCode = false + // Backtick count of the open code span, and the index in `out` where its + // opening run starts. A code span cannot nest, so one pair of scalars is + // enough; `out` is append-only, so the index stays valid in the joined result. + let codeRun = 0 + let codeOpenOut = -1 let inMath = false let inBlockMath = false let inLatexI = false @@ -342,17 +352,27 @@ function healInline(text: string, opts: HealOpts): string { let doubleAsteriskCount = 0 let tripleCount = 0 + const push = (m: Marker, pos: number) => { + stack.push(m) + stackPos.push(pos) + } + + const pop = () => { + stack.pop() + stackPos.pop() + } + /** Open only when the run is not followed by space; close only when not preceded by space. */ - const toggleFlanking = (m: Marker, prevCh: string, afterCh: string) => { + const toggleFlanking = (m: Marker, prevCh: string, afterCh: string, pos: number) => { const canClose = !isSpace(prevCh) && stack[stack.length - 1] === m const canOpen = !isSpace(afterCh) - if (canClose) stack.pop() - else if (canOpen) stack.push(m) + if (canClose) pop() + else if (canOpen) push(m, pos) } - const toggle = (m: Marker) => { - if (stack[stack.length - 1] === m) stack.pop() - else stack.push(m) + const toggle = (m: Marker, pos: number) => { + if (stack[stack.length - 1] === m) pop() + else push(m, pos) } for (let i = 0; i < len; i++) { @@ -460,11 +480,24 @@ function healInline(text: string, opts: HealOpts): string { // Regions that protect markers if (inCode) { - out.push(ch) - if (ch === '`' && next !== '`' && prev !== '`') { - inCode = false - if (stack[stack.length - 1] === '`') stack.pop() + if (ch === '`') { + // A code span closes on a backtick run of the same length as its opener. + // A shorter or longer run is literal content (CommonMark), which is what + // keeps ``Use `code` in your file.`` intact. + let end = i + while (end + 1 < len && text[end + 1] === '`') end++ + const run = end - i + 1 + for (let k = i; k <= end; k++) out.push('`') + if (run === codeRun) { + inCode = false + codeRun = 0 + codeOpenOut = -1 + if (stack[stack.length - 1] === '`') pop() + } + i = end + continue } + out.push(ch) continue } if (inBlockMath) { @@ -473,7 +506,7 @@ function healInline(text: string, opts: HealOpts): string { out.push('$') i++ inBlockMath = false - if (stack[stack.length - 1] === '$$') stack.pop() + if (stack[stack.length - 1] === '$$') pop() } continue } @@ -481,7 +514,7 @@ function healInline(text: string, opts: HealOpts): string { out.push(ch) if (ch === '$' && next !== '$') { inMath = false - if (stack[stack.length - 1] === '$') stack.pop() + if (stack[stack.length - 1] === '$') pop() } continue } @@ -551,15 +584,23 @@ function healInline(text: string, opts: HealOpts): string { // Code if (ch === '`') { - if (next === '`' && text[i + 2] === '`') { - // triple on non-line-start — copy - out.push('`', '`', '`') - i += 2 + let end = i + while (end + 1 < len && text[end + 1] === '`') end++ + const run = end - i + 1 + if (run >= 3) { + // The fence handling above only fires at the start of a line, so this is + // what keeps a mid-line run of three or more from opening a span: it is + // copied verbatim and never becomes an inline span. + for (let k = i; k <= end; k++) out.push('`') + i = end continue } - out.push(ch) + codeOpenOut = out.length + for (let k = i; k <= end; k++) out.push('`') inCode = true - stack.push('`') + codeRun = run + push('`', i) + i = end continue } @@ -571,12 +612,12 @@ function healInline(text: string, opts: HealOpts): string { i++ if (opts.math) { inBlockMath = !inBlockMath - toggle('$$') + toggle('$$', i - 1) } } else if (opts.math && looksLikeInlineMathOpen(text, i)) { // Skip currency (`$100`) and component names (`::$special`) inMath = true - stack.push('$') + push('$', i) } continue } @@ -605,10 +646,10 @@ function healInline(text: string, opts: HealOpts): string { continue } asteriskTotal += run - if (run === 1) toggleFlanking('*', prev, after) + if (run === 1) toggleFlanking('*', prev, after, i) else if (run === 2) { doubleAsteriskCount++ - toggleFlanking('**', prev, after) + toggleFlanking('**', prev, after, i) } else if (run >= 3) { // Horizontal rule: a whole line of ≥3 * (with only spaces) is not emphasis let ls = i @@ -636,21 +677,24 @@ function healInline(text: string, opts: HealOpts): string { const hasBold = stack.includes('**') if (hasStar && hasBold && !leftSpace) { for (let si = stack.length - 1; si >= 0; si--) { - if (stack[si] === '*' || stack[si] === '**') stack.splice(si, 1) + if (stack[si] === '*' || stack[si] === '**') { + stack.splice(si, 1) + stackPos.splice(si, 1) + } } doubleAsteriskCount++ } else { tripleCount++ - toggleFlanking('***', prev, after) + toggleFlanking('***', prev, after, i) } } else { // ****+ const pairs = Math.floor(run / 2) for (let p = 0; p < pairs; p++) { doubleAsteriskCount++ - toggleFlanking('**', prev, after) + toggleFlanking('**', prev, after, i) } - if (run % 2 === 1) toggleFlanking('*', prev, after) + if (run % 2 === 1) toggleFlanking('*', prev, after, i) } i = end continue @@ -690,13 +734,13 @@ function healInline(text: string, opts: HealOpts): string { } if (!(isWord(prev) && isWord(after)) && !surrounded) { - if (run === 1) toggleFlanking('_', prev, after) + if (run === 1) toggleFlanking('_', prev, after, i) else if (run >= 2) { const pairs = Math.floor(run / 2) for (let p = 0; p < pairs; p++) { - toggleFlanking('__', prev, after) + toggleFlanking('__', prev, after, i) } - if (run % 2 === 1) toggleFlanking('_', prev, after) + if (run % 2 === 1) toggleFlanking('_', prev, after, i) } } i = end @@ -712,7 +756,7 @@ function healInline(text: string, opts: HealOpts): string { for (let k = i; k <= end; k++) out.push('~') if (!surrounded && run >= 2) { const pairs = Math.floor(run / 2) - for (let p = 0; p < pairs; p++) toggleFlanking('~~', prev, after) + for (let p = 0; p < pairs; p++) toggleFlanking('~~', prev, after, i) } // single ~ not stacked (SPEC escapes or leaves alone) i = end @@ -760,8 +804,7 @@ function healInline(text: string, opts: HealOpts): string { // SPEC: `**bold with `code` → `**bold with `code**`` // Markers that opened *before* the code span must close inside it. if (inCode) { - const lastBq = result.lastIndexOf('`') - const afterBq = lastBq >= 0 ? result.slice(lastBq + 1) : '' + const afterBq = codeOpenOut >= 0 ? result.slice(codeOpenOut + codeRun) : '' if (afterBq.length > 0) { // Markers still on stack before the open ` need closing inside the span. // Open order is outer→inner left-to-right; close reverse order after content. @@ -780,13 +823,20 @@ function healInline(text: string, opts: HealOpts): string { const m = stack[si] if (m === '**' || m === '*' || m === '__' || m === '_' || m === '~~' || m === '***') inner += m } - return result + inner + '`' + // A backtick run already at the end merges with the closer we are about to + // append, so only add what that run still needs. A run longer than the + // opener can never be turned into a closer, so leave the text alone. + const base = result + inner + let trail = 0 + while (trail < base.length && base[base.length - 1 - trail] === '`') trail++ + if (trail > codeRun) return result + return base + '`'.repeat(codeRun - trail) } return result } // Build suffix inside-out with half-close handling - result = closeOpenStack(result, stack, { + result = closeOpenStack(result, stack, stackPos, { asteriskTotal, doubleAsteriskCount, tripleCount, @@ -832,9 +882,16 @@ function isBareOrHr(text: string): boolean { return false } +/** An open marker plus the source index of the delimiter run that opened it. */ +interface OpenMarker { + m: Marker + pos: number +} + function closeOpenStack( text: string, stack: Marker[], + stackPos: number[], counts: { asteriskTotal: number; doubleAsteriskCount: number; tripleCount: number } ): string { // Half-closes first @@ -853,9 +910,9 @@ function closeOpenStack( const balancedOverlap = counts.doubleAsteriskCount >= 2 && counts.doubleAsteriskCount % 2 === 0 && counts.asteriskTotal % 2 === 0 - let workStack = stack.slice() + let workStack: OpenMarker[] = stack.map((m, i) => ({ m, pos: stackPos[i] })) if (balancedOverlap) { - workStack = workStack.filter((m) => m !== '***' && m !== '**' && m !== '*') + workStack = workStack.filter(({ m }) => m !== '***' && m !== '**' && m !== '*') } // SPEC nested formatting: when multiple markers are open, close from the inside @@ -866,38 +923,36 @@ function closeOpenStack( // `_italic and **bold` → `_italic and **bold**_` // `~~strike with **bold` → `~~strike with **bold**~~` - const closable: Marker[] = [] + const closable: OpenMarker[] = [] // Scan stack from top (innermost) for (let i = workStack.length - 1; i >= 0; i--) { - const m = workStack[i] - if (m === '$$') { - closable.push('$$') - continue - } - if (m === '$') { - closable.push('$') + const open = workStack[i] + const m = open.m + if (m === '$$' || m === '$') { + closable.push(open) continue } if (m === '`') continue const token = m - const pos = text.lastIndexOf(token) - if (pos < 0) continue - const after = text.slice(pos + token.length) + const at = text.lastIndexOf(token) + if (at < 0) continue + const after = text.slice(at + token.length) if (!hasClosableContentAfter(after)) continue - closable.push(m) + closable.push(open) } if (closable.length === 0) return text // Collapse same-family asterisk closers: if both *** / ** / * appear, keep only innermost needed. // Prefer: if top (first in closable which is reverse stack) is * and ** is also closable, only *. - const hasStarFamily = closable.includes('*') || closable.includes('**') || closable.includes('***') + const isStar = (m: Marker) => m === '*' || m === '**' || m === '***' + const hasStarFamily = closable.some(({ m }) => isStar(m)) if (hasStarFamily) { // Innermost open asterisk marker is first in closable (stack was reversed) let firstStar: Marker | null = null - for (const m of closable) { - if (m === '*' || m === '**' || m === '***') { + for (const { m } of closable) { + if (isStar(m)) { firstStar = m break } @@ -906,23 +961,34 @@ function closeOpenStack( // If only ** open, close ** // If *** open, close *** // Exception cross nests are separate tokens - if (firstStar === '*' && closable.includes('**') && !closable.includes('***')) { + if (firstStar === '*' && closable.some(({ m }) => m === '**') && !closable.some(({ m }) => m === '***')) { // **bold and *italic → only * // BUT *italic with **bold → stack [*, **] top is ** → firstStar ** → close ***? // For * outer + ** inner: firstStar is ** (top), emit ** then * = *** which matches SPEC // So only strip ** when * is TOP (innermost) // closable[0] is top of stack - if (closable[0] === '*') { + if (closable[0].m === '*') { // remove ** and *** from closable for (let ci = closable.length - 1; ci >= 0; ci--) { - if (closable[ci] === '**' || closable[ci] === '***') closable.splice(ci, 1) + if (closable[ci].m === '**' || closable[ci].m === '***') closable.splice(ci, 1) } } } } + // Two same-token closers emitted back to back merge into a different marker: + // `a _b and _c` produced `__`, which reads as strong and nested an em inside + // an em. Collapse each run to one closer, but only when the two openers came + // from different delimiter runs. `____a` opens `__` twice from one run of four + // and needs both closers, and non-adjacent repeats are left alone anyway, + // since `_a **b _c` legitimately closes `_`, `**`, `_`. + const emitted = closable.filter((tok, ci) => { + const prev = closable[ci - 1] + return !prev || tok.m !== prev.m || tok.pos === prev.pos + }) + let suffix = '' - for (const m of closable) { + for (const { m } of emitted) { if (m === '$$') { if (text.endsWith('$') && !text.endsWith('$$')) suffix += '$' else { diff --git a/packages/comark/src/parse.ts b/packages/comark/src/parse.ts index fa55a422..4a9074ca 100644 --- a/packages/comark/src/parse.ts +++ b/packages/comark/src/parse.ts @@ -1,5 +1,6 @@ import type { ComarkParseFn, + ComarkParseFnOptions, ComarkParsePostState, ComarkPlugin, MarkdownExitPlugin, @@ -63,7 +64,7 @@ export { defineComarkPlugin } from './utils/helpers.ts' export function createMarkdownParser[] = []>( options: ParserOptions = {} as ParserOptions ): ComarkParseFn>, ResolvedFrontmatter>> { - const { autoUnwrap = true, autoClose = true, tracer = noopTracer } = options + const { autoUnwrap = true, autoClose = 'streaming', tracer = noopTracer } = options // Tag set to strip from the top level of the tree (MDC `unwrap`). Resolved once. const unwrapTags = resolveUnwrapTags(options.unwrap) @@ -138,9 +139,11 @@ export function createMarkdownParser autoClose(state.markdown)) - } else if (autoClose) { + } else if (autoClose === 'streaming' ? opts.streaming : autoClose) { state.markdown = withSpan(tracer, 'comark:autoclose', () => autoCloseMarkdown(state.markdown, { frontmatter: hasPlugin('frontmatter') && opts.streaming, @@ -255,17 +258,21 @@ export function createMarkdownParser[] = []>( markdown: string, - options: ParserOptions = {} as ParserOptions + options: ParserOptions = {} as ParserOptions, + parseOptions: ComarkParseFnOptions = {} ): Promise< MarkdownDocument>, ResolvedFrontmatter>> > { const parser = createMarkdownParser(options) - return await parser(markdown) + return await parser(markdown, parseOptions) } /** diff --git a/packages/comark/src/types.ts b/packages/comark/src/types.ts index 50ceeae9..9a336d96 100644 --- a/packages/comark/src/types.ts +++ b/packages/comark/src/types.ts @@ -462,11 +462,26 @@ export interface ParserOptions[ unwrap?: boolean | string | string[] /** - * Whether to automatically close unclosed markdown and Comark components, - * or a custom function that rewrites incomplete markdown before tokenization. - * @default true + * Controls healing of incomplete markdown and Comark components before + * tokenization. + * + * - `'streaming'` (default) heals only when the parse is called with + * `{ streaming: true }`. A plain `parseMarkdown(md)` follows CommonMark, so + * `a _b` stays literal text. + * - `true` heals on every parse. Use it when the input is a complete string + * that may have been cut off, such as a stored partial AI response. + * - `false` never heals. + * - An {@link AutoCloseFunction} replaces the built-in healer and runs on + * every parse, streaming or not. + * + * @default 'streaming' + * @example + * // Default: CommonMark on a plain parse, healed while streaming + * await parseMarkdown('a _b') // 'a _b' + * await parseMarkdown('a _b', {}, { streaming: true }) // b + * await parseMarkdown('a _b', { autoClose: true }) // b */ - autoClose?: boolean | AutoCloseFunction + autoClose?: boolean | 'streaming' | AutoCloseFunction /** * @deprecated Use `registerDefaultPlugins: false` and register plugins explicitly diff --git a/packages/comark/test/auto-close-default.test.ts b/packages/comark/test/auto-close-default.test.ts new file mode 100644 index 00000000..e8208be6 --- /dev/null +++ b/packages/comark/test/auto-close-default.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { createMarkdownParser, parseMarkdown } from '../src/parse.ts' + +describe('ParserOptions.autoClose', () => { + describe("default ('streaming')", () => { + it('leaves an unmatched emphasis opener literal on a plain parse', async () => { + const tree = await parseMarkdown('a _b') + expect(tree.nodes).toEqual([['p', {}, 'a _b']]) + }) + + it('heals through parseMarkdown when the per-call options say streaming', async () => { + // The third argument is the public streaming entry point. Nothing else + // exercises it, so a merge that drops it would otherwise stay green. + const tree = await parseMarkdown('a _b', {}, { streaming: true }) + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a ', ['em', {}, 'b']]]) + }) + + it('heals when the parse call opts into streaming', async () => { + const parse = createMarkdownParser() + const tree = await parse('a _b', { streaming: true }) + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a ', ['em', {}, 'b']]]) + }) + + it('leaves an unclosed component fence to the components plugin', async () => { + const tree = await parseMarkdown('::alert\nHello') + expect(tree.nodes).toEqual([['alert', {}, 'Hello']]) + }) + + it('keeps a trailing `::` literal instead of dropping it', async () => { + // Healing strips a half-typed `::` on its own line. A plain parse must not, + // so the two nested-component fixtures no longer carry a stray closer. + const tree = await parseMarkdown('para\n\n::') + expect(tree.nodes).toEqual([ + ['p', {}, 'para'], + ['p', {}, '::'], + ]) + }) + }) + + describe('true', () => { + it('heals on a plain, non-streaming parse', async () => { + const tree = await parseMarkdown('a _b', { autoClose: true }) + expect(tree.nodes).toEqual([['p', {}, 'a ', ['em', {}, 'b']]]) + }) + }) + + describe('false', () => { + it('never heals, even while streaming', async () => { + const parse = createMarkdownParser({ autoClose: false }) + const tree = await parse('a _b', { streaming: true }) + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a _b']]) + }) + }) + + describe('custom function', () => { + it('runs on a non-streaming parse', async () => { + const tree = await parseMarkdown('a _b', { autoClose: (markdown) => `${markdown}_` }) + expect(tree.nodes).toEqual([['p', {}, 'a ', ['em', {}, 'b']]]) + }) + + it('runs on a streaming parse', async () => { + const parse = createMarkdownParser({ autoClose: (markdown) => `${markdown}_` }) + const tree = await parse('a _b', { streaming: true }) + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a ', ['em', {}, 'b']]]) + }) + }) +}) diff --git a/packages/comark/test/auto-close-idempotent.test.ts b/packages/comark/test/auto-close-idempotent.test.ts new file mode 100644 index 00000000..9f403765 --- /dev/null +++ b/packages/comark/test/auto-close-idempotent.test.ts @@ -0,0 +1,88 @@ +/** + * Healing has to be a fixed point: a stream re-heals the same line on every chunk, + * so `autoCloseMarkdown(autoCloseMarkdown(x))` must equal `autoCloseMarkdown(x)`. + * A healer that grows its own output makes markers flash and drift while typing. + */ +import { autoCloseMarkdown } from '../src/internal/parse/auto-close/index.ts' +import { describe, expect, it } from 'vitest' + +const TOKENS = ['`', '``', '```', '_', '__', '*', '**', '~~', 'a', 'b', ' '] +const MAX_TOKENS = 5 + +/** Every token string of length 2 to MAX_TOKENS. */ +function corpus(): string[] { + const out: string[] = [] + const build = (prefix: string, depth: number) => { + if (depth >= 2) out.push(prefix) + if (depth === MAX_TOKENS) return + for (const token of TOKENS) build(prefix + token, depth + 1) + } + for (const token of TOKENS) build(token, 1) + return out +} + +const trailingBackticks = (text: string) => text.match(/`+$/)?.[0].length ?? 0 +const longestBacktickRun = (text: string) => Math.max(0, ...[...text.matchAll(/`+/g)].map((m) => m[0].length)) + +/** + * Rules that already needed more than one pass before the two fixes this file + * landed with, all of them in `closeOpenStack`. They are listed as shapes rather + * than inputs because the corpus hits each one a few hundred times. + */ +const PRE_EXISTING = { + /** Half-close: completing a partly typed closer returns early, so any outer marker waits a pass. */ + halfClosed: (text: string) => + [/\*\*\*[^*]+\*{1,2}$/, /\*\*[^*]+\*$/, /__[^_]+_$/, /~~[^~]+~$/].some((re) => + re.test(text.replace(/(? { + const runs = [...text.matchAll(/\*+/g)].map((m) => m[0].length) + return runs.length > 1 && runs[runs.length - 1] === 1 && runs.slice(0, -1).some((n) => n >= 2) + }, +} + +const converges = (input: string, healed: string) => + PRE_EXISTING.halfClosed(input) || PRE_EXISTING.halfClosed(healed) || PRE_EXISTING.starCollapse(input) + +describe('autoCloseMarkdown is idempotent', () => { + const inputs = corpus() + + it('heals to a fixed point', () => { + const failures: string[] = [] + let skipped = 0 + + for (const input of inputs) { + const once = autoCloseMarkdown(input) + const twice = autoCloseMarkdown(once) + if (twice === once) continue + if (converges(input, once)) { + skipped++ + continue + } + failures.push(`${JSON.stringify(input)} → ${JSON.stringify(once)} → ${JSON.stringify(twice)}`) + } + + expect(failures).toEqual([]) + // The skipped shapes are a rounding error on the corpus. If this ever trips it + // means a rule above swallowed the whole suite instead of a known exception. + expect(skipped).toBeLessThan(inputs.length / 100) + }) + + it('never closes a code span with a longer backtick run than the input opened', () => { + const failures: string[] = [] + + for (const input of inputs) { + const healed = autoCloseMarkdown(input) + if (trailingBackticks(healed) > longestBacktickRun(input)) { + failures.push(`${JSON.stringify(input)} → ${JSON.stringify(healed)}`) + } + } + + expect(failures).toEqual([]) + }) + + it('covers a large corpus', () => { + expect(inputs.length).toBeGreaterThan(100_000) + }) +}) diff --git a/packages/comark/test/auto-close-parse.test.ts b/packages/comark/test/auto-close-parse.test.ts new file mode 100644 index 00000000..09d6f0c0 --- /dev/null +++ b/packages/comark/test/auto-close-parse.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest' +import { createMarkdownParser } from '../src/parse.ts' + +// The string-level contract lives in SPEC/auto-close.md. These pin the same fixes +// at AST level, so a healed string that still parses wrongly cannot slip through. +const parse = createMarkdownParser({ plugins: [] }) +const heal = (markdown: string) => parse(markdown, { streaming: true }) + +describe('auto-close, parsed', () => { + describe('code span delimiter runs', () => { + it('does not leak a backtick after a finished double-backtick span', async () => { + const tree = await heal('a ``x`` b') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a ', ['code', {}, 'x'], ' b']]) + }) + + it('closes markers after the span without leaking a backtick', async () => { + const tree = await heal('use ``a _b`` then _c') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'use ', ['code', {}, 'a _b'], ' then ', ['em', {}, 'c']]]) + }) + + it('keeps a single backtick inside a double-backtick span literal', async () => { + const tree = await heal('``Use `code` in your Markdown file.``') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, ['code', {}, 'Use `code` in your Markdown file.']]]) + }) + + it('closes an unclosed double-backtick span with a matching run', async () => { + const tree = await heal('``{ modelValue: _Number }') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, ['code', {}, '{ modelValue: _Number }']]]) + }) + }) + + describe('multiple openers on one line', () => { + it('emits one closer for a repeated marker instead of nesting em in em', async () => { + const tree = await heal('a _b and _c') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a _b and ', ['em', {}, 'c']]]) + }) + + it('collapses a repeated strong marker', async () => { + const tree = await heal('a __b and __c') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a __b and ', ['strong', {}, 'c']]]) + }) + + it('collapses a repeated strikethrough marker', async () => { + const tree = await heal('a ~~b and ~~c') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a ~~b and ', ['del', {}, 'c']]]) + }) + + it('still nests markers of different families', async () => { + const tree = await heal('_a **b _c') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, ['em', {}, 'a ', ['strong', {}, 'b ', ['em', {}, 'c']]]]]) + }) + + it('still closes a strong opened inside an em', async () => { + const tree = await heal('a _b and __c') + expect(tree.nodes).toEqual([['p', { $: { line: 1 } }, 'a ', ['em', {}, 'b and ', ['strong', {}, 'c']]]]) + }) + }) +}) diff --git a/packages/comark/test/index.test.ts b/packages/comark/test/index.test.ts index 20819467..9e9b2678 100644 --- a/packages/comark/test/index.test.ts +++ b/packages/comark/test/index.test.ts @@ -246,6 +246,8 @@ describe('Comark Tests', () => { ) } + // Fixtures never stream, so the `'streaming'` autoClose default leaves + // input untouched and the corpus reads as a conformance suite. const parseOptions: ParserOptions = { autoUnwrap: testCase.options?.autoUnwrap === false ? false : true, } diff --git a/packages/comark/test/nested-component-blank-lines.test.ts b/packages/comark/test/nested-component-blank-lines.test.ts index 0570afdf..7b5adb66 100644 --- a/packages/comark/test/nested-component-blank-lines.test.ts +++ b/packages/comark/test/nested-component-blank-lines.test.ts @@ -82,7 +82,7 @@ describe('nested component with a run of blank lines between siblings', () => { ) it('still terminates at the next `#slot` marker regardless of a preceding blank run', async () => { - const src = '::outer\n #title\n Hello\n\n\n #footer\n Bye\n ::\n::' + const src = '::outer\n #title\n Hello\n\n\n #footer\n Bye\n ::' const tree = await parseMarkdown(src) expect(tree.nodes).toEqual([ ['outer', {}, ['template', { name: 'title' }, 'Hello'], ['template', { name: 'footer' }, 'Bye']], @@ -90,7 +90,7 @@ describe('nested component with a run of blank lines between siblings', () => { }) it('still terminates at the parent close after a blank run (no over-absorption)', async () => { - const src = '::outer\n #title\n Hello\n\n\n ::\nafter\n::' + const src = '::outer\n #title\n Hello\n\n\n ::\nafter' const tree = await parseMarkdown(src) expect(tree.nodes).toEqual([ ['outer', {}, ['template', { name: 'title' }, 'Hello']], diff --git a/packages/comark/test/perf.test.ts b/packages/comark/test/perf.test.ts index c53223e2..cfa678cf 100644 --- a/packages/comark/test/perf.test.ts +++ b/packages/comark/test/perf.test.ts @@ -72,7 +72,9 @@ hi describe('ParserOptions.tracer', () => { it('records phase and per-plugin spans in pipeline order', async () => { const { tracer, spans } = createRecorder() - const parse = createMarkdownParser({ tracer }) + // `autoClose: true` forces healing on a non-streaming parse so the + // `comark:autoclose` span is recorded; the default only heals while streaming. + const parse = createMarkdownParser({ tracer, autoClose: true }) const tree = await parse(markdown) expect(tree.frontmatter).toEqual({ title: 'Hello' }) diff --git a/packages/comark/test/plugins/default-plugins.test.ts b/packages/comark/test/plugins/default-plugins.test.ts index d47605e5..9d852119 100644 --- a/packages/comark/test/plugins/default-plugins.test.ts +++ b/packages/comark/test/plugins/default-plugins.test.ts @@ -101,7 +101,9 @@ describe('default plugin options', () => { }) it('still auto-closes markdown markers when disabled', async () => { - const tree = await parseMarkdown('**bold', { registerDefaultPlugins: false }) + // The assertion is that `registerDefaultPlugins: false` does not turn healing + // off, so ask for healing explicitly: the default only heals while streaming. + const tree = await parseMarkdown('**bold', { registerDefaultPlugins: false, autoClose: true }) expect(tree.nodes).toEqual([['p', {}, ['strong', {}, 'bold']]]) }) }) diff --git a/test/bundle.test.ts b/test/bundle.test.ts index 6513ed58..966209eb 100644 --- a/test/bundle.test.ts +++ b/test/bundle.test.ts @@ -64,10 +64,10 @@ describe('package bundle size', { timeout: 60_000 }, () => { "@comark/ansi": "36.6k (98 files)", "@comark/html": "15.7k (58 files)", "@comark/nuxt": "11.8k (58 files)", - "@comark/react": "36.8k (74 files)", + "@comark/react": "36.9k (74 files)", "@comark/svelte": "43.9k (82 files)", "@comark/vue": "54.7k (78 files)", - "comark": "364k (158 files)", + "comark": "367k (158 files)", } `) })