diff --git a/Sources/MarkdownUtilities/FrontMatter/FrontMatterParser.swift b/Sources/MarkdownUtilities/FrontMatter/FrontMatterParser.swift index 5f3a85f..550fc00 100644 --- a/Sources/MarkdownUtilities/FrontMatter/FrontMatterParser.swift +++ b/Sources/MarkdownUtilities/FrontMatter/FrontMatterParser.swift @@ -41,6 +41,31 @@ struct FrontMatterParser: Parsing.Parser { } } + /// Returns true if the raw YAML string contains any YAML comments. + /// + /// This is a naive check that detects standalone comment lines — lines where + /// the first non-whitespace character is `#`. It does not detect inline + /// comments (e.g. `key: value # comment`). + /// + /// Detection runs on the raw frontmatter string before Yams parses it, + /// because Yams (via libYAML) discards comments and they are unrecoverable + /// from the parsed AST. + static func containsYAMLComments(_ rawYAML: String) -> Bool { + // Use swift-parsing's Prefix to consume leading horizontal whitespace, + // then check if the first remaining character is '#'. + // This catches standalone comment lines; inline comments (key: value # …) + // are out of scope for this naive check. + let whitespace = Prefix(while: { $0 == " " || $0 == "\t" }) + for line in rawYAML.split(separator: "\n", omittingEmptySubsequences: false) { + var lineInput = line[...] + _ = try? whitespace.parse(&lineInput) + if lineInput.first == "#" { + return true + } + } + return false + } + /// Parser that extracts only the frontmatter content (between delimiters) private var frontMatterOnlyParser: some Parsing.Parser { Parse { diff --git a/Sources/MarkdownUtilities/MarkdownDocument.swift b/Sources/MarkdownUtilities/MarkdownDocument.swift index eb15087..8bdc410 100644 --- a/Sources/MarkdownUtilities/MarkdownDocument.swift +++ b/Sources/MarkdownUtilities/MarkdownDocument.swift @@ -21,6 +21,15 @@ public struct MarkdownDocument: @unchecked Sendable { /// otherwise it contains the entire document content. public var body: String + /// Whether the original frontmatter source contained YAML comments. + /// + /// When `true`, any write operation via `md-utils` will silently discard those comments. + /// This is a known limitation: Yams (via libYAML) strips comments before they can be + /// stored in the parsed AST. See `docs/architecture.md` for details. + /// + /// Only set by `init(content:)` — always `false` when constructing programmatically. + public let containsYAMLComments: Bool + /// Initialize a markdown document by parsing the content to separate frontmatter from body. /// /// This initializer uses `FrontMatterParser` to detect and separate YAML frontmatter @@ -36,6 +45,7 @@ public struct MarkdownDocument: @unchecked Sendable { self.frontMatter = try YAMLConversion.parse(rawFrontMatter) self.body = body + self.containsYAMLComments = FrontMatterParser.containsYAMLComments(rawFrontMatter) } /// Initialize a markdown document directly from its parsed components. @@ -49,6 +59,7 @@ public struct MarkdownDocument: @unchecked Sendable { public init(frontMatter: Yams.Node.Mapping, body: String) { self.frontMatter = frontMatter self.body = body + self.containsYAMLComments = false } /// Parse the body text into a Markdown AST. diff --git a/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift b/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift index 9817357..090de01 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift @@ -67,6 +67,10 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { let content: String = try path.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(path): frontmatter contains YAML comments which will be lost\n", stderr) + } + // Get array (creates empty if doesn't exist, errors if not an array) let sequence = try ArrayHelpers.getOrCreateArrayKey(key, in: doc, path: path) diff --git a/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift b/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift index 984bf36..4bb9062 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift @@ -68,6 +68,10 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { let content: String = try path.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(path): frontmatter contains YAML comments which will be lost\n", stderr) + } + // Get array (creates empty if doesn't exist, errors if not an array) let sequence = try ArrayHelpers.getOrCreateArrayKey(key, in: doc, path: path) diff --git a/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift b/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift index ba22d6c..71e0bde 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift @@ -62,6 +62,10 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { let content: String = try path.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(path): frontmatter contains YAML comments which will be lost\n", stderr) + } + // Validate array exists let sequence = try ArrayHelpers.validateArrayKey(key, in: doc, path: path) diff --git a/Sources/md-utils/FrontMatterCommands/Remove.swift b/Sources/md-utils/FrontMatterCommands/Remove.swift index cc29708..619dd57 100644 --- a/Sources/md-utils/FrontMatterCommands/Remove.swift +++ b/Sources/md-utils/FrontMatterCommands/Remove.swift @@ -42,6 +42,10 @@ extension CLIEntry.FrontMatterCommands { let content: String = try file.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(file): frontmatter contains YAML comments which will be lost\n", stderr) + } + doc.removeValue(forKey: key) let updated = try doc.render() diff --git a/Sources/md-utils/FrontMatterCommands/Rename.swift b/Sources/md-utils/FrontMatterCommands/Rename.swift index 72cf575..5e90289 100644 --- a/Sources/md-utils/FrontMatterCommands/Rename.swift +++ b/Sources/md-utils/FrontMatterCommands/Rename.swift @@ -53,6 +53,10 @@ extension CLIEntry.FrontMatterCommands { let content: String = try file.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(file): frontmatter contains YAML comments which will be lost\n", stderr) + } + try doc.renameKey(from: key, to: newKey) let updated = try doc.render() diff --git a/Sources/md-utils/FrontMatterCommands/Replace.swift b/Sources/md-utils/FrontMatterCommands/Replace.swift index d42b44d..1975110 100644 --- a/Sources/md-utils/FrontMatterCommands/Replace.swift +++ b/Sources/md-utils/FrontMatterCommands/Replace.swift @@ -155,6 +155,10 @@ extension CLIEntry.FrontMatterCommands { let content: String = try path.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(path): frontmatter contains YAML comments which will be lost\n", stderr) + } + // Replace frontmatter (direct assignment) doc.frontMatter = newFrontMatter diff --git a/Sources/md-utils/FrontMatterCommands/Set.swift b/Sources/md-utils/FrontMatterCommands/Set.swift index 214e096..e3ff601 100644 --- a/Sources/md-utils/FrontMatterCommands/Set.swift +++ b/Sources/md-utils/FrontMatterCommands/Set.swift @@ -46,6 +46,10 @@ extension CLIEntry.FrontMatterCommands { let content: String = try file.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(file): frontmatter contains YAML comments which will be lost\n", stderr) + } + doc.setValue(value, forKey: key) let updated = try doc.render() diff --git a/Sources/md-utils/FrontMatterCommands/SortKeys.swift b/Sources/md-utils/FrontMatterCommands/SortKeys.swift index b25454f..35a5e1d 100644 --- a/Sources/md-utils/FrontMatterCommands/SortKeys.swift +++ b/Sources/md-utils/FrontMatterCommands/SortKeys.swift @@ -54,6 +54,10 @@ extension CLIEntry.FrontMatterCommands { let content: String = try file.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(file): frontmatter contains YAML comments which will be lost\n", stderr) + } + doc.sortKeys(by: method, reverse: reverse) let updated = try doc.render() diff --git a/Sources/md-utils/FrontMatterCommands/Touch.swift b/Sources/md-utils/FrontMatterCommands/Touch.swift index 0be211e..cfaac82 100644 --- a/Sources/md-utils/FrontMatterCommands/Touch.swift +++ b/Sources/md-utils/FrontMatterCommands/Touch.swift @@ -61,6 +61,10 @@ extension CLIEntry.FrontMatterCommands { let content: String = try file.read() var doc = try MarkdownDocument(content: content) + if doc.containsYAMLComments { + fputs("warning: \(file): frontmatter contains YAML comments which will be lost\n", stderr) + } + // Add each key if it doesn't exist for key in keyList { if !doc.hasKey(key) { diff --git a/Tests/MarkdownUtilitiesTests/FrontMatter/YAMLCommentDetectionTests.swift b/Tests/MarkdownUtilitiesTests/FrontMatter/YAMLCommentDetectionTests.swift new file mode 100644 index 0000000..08abd17 --- /dev/null +++ b/Tests/MarkdownUtilitiesTests/FrontMatter/YAMLCommentDetectionTests.swift @@ -0,0 +1,175 @@ +// +// YAMLCommentDetectionTests.swift +// MarkdownUtilitiesTests +// +// Tests for YAML comment detection in frontmatter. +// +// These tests cover FrontMatterParser.containsYAMLComments(_:) (the raw-string +// check) and MarkdownDocument.containsYAMLComments (the document-level property). +// + +import Testing +@testable import MarkdownUtilities + +// MARK: - FrontMatterParser.containsYAMLComments(_:) + +@Suite("FrontMatterParser.containsYAMLComments") +struct FrontMatterParserCommentDetectionTests { + + // MARK: True cases + + @Test + func `Standalone comment line is detected`() { + let yaml = """ + # This is a comment + title: Test + """ + #expect(FrontMatterParser.containsYAMLComments(yaml) == true) + } + + @Test + func `Indented comment line is detected`() { + let yaml = """ + title: Test + # indented comment + author: Jane + """ + #expect(FrontMatterParser.containsYAMLComments(yaml) == true) + } + + @Test + func `Tab-indented comment line is detected`() { + let yaml = "title: Test\n\t# tab-indented comment\nauthor: Jane" + #expect(FrontMatterParser.containsYAMLComments(yaml) == true) + } + + @Test + func `Comment at start of string is detected`() { + let yaml = "# first line is a comment\ntitle: Test" + #expect(FrontMatterParser.containsYAMLComments(yaml) == true) + } + + @Test + func `Comment as the only content is detected`() { + let yaml = "# just a comment" + #expect(FrontMatterParser.containsYAMLComments(yaml) == true) + } + + // MARK: False cases + + @Test + func `Comment-free YAML returns false`() { + let yaml = """ + title: Test + author: Jane + count: 42 + """ + #expect(FrontMatterParser.containsYAMLComments(yaml) == false) + } + + @Test + func `Empty string returns false`() { + #expect(FrontMatterParser.containsYAMLComments("") == false) + } + + @Test + func `Hash inside a string value is not a comment`() { + // These contain '#' but not as a standalone comment line + let yaml = """ + color: "#FF5733" + url: https://example.com/page#anchor + tags: + - hash#tag + """ + #expect(FrontMatterParser.containsYAMLComments(yaml) == false) + } + + @Test + func `Inline comment is not detected by naive check`() { + // Intentional limitation: inline comments are out of scope + let yaml = "title: Test # inline comment" + #expect(FrontMatterParser.containsYAMLComments(yaml) == false) + } +} + +// MARK: - MarkdownDocument.containsYAMLComments + +@Suite("MarkdownDocument.containsYAMLComments") +struct MarkdownDocumentCommentDetectionTests { + + @Test + func `Document with comment in frontmatter sets flag to true`() throws { + let content = """ + --- + # section header comment + title: Test + --- + Body + """ + let doc = try MarkdownDocument(content: content) + #expect(doc.containsYAMLComments == true) + } + + @Test + func `Document without comments sets flag to false`() throws { + let content = """ + --- + title: Test + author: Jane + --- + Body + """ + let doc = try MarkdownDocument(content: content) + #expect(doc.containsYAMLComments == false) + } + + @Test + func `Document with no frontmatter sets flag to false`() throws { + let doc = try MarkdownDocument(content: "Just body content") + #expect(doc.containsYAMLComments == false) + } + + @Test + func `Document with empty frontmatter sets flag to false`() throws { + let content = """ + --- + --- + Body + """ + let doc = try MarkdownDocument(content: content) + #expect(doc.containsYAMLComments == false) + } + + @Test + func `Programmatic init always sets flag to false`() { + let doc = MarkdownDocument(frontMatter: .init(), body: "Body") + #expect(doc.containsYAMLComments == false) + } + + @Test + func `Hash in string value does not set flag`() throws { + let content = """ + --- + color: "#FF5733" + url: https://example.com/#anchor + --- + Body + """ + let doc = try MarkdownDocument(content: content) + #expect(doc.containsYAMLComments == false) + } + + @Test + func `Comment in body does not affect flag`() throws { + // HTML comments in the body should not influence the frontmatter flag + let content = """ + --- + title: Test + --- + + Body content + """ + let doc = try MarkdownDocument(content: content) + #expect(doc.containsYAMLComments == false) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 33e334d..77695fb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -213,6 +213,27 @@ Select content by heading or line range. - **CLI**: `md-utils body` (extract body without frontmatter), `md-utils lines` (extract line ranges), `md-utils extract` (extract by section) +## Known Limitations + +### YAML Comment Loss on Frontmatter Write + +**Affects**: all `fm` write commands (`set`, `remove`, `rename`, `replace`, `sort-keys`, `touch`, `array append`, `array prepend`, `array remove`) + +YAML comments (lines beginning with `#`) in frontmatter are permanently lost whenever any write operation is performed. This is a fundamental limitation of the YAML parsing stack: + +- **Root cause**: Yams is built on libYAML, a streaming event-based parser. libYAML silently discards comment tokens — they never surface as parse events and therefore cannot be stored in `Yams.Node.Mapping` or any downstream structure. +- **No workaround within Yams**: Comments are unrecoverable from the parsed AST regardless of serialization settings. + +**Detection**: `MarkdownDocument.containsYAMLComments` is set to `true` at `init(content:)` time if `FrontMatterParser.containsYAMLComments(_:)` finds any comment lines in the raw frontmatter string (before Yams parses it). All write CLI commands emit a warning to stderr when this is true: + +``` +warning: path/to/file.md: frontmatter contains YAML comments which will be lost +``` + +**Scope of detection**: The check is intentionally naive — it detects standalone comment lines (where the first non-whitespace character on a line is `#`). It does **not** detect inline comments (`key: value # comment`). This tradeoff avoids false positives on YAML string values that legitimately contain `#` characters (e.g. URLs, hex color codes). + +**Possible future fix**: Replace Yams with a YAML library that preserves comments in its AST (uncommon), or implement raw-text frontmatter surgery that avoids a full parse/serialize round-trip. + ## Planned Features The following features are **NOT YET IMPLEMENTED**: