From 6558cbdb0a2a921287f0b5339ff6439081c3b644 Mon Sep 17 00:00:00 2001 From: Daniel Lyons <72824209+DandyLyons@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:59:51 -0500 Subject: [PATCH 1/3] Remove --include-non-md from fm dump --- .../Documentation.docc/FrontmatterCommands.md | 6 + .../md-utils/FrontMatterCommands/Dump.swift | 74 +++++++----- .../NonMarkdownFrontMatterSupport.swift | 30 ++++- Sources/md-utils/GlobalOptions.swift | 2 +- .../FrontMatterCommands/DumpTests.swift | 110 +++++++++++++++++- .../references/common-use-cases.md | 6 +- .../references/frontmatter.md | 17 ++- 7 files changed, 205 insertions(+), 40 deletions(-) diff --git a/Sources/md-utils/Documentation.docc/FrontmatterCommands.md b/Sources/md-utils/Documentation.docc/FrontmatterCommands.md index 6a8e12d..25678cc 100644 --- a/Sources/md-utils/Documentation.docc/FrontmatterCommands.md +++ b/Sources/md-utils/Documentation.docc/FrontmatterCommands.md @@ -50,3 +50,9 @@ literals from shell command substitution. ## Output Semantics Commands that report values preserve the distinction between a missing key and a key whose YAML value is null. Machine-readable formats should be preferred when that distinction matters. + +`fm dump` is read-only and automatically selects Markdown, plain-text, and +mapped non-Markdown files. A sole explicit file outputs its mapping directly. +Directory and explicit file-list invocations output an object whose +`frontMatter`, `noFrontMatter`, and `emptyFrontMatter` members distinguish +nonempty mappings, absent blocks, and complete blocks with empty mappings. diff --git a/Sources/md-utils/FrontMatterCommands/Dump.swift b/Sources/md-utils/FrontMatterCommands/Dump.swift index a844a22..9d8023a 100644 --- a/Sources/md-utils/FrontMatterCommands/Dump.swift +++ b/Sources/md-utils/FrontMatterCommands/Dump.swift @@ -19,7 +19,7 @@ extension CLIEntry.FrontMatterCommands { static let configuration = CommandConfiguration( commandName: "dump", abstract: "Dump entire frontmatter in specified format", - discussion: NonMarkdownFrontMatterHelp.appending(to: """ + discussion: NonMarkdownFrontMatterHelp.appendingForDump(to: """ Outputs the complete frontmatter from files in various formats: JSON, YAML, raw, or plist. Supports multiple files and directory processing with recursive mode. @@ -28,12 +28,14 @@ extension CLIEntry.FrontMatterCommands { Single-file dumps output the frontmatter directly with no wrapper. The --format flag controls the output format. - MULTIPLE FILES: - By default, multiple files output a single parseable collection (JSON array, - YAML sequence, or plist array) with a "$path" key injected into each entry. + COLLECTION MODE: + Directories and explicit file lists output a parseable object with three keys: + "frontMatter" contains nonempty mappings with a "$path" key, "noFrontMatter" + contains paths with no complete block, and "emptyFrontMatter" contains paths + whose complete block has an empty mapping. The --include-delimiters flag is ignored in collection mode. - Use --cat-headers for the legacy cat-style header format (==> path <==). + Use --cat-headers for the legacy cat-style format (==> path <==). Examples: # Dump single file as JSON (default) @@ -45,7 +47,7 @@ extension CLIEntry.FrontMatterCommands { # Dump with delimiters md-utils fm dump post.md --format yaml --include-delimiters - # Dump multiple files as JSON array with $path + # Dump multiple files as a categorized JSON object md-utils fm dump posts/ -r --format json # Dump multiple files with cat-style headers @@ -59,19 +61,19 @@ extension CLIEntry.FrontMatterCommands { into jq or yq for further filtering and transformation. # List all titles - md-utils fm dump posts/ -r | jq '.[].title' + md-utils fm dump posts/ -r | jq '.frontMatter[].title' # Find drafts - md-utils fm dump posts/ -r | jq '[.[] | select(.status == "draft")]' + md-utils fm dump posts/ -r | jq '[.frontMatter[] | select(.status == "draft")]' # Get paths of posts tagged "swift" - md-utils fm dump posts/ -r | jq '[.[] | select(.tags | index("swift")) | ."$path"]' + md-utils fm dump posts/ -r | jq '[.frontMatter[] | select(.tags | index("swift")) | ."$path"]' # Same with yq (YAML output) - md-utils fm dump posts/ -r --format yaml | yq '.[].title' + md-utils fm dump posts/ -r --format yaml | yq '.frontMatter[].title' # Count entries - md-utils fm dump posts/ -r | jq 'length' + md-utils fm dump posts/ -r | jq '.frontMatter | length' """), aliases: ["d"] ) @@ -87,25 +89,24 @@ extension CLIEntry.FrontMatterCommands { @Flag(name: .long, help: "Use cat-style headers (==> path <==) instead of collection output for multiple files") var catHeaders: Bool = false - @Flag(name: .long, help: "Process mapped non-Markdown files") - var includeNonMD = false /// Runs the command using the parsed command-line arguments. /// /// See for workflow details. mutating func run() async throws { - let files = try options.resolvedFrontMatterPaths(includeNonMarkdown: includeNonMD) + let files = try options.resolvedFrontMatterPaths(includeNonMarkdown: true) guard !files.isEmpty else { - throw ValidationError("No Markdown files found to process") + throw ValidationError("No supported files found to process") } - let isMultipleFiles = files.count > 1 + let isSingleExplicitFile = options.paths.count == 1 && options.paths[0].isFile - // Single file: output directly - if !isMultipleFiles { + // A sole explicit file outputs directly. Directory scans and explicit file + // lists keep the collection envelope even when only one file is selected. + if isSingleExplicitFile { let file = files[0] do { - let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: includeNonMD) + let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: true) if includeDelimiters && (format == .yaml || format == .raw) { Swift.print("---") @@ -123,14 +124,14 @@ extension CLIEntry.FrontMatterCommands { return } - // Multiple files + // Collection-oriented invocation var hasErrors = false if catHeaders { // Cat-style output with headers for (index, file) in files.enumerated() { Swift.print("==> \(file) <==") do { - let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: includeNonMD) + let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: true) if includeDelimiters && (format == .yaml || format == .raw) { Swift.print("---") @@ -152,26 +153,47 @@ extension CLIEntry.FrontMatterCommands { } } } else { - // Collection mode: single parseable document with $path metadata - var collection: [[String: Any]] = [] + // Collection mode: categorize files by frontmatter state. + var frontMatter: [[String: Any]] = [] + var noFrontMatter: [String] = [] + var emptyFrontMatter: [String] = [] for file in files { do { - let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: includeNonMD) - let node = Yams.Node.mapping(doc.frontMatter) + let parsed = try FrontMatterCLIMutator.parsedFile( + at: file, + includeNonMarkdown: true + ) + + guard parsed.hasFrontMatterBlock else { + noFrontMatter.append(file.string) + continue + } + + guard parsed.document.frontMatter.isEmpty == false else { + emptyFrontMatter.append(file.string) + continue + } + + let node = Yams.Node.mapping(parsed.document.frontMatter) guard var dict = try YAMLConversion.safeNodeToSwiftValue(node) as? [String: Any] else { continue } dict["$path"] = file.string - collection.append(dict) + frontMatter.append(dict) } catch { CLIStyle.writeError("\(CLIStyle.path(file.string)): \(error.localizedDescription)") hasErrors = true } } + let collection: [String: Any] = [ + "frontMatter": frontMatter, + "noFrontMatter": noFrontMatter, + "emptyFrontMatter": emptyFrontMatter, + ] try printAny(collection, format: format) } if hasErrors { throw ExitCode.failure } diff --git a/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift b/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift index 8b8ac92..df6de82 100644 --- a/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift +++ b/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift @@ -14,8 +14,16 @@ enum NonMarkdownFrontMatterHelp { discussion + "\n\n" + section } - /// The exact syntax and selection contract shown on relevant help pages. - private static let section = """ + /// Appends the wrapped-frontmatter contract used by `fm dump`. + /// + /// Dump is read-only, so it discovers every shipped syntax without requiring + /// the opt-in used by batch mutation commands. + static func appendingForDump(to discussion: String) -> String { + discussion + "\n\n" + dumpSection + } + + /// The syntax contract shared by frontmatter command help pages. + private static let syntaxContract = """ FRONTMATTER ON NON-MD FILES A supported non-Markdown file uses the wrapper mapped from its extension. A complete wrapped frontmatter block must contain, on separate complete LF @@ -30,12 +38,26 @@ enum NonMarkdownFrontMatterHelp { The block may occur anywhere, though placement near the beginning is recommended. Incomplete blocks are treated as absent; multiple complete - blocks are invalid. Plain .txt uses ordinary Markdown-style frontmatter and - only participates with --include-non-md. + blocks are invalid. Plain .txt uses ordinary Markdown-style frontmatter. + """ + + /// The exact opt-in selection contract shown on relevant help pages. + private static let section = syntaxContract + """ + + + Plain .txt only participates with --include-non-md. A sole explicit mapped file is selected automatically. Multi-file and directory operations require --include-non-md for mapped files. """ + + /// The exact syntax and automatic-selection contract shown by `fm dump`. + private static let dumpSection = syntaxContract + """ + + + Dump automatically selects Markdown, plain-text, and mapped non-Markdown + files in explicit file lists and directory operations. + """ } /// Describes how a selected file represents frontmatter. diff --git a/Sources/md-utils/GlobalOptions.swift b/Sources/md-utils/GlobalOptions.swift index dc74547..939bd0d 100644 --- a/Sources/md-utils/GlobalOptions.swift +++ b/Sources/md-utils/GlobalOptions.swift @@ -35,7 +35,7 @@ struct GlobalOptions: ParsableArguments { /// File extensions to process (comma-separated). @Option( name: .long, - help: "File extensions to process (comma-separated, default: md,markdown)" + help: "File extensions to process (comma-separated; default: md,markdown, while fm dump scans all supported extensions)" ) var extensions: String = "md,markdown" diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift index 2716997..6d32dc8 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift @@ -60,7 +60,7 @@ struct DumpTests { ]) var command = try #require(command_ as? CLIEntry.FrontMatterCommands.Dump) - // Should succeed — outputs JSON array with $path + // Should succeed — outputs a categorized JSON object. try await command.run() } @@ -278,6 +278,106 @@ struct DumpTests { try await command.run() } + @Test + func `collection categorizes Markdown text and mapped files without opt-in`() throws { + let tempDir = try createProjectTempDirectory(prefix: "md-utils-dump-categories") + defer { try? tempDir.delete() } + + try (tempDir + "filled.md").write(""" + --- + title: Markdown + --- + Body + """) + try (tempDir + "empty.md").write(""" + --- + --- + Body + """) + try (tempDir + "missing.md").write("# No frontmatter\n") + try (tempDir + "Source.swift").write(""" + /* + --- + title: Swift + --- + */ + import Foundation + """) + try (tempDir + "Empty.swift").write(""" + /* + --- + {} + --- + */ + """) + try (tempDir + "notes.txt").write(""" + --- + title: Text + --- + Notes + """) + try (tempDir + "plain.txt").write("Plain text without frontmatter\n") + + let result = try CLIProcessTestHelper.run(["fm", "dump", tempDir.string]) + + #expect(result.status == 0, "Command failed: \(result.standardError)") + let data = try #require(result.standardOutput.data(using: .utf8)) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + let frontMatter = try #require(object["frontMatter"] as? [[String: Any]]) + let noFrontMatter = try #require(object["noFrontMatter"] as? [String]) + let emptyFrontMatter = try #require(object["emptyFrontMatter"] as? [String]) + + #expect(Set(frontMatter.compactMap { $0["title"] as? String }) == ["Markdown", "Swift", "Text"]) + #expect(Set(noFrontMatter.map { Path($0).lastComponent }) == ["missing.md", "plain.txt"]) + #expect(Set(emptyFrontMatter.map { Path($0).lastComponent }) == ["empty.md", "Empty.swift"]) + } + + @Test + func `single text file dumps frontmatter without opt-in`() throws { + let tempDir = try createProjectTempDirectory(prefix: "md-utils-dump-text") + defer { try? tempDir.delete() } + let file = tempDir + "notes.txt" + try file.write(""" + --- + title: Plain Text + --- + Notes + """) + + let result = try CLIProcessTestHelper.run(["fm", "dump", file.string]) + + #expect(result.status == 0, "Command failed: \(result.standardError)") + let data = try #require(result.standardOutput.data(using: .utf8)) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect(object["title"] as? String == "Plain Text") + } + + @Test + func `directory with one file still uses collection envelope`() throws { + let tempDir = try createProjectTempDirectory(prefix: "md-utils-dump-single-directory") + defer { try? tempDir.delete() } + let file = tempDir + "missing.md" + try file.write("# No frontmatter\n") + + let result = try CLIProcessTestHelper.run(["fm", "dump", tempDir.string]) + + #expect(result.status == 0, "Command failed: \(result.standardError)") + let data = try #require(result.standardOutput.data(using: .utf8)) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect((object["frontMatter"] as? [[String: Any]])?.isEmpty == true) + #expect((object["emptyFrontMatter"] as? [String])?.isEmpty == true) + let noFrontMatter = try #require(object["noFrontMatter"] as? [String]) + #expect(noFrontMatter == [file.string]) + } + + @Test + func `include-non-md is not accepted by dump`() throws { + let result = try CLIProcessTestHelper.run(["fm", "dump", "--help"]) + + #expect(result.status == 0) + #expect(result.standardOutput.contains("--include-non-md") == false) + } + // MARK: - Test Helpers private func createTempFile(content: String, name: String) throws -> Path { @@ -286,4 +386,12 @@ struct DumpTests { try tempFile.write(content) return tempFile } + + private func createProjectTempDirectory(prefix: String) throws -> Path { + let tempRoot = Path(FileManager.default.currentDirectoryPath) + "tmp/" + try tempRoot.mkpath() + let directory = tempRoot + "\(prefix)-\(UUID().uuidString)/" + try directory.mkpath() + return directory + } } diff --git a/skill/markdown-utilities/skills/markdown-utilities/references/common-use-cases.md b/skill/markdown-utilities/skills/markdown-utilities/references/common-use-cases.md index 9618a16..14b31c1 100644 --- a/skill/markdown-utilities/skills/markdown-utilities/references/common-use-cases.md +++ b/skill/markdown-utilities/skills/markdown-utilities/references/common-use-cases.md @@ -61,16 +61,16 @@ md-utils headings promote --index 3 document.md --in-place ```bash # Get all titles across a directory -md-utils fm dump posts/ | jq -r '.[].title' +md-utils fm dump posts/ | jq -r '.frontMatter[].title' # Dump all frontmatter as YAML md-utils fm dump posts/ --format yaml # Count published posts -md-utils fm dump posts/ | jq '[.[] | select(.status == "published")] | length' +md-utils fm dump posts/ | jq '[.frontMatter[] | select(.status == "published")] | length' # Get unique authors -md-utils fm dump posts/ | jq -r '.[].author' | sort -u +md-utils fm dump posts/ | jq -r '.frontMatter[].author' | sort -u # Extract specific lines with line numbers md-utils lines document.md -s 1 -e 50 --numbered diff --git a/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md b/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md index 6a3bf2a..6e97fa9 100644 --- a/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md +++ b/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md @@ -70,14 +70,14 @@ md-utils fm dump post.md # YAML format md-utils fm dump post.md --format yaml -# Multiple files: outputs JSON array with "$path" key injected +# Multiple files: categorizes populated, absent, and empty frontmatter md-utils fm dump posts/ --format json # Pipe to jq -md-utils fm dump posts/ | jq '.[].title' +md-utils fm dump posts/ | jq '.frontMatter[].title' # Pipe to yq -md-utils fm dump posts/ --format yaml | yq '.[].title' +md-utils fm dump posts/ --format yaml | yq '.frontMatter[].title' # Cat-style headers (legacy) md-utils fm dump posts/ --cat-headers @@ -85,6 +85,13 @@ md-utils fm dump posts/ --cat-headers **Formats:** `json` (default), `yaml`, `raw`, `plist` +`fm dump` automatically processes Markdown, `.txt`, and mapped non-Markdown +files; it does not accept `--include-non-md`. Multi-file output is an object: + +- `frontMatter` contains nonempty mappings with a `$path` key. +- `noFrontMatter` contains paths with no complete frontmatter block. +- `emptyFrontMatter` contains paths whose complete block has an empty mapping. + ## Search with JMESPath `fm search` filters files using a JMESPath expression evaluated against each file's frontmatter. Outputs matching file paths. @@ -158,10 +165,10 @@ md-utils fm array contains --key tags --value swift posts/ \ | xargs -I {} sh -c 'md-utils fm array contains --key tags --value tutorial {} && echo {}' # List all unique authors across a directory -md-utils fm dump posts/ | jq -r '.[].author' | sort -u +md-utils fm dump posts/ | jq -r '.frontMatter[].author' | sort -u # Count published posts -md-utils fm dump posts/ | jq '[.[] | select(.status == "published")] | length' +md-utils fm dump posts/ | jq '[.frontMatter[] | select(.status == "published")] | length' # Find files missing a required key find posts/ -name "*.md" | xargs -I {} sh -c 'md-utils fm has --key author {} || echo {}' From a3bfff7464204be913226c23302492a400920102 Mon Sep 17 00:00:00 2001 From: Daniel Lyons <72824209+DandyLyons@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:10:17 -0500 Subject: [PATCH 2/3] feat: add TOML frontmatter support --- AGENTS.md | 4 +- IntegrationTests/WasmCoreSmoke/main.swift | 17 ++ Package.resolved | 11 +- Package.swift | 2 + README.md | 10 +- .../FormatConversion/CSV/CSVConverter.swift | 44 ++-- .../FormatConversion/CSV/CSVOptions.swift | 2 +- .../MarkdownTypeFileRegistryLoader.swift | 3 + .../FrontmatterWorkflows.md | 18 +- .../Documentation.docc/MarkdownTypes.md | 6 +- .../Documentation.docc/RecordIdentity.md | 2 +- .../WorkingWithMarkdownDocuments.md | 6 +- .../Explore/ExploreDocument.swift | 12 +- .../MarkdownDocument+FormatConversion.swift | 14 +- .../PlainText/PlainTextOptions.swift | 4 +- .../Protocols/ConversionOptions.swift | 2 +- .../MarkdownDocument+Formatting.swift | 6 +- .../FrontMatter/FrontMatter.swift | 204 +++++++++++++++++ .../FrontMatter/FrontMatterConversion.swift | 207 ++++++++++++++++++ .../FrontMatter/FrontMatterParser.swift | 67 +++--- .../MarkdownDocument+FrontMatter.swift | 17 +- ...MarkdownDocument+FrontMatterMutation.swift | 30 +-- .../FrontMatter/WrappedFrontMatter.swift | 38 ++-- .../MarkdownDocument+HeadingAdjustment.swift | 21 +- .../MarkdownDocument.swift | 32 ++- .../Rules/MarkdownRuleChecker.swift | 5 +- .../MarkdownDocument+SectionExtraction.swift | 21 +- .../MarkdownDocument+SectionInsertion.swift | 16 +- .../MarkdownDocument+SectionReplacement.swift | 18 +- .../MarkdownDocument+SectionReordering.swift | 18 +- .../Types/MarkdownRecordAnalyzer.swift | 37 ++-- .../Types/MarkdownTypeDefinitionDecoder.swift | 9 + .../Types/MarkdownTypeFixer.swift | 50 +++-- .../Wikilink/MarkdownDocument+Wikilink.swift | 26 +-- Sources/md-utils/Commands/Body.swift | 4 +- .../md-utils/Commands/ExtractSection.swift | 15 +- Sources/md-utils/Commands/FormatCommand.swift | 4 +- Sources/md-utils/ConvertCommands/ToCSV.swift | 2 +- Sources/md-utils/ConvertCommands/ToText.swift | 2 +- .../Documentation.docc/FrontmatterCommands.md | 12 +- .../RulesValidationCommands.md | 6 +- .../Documentation.docc/TypesCommands.md | 5 +- .../md-utils/ExploreCommands/Explore.swift | 2 +- .../FrontMatterCommands/ArrayAppend.swift | 6 +- .../FrontMatterCommands/ArrayCommands.swift | 2 +- .../FrontMatterCommands/ArrayContains.swift | 3 +- .../FrontMatterCommands/ArrayHelpers.swift | 37 ++-- .../FrontMatterCommands/ArrayPrepend.swift | 6 +- .../FrontMatterCommands/ArrayRemove.swift | 2 +- .../md-utils/FrontMatterCommands/Dump.swift | 41 ++-- .../FrontMatterCommands.swift | 8 +- .../FrontMatterJMESPath.swift | 6 +- .../md-utils/FrontMatterCommands/Get.swift | 53 ++--- .../md-utils/FrontMatterCommands/List.swift | 6 +- .../NonMarkdownFrontMatterSupport.swift | 38 +++- .../FrontMatterCommands/Replace.swift | 25 ++- .../md-utils/FrontMatterCommands/Search.swift | 9 +- .../md-utils/FrontMatterCommands/Set.swift | 4 + .../md-utils/FrontMatterCommands/Touch.swift | 4 + .../md-utils/FrontMatterCommands/Unique.swift | 7 +- .../HeadingCommands/DemoteHeading.swift | 15 +- .../HeadingCommands/PromoteHeading.swift | 15 +- Sources/md-utils/OKFCommands/OKFSupport.swift | 16 +- Sources/md-utils/OutputFormat.swift | 35 +++ Sources/md-utils/Resources/SKILL.md | 7 +- .../md-utils/RulesCommands/RulesSupport.swift | 17 +- .../SectionCommands/InsertSection.swift | 11 +- .../SectionCommands/MoveSectionDown.swift | 12 +- .../SectionCommands/MoveSectionTo.swift | 12 +- .../SectionCommands/MoveSectionUp.swift | 12 +- .../SectionCommands/RemoveSection.swift | 11 +- .../md-utils/SectionCommands/SetSection.swift | 12 +- .../TypesCommands/TypesSubcommands.swift | 18 +- .../md-utils/TypesCommands/TypesSupport.swift | 28 ++- .../Formatting/FormattingTests.swift | 5 +- .../FrontMatterEdgeCasesTests.swift | 24 +- .../FrontMatterMutationTests.swift | 72 +++--- .../FrontMatter/FrontMatterParsingTests.swift | 4 +- .../FrontMatterSeparationTests.swift | 10 +- .../FrontMatter/TOMLFrontMatterTests.swift | 139 ++++++++++++ .../WrappedFrontMatterParserTests.swift | 9 + .../MarkdownASTTests.swift | 4 +- .../Types/MarkdownTypeDefinitionTests.swift | 27 +++ .../ArrayAppendTests.swift | 31 ++- .../ArrayPrependTests.swift | 7 +- .../ArrayRemoveTests.swift | 7 +- .../FrontMatterCommands/DumpTests.swift | 35 +++ .../FrontMatterCommands/RenameTests.swift | 26 +-- .../FrontMatterCommands/ReplaceTests.swift | 52 +++-- .../FrontMatterCommands/SearchTests.swift | 23 ++ .../FrontMatterCommands/SetTests.swift | 33 ++- .../FrontMatterCommands/SortKeysTests.swift | 40 ++-- .../Commands/TypesCommandsTests.swift | 18 ++ .../expected/replace.swift | 2 +- docs/architecture.md | 23 +- docs/common-use-cases.md | 20 +- docs/portability-audit.md | 3 +- docs/rfcs/0001-mdtype.md | 5 +- docs/webassembly.md | 7 +- scripts/build-wasm.sh | 18 +- .../wasm-patches/swift-toml-2.0.0-wasi.patch | 46 ++++ .../skills/markdown-utilities/SKILL.md | 7 +- .../references/frontmatter.md | 18 +- 103 files changed, 1545 insertions(+), 679 deletions(-) create mode 100644 Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatter.swift create mode 100644 Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterConversion.swift create mode 100644 Tests/MarkdownUtilitiesCoreTests/FrontMatter/TOMLFrontMatterTests.swift create mode 100644 scripts/wasm-patches/swift-toml-2.0.0-wasi.patch diff --git a/AGENTS.md b/AGENTS.md index 296e487..210ed65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ md-utils is a Swift package for parsing and manipulating Markdown files. It cons ## Project Brief - **Language**: Swift 6.2+ -- **Frameworks/Libraries**: Foundation, MarkdownSyntax, swift-parsing, PathKit, Yams, JMESPath, JSONSchema.swift, swift-argument-parser, Rainbow +- **Frameworks/Libraries**: Foundation, MarkdownSyntax, swift-parsing, PathKit, Yams, swift-toml, JMESPath, JSONSchema.swift, swift-argument-parser, Rainbow - **Package Manager / Build Tool**: Swift Package Manager - **CLI Target**: `md-utils` - **Library Targets**: `MarkdownUtilitiesCore`, `MarkdownUtilities` @@ -21,7 +21,7 @@ md-utils is a Swift package for parsing and manipulating Markdown files. It cons - **Test Command**: `swift test` - **Formatter/Linter**: No dedicated formatter or linter is configured in-package - **Documentation**: README.md, AGENTS.md, docs/*.md, generated CLI help, and bundled Agent Skill docs -- **Security**: Avoid unsafe optional force unwraps; treat filesystem and YAML/JSON parsing failures as user-visible errors +- **Security**: Avoid unsafe optional force unwraps; treat filesystem and YAML/TOML/JSON parsing failures as user-visible errors - **CI/Coverage**: No project-specific CI or coverage command is documented in this repo ## Requirements diff --git a/IntegrationTests/WasmCoreSmoke/main.swift b/IntegrationTests/WasmCoreSmoke/main.swift index dad0f9b..b06acc9 100644 --- a/IntegrationTests/WasmCoreSmoke/main.swift +++ b/IntegrationTests/WasmCoreSmoke/main.swift @@ -4,6 +4,7 @@ enum WasmCoreSmokeError: Error { case frontmatterNotParsed case emptyAST case renderMismatch + case tomlMismatch case typeAssessmentFailed } @@ -39,6 +40,22 @@ struct WasmCoreSmoke { throw WasmCoreSmokeError.renderMismatch } + let tomlDocument = try MarkdownDocument(content: """ + +++ + title = "WebAssembly TOML" + tags = ["swift", "wasm"] + +++ + # TOML + """) + let tomlRendered = try tomlDocument.render() + guard tomlDocument.frontMatterFormat == .toml, + tomlDocument.frontMatter["tags"]?.sequence?.count == 2, + tomlRendered.hasPrefix("+++\n"), + tomlRendered.contains("title = \"WebAssembly TOML\"") + else { + throw WasmCoreSmokeError.tomlMismatch + } + let definition = MarkdownTypeDefinition( name: MarkdownTypeName(rawValue: "WasmDocument"), version: "smoke", diff --git a/Package.resolved b/Package.resolved index 37adb53..3bb150b 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "7fc1c58cb7b389f547361da99ecadc597f9dbcbc3c638db56226030e12569835", + "originHash" : "b958849452099a703c0fa144f682a70c8c022f42586f75f3b038e9cb85799aa4", "pins" : [ { "identity" : "jmespath.swift", @@ -118,6 +118,15 @@ "version" : "602.0.0" } }, + { + "identity" : "swift-toml", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/swift-toml.git", + "state" : { + "revision" : "827506c90475e82d5a7f191f950fb3025cbdc0d6", + "version" : "2.0.0" + } + }, { "identity" : "xctest-dynamic-overlay", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 9e4163c..6fdfd2b 100644 --- a/Package.swift +++ b/Package.swift @@ -33,6 +33,7 @@ let package = Package( .package(url: "https://github.com/kylef/PathKit", from: "1.0.1"), .package(url: "https://github.com/kylef/JSONSchema.swift", from: "0.6.0"), .package(url: "https://github.com/jpsim/Yams.git", from: "6.1.0"), + .package(url: "https://github.com/mattt/swift-toml.git", from: "2.0.0"), .package(url: "https://github.com/adam-fowler/jmespath.swift.git", from: "1.0.3"), .package(url: "https://github.com/onevcat/Rainbow", from: "4.2.1"), .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.4.0"), @@ -46,6 +47,7 @@ let package = Package( .product(name: "Parsing", package: "swift-parsing"), .product(name: "JSONSchema", package: "JSONSchema.swift"), "Yams", + .product(name: "TOML", package: "swift-toml"), ] ), .testTarget( diff --git a/README.md b/README.md index 7287879..34801e6 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ This project is on a `0.x.x` release and is **not yet API stable**. The API and - **Heading Manipulation** — Promote/demote headings while maintaining nested structure - **Section Operations** — Extract sections by name or index; reorder sections (move up/down/to position) - **Content Selection** — Extract body without frontmatter, select by line range, extract by section -- **YAML Front Matter** — Full CRUD operations with 12+ subcommands including get, set, remove, rename, search (JMESPath), sort keys, array manipulation, and multi-format dump (JSON, YAML, raw, PropertyList) +- **YAML and TOML Front Matter** — Format-preserving CRUD, array manipulation, and multi-format output. JMESPath `fm search` remains YAML-only. - **Format Conversion** — Convert Markdown to plain text or CSV - **File Metadata** — Read file metadata including standard and extended attributes (xattr) - **Wikilink Parsing & Resolution** — Parse Obsidian-flavored wikilinks, resolve against a vault directory, detect broken/ambiguous links, find backlinks @@ -151,6 +151,9 @@ swift run md-utils fm get --key title posts/ | jq '.[] | select(has("value")) | # Set a frontmatter value swift run md-utils fm set --key tags --value "[swift, cli]" document.md +# Create TOML frontmatter in a document that has none +swift run md-utils fm set --key title --value "TOML Note" --frontmatter-format toml document.md + # Dump frontmatter as JSON swift run md-utils fm dump document.md @@ -383,11 +386,11 @@ selected automatically, except `.txt`, which requires `--include-non-md`. Mapped extensions use the same `c-block`, `html-comment`, `python-docstring`, `powershell-block`, and `lua-block` wrappers documented in [Frontmatter in Non-Markdown Text Files](docs/common-use-cases.md#frontmatter-in-non-markdown-text-files). -Wrapped YAML supports frontmatter predicates, JMESPath, type hints, and JSON +Wrapped YAML or TOML supports frontmatter predicates, JMESPath, type hints, and JSON Schema checks. Raw body predicates operate on wrapper-excluded host text, while Markdown headings, sections, and wikilinks are explicitly unsupported. -If a file matches multiple rules, all matching checks apply. Files matching no rules are ignored. Invalid YAML frontmatter is reported as an error for matched rules because frontmatter predicates and schema checks cannot proceed. +If a file matches multiple rules, all matching checks apply. Files matching no rules are ignored. Invalid YAML or TOML frontmatter is reported as an error for matched rules because frontmatter predicates and schema checks cannot proceed. ## GitHub Pages @@ -427,6 +430,7 @@ When the Pages workflow prepares its artifact, it copies `site/schemas/$CURRENT_ - [PathKit](https://github.com/kylef/PathKit) — File path handling - [JSONSchema.swift](https://github.com/kylef/JSONSchema.swift) — JSON Schema validation - [Yams](https://github.com/jpsim/Yams) — YAML parsing and serialization +- [swift-toml](https://github.com/mattt/swift-toml) — TOML parsing and serialization - [jmespath.swift](https://github.com/nicktmro/jmespath.swift) — JMESPath query language for JSON ## Platform Compatibility diff --git a/Sources/MarkdownUtilities/FormatConversion/CSV/CSVConverter.swift b/Sources/MarkdownUtilities/FormatConversion/CSV/CSVConverter.swift index bbadbb4..8c18470 100644 --- a/Sources/MarkdownUtilities/FormatConversion/CSV/CSVConverter.swift +++ b/Sources/MarkdownUtilities/FormatConversion/CSV/CSVConverter.swift @@ -3,7 +3,7 @@ import MarkdownUtilitiesCore import PathKit import Yams -/// Converts Markdown documents with YAML frontmatter to CSV format. +/// Converts Markdown documents with YAML or TOML frontmatter to CSV format. /// /// This converter takes a collection of Markdown documents and their file paths, /// extracts their frontmatter keys, and generates a CSV with one row per document. @@ -83,9 +83,7 @@ public struct CSVConverter { for (_, document) in documents { for (key, _) in document.frontMatter { - if case .scalar(let scalar) = key { - keySet.insert(scalar.string) - } + keySet.insert(key) } } @@ -194,34 +192,38 @@ public struct CSVConverter { /// Get a frontmatter value for a specific key func getFrontmatterValue(document: MarkdownDocument, key: String) throws -> String { // Look up the key in frontmatter - let keyNode = Yams.Node.scalar(.init(key)) - - guard let valueNode = document.frontMatter[keyNode] else { + guard let value = document.frontMatter[key] else { // Key not present in this document return "" } // Convert the value to a string - return try frontmatterValueToString(valueNode) + return try frontmatterValueToString(value) } /// Convert a Yams.Node to a string representation /// /// - Scalars: Return as-is /// - Complex types (arrays, objects): Serialize to JSON - func frontmatterValueToString(_ node: Yams.Node) throws -> String { - switch node { - case .scalar(let scalar): - return scalar.string - - case .sequence, .mapping: - // Serialize complex types as JSON (compact, no pretty printing) - return try YAMLConversion.nodeToJSON(node, options: []) - - case .alias: - // YAML aliases should be resolved by the parser, but if we encounter one, - // serialize it as JSON for safety - return try YAMLConversion.nodeToJSON(node, options: []) + func frontmatterValueToString(_ value: FrontMatterValue) throws -> String { + switch value { + case .null: + return "" + case .string(let value): + return value + case .boolean(let value): + return String(value) + case .integer(let value): + return String(value) + case .number(let value): + return String(value) + case .offsetDateTime, .localDateTime, .localDate, .localTime: + return String(describing: FrontMatterConversion.foundationValue(value)) + case .array, .object: + return try YAMLConversion.anyToJSON( + FrontMatterConversion.foundationValue(value), + options: [] + ) } } // MARK: - CSV Escaping diff --git a/Sources/MarkdownUtilities/FormatConversion/CSV/CSVOptions.swift b/Sources/MarkdownUtilities/FormatConversion/CSV/CSVOptions.swift index 405a253..cd1cb0f 100644 --- a/Sources/MarkdownUtilities/FormatConversion/CSV/CSVOptions.swift +++ b/Sources/MarkdownUtilities/FormatConversion/CSV/CSVOptions.swift @@ -17,7 +17,7 @@ import MarkdownUtilitiesCore public struct CSVOptions: ConversionOptions, Sendable { // MARK: - ConversionOptions Conformance - /// Whether to include YAML frontmatter in the CSV output + /// Whether to include YAML or TOML frontmatter in the CSV output /// /// This is always `true` for CSV conversion since the entire purpose /// is to export frontmatter as columns. diff --git a/Sources/MarkdownUtilities/Types/MarkdownTypeFileRegistryLoader.swift b/Sources/MarkdownUtilities/Types/MarkdownTypeFileRegistryLoader.swift index f6792ca..61ae92a 100644 --- a/Sources/MarkdownUtilities/Types/MarkdownTypeFileRegistryLoader.swift +++ b/Sources/MarkdownUtilities/Types/MarkdownTypeFileRegistryLoader.swift @@ -10,6 +10,7 @@ public enum MarkdownTypeFileRegistryLoader { ".mdtype.yaml", ".mdtype.yml", ".mdtype.json", + ".mdtype.toml", ] public static func load(projectRoot: Path) throws -> MarkdownTypeRegistry { @@ -61,6 +62,8 @@ public enum MarkdownTypeFileRegistryLoader { return .yaml case "json": return .json + case "toml": + return .toml default: throw MarkdownTypeFileLoaderError.unsupportedDefinitionFormat(path.string) } diff --git a/Sources/MarkdownUtilitiesCore/Documentation.docc/FrontmatterWorkflows.md b/Sources/MarkdownUtilitiesCore/Documentation.docc/FrontmatterWorkflows.md index 36b7382..6d907f6 100644 --- a/Sources/MarkdownUtilitiesCore/Documentation.docc/FrontmatterWorkflows.md +++ b/Sources/MarkdownUtilitiesCore/Documentation.docc/FrontmatterWorkflows.md @@ -1,24 +1,26 @@ # Reading and Mutating Frontmatter -Read, write, and convert YAML frontmatter while preserving the Markdown body. +Read, write, and convert YAML or TOML frontmatter while preserving the Markdown body. ## Overview -Frontmatter support starts with `FrontMatterParser`, which detects YAML delimited by `---` markers and separates it from the body text. `MarkdownDocument` then parses that YAML into a Yams mapping for structured access. +Frontmatter support starts with `FrontMatterParser`, which detects YAML delimited by `---` or TOML delimited by `+++` and separates it from the body text. `MarkdownDocument` parses either format into the ordered, format-neutral ``FrontMatter`` model and records the source ``FrontMatterFormat``. Non-Markdown text uses ``WrappedFrontMatterParser`` with a ``FrontMatterSyntax``. -The parser scans LF text for complete host wrappers containing complete `---` YAML -blocks. It returns the first block's raw YAML and snapshot-relative source range, +The parser scans LF text for complete host wrappers containing complete YAML or TOML +blocks. It returns the first block's raw frontmatter, format, and snapshot-relative source range, plus the 1-based opening lines of later complete blocks. Incomplete candidates are treated as absent. The parser does not model the host content as Markdown and does -not normalize YAML indentation. +not normalize indentation. -Mutation helpers on `MarkdownDocument` update frontmatter values without changing the body. Conversion helpers in `YAMLConversion` translate YAML nodes and mappings to Swift values, JSON, Property List, and YAML output. +Mutation helpers on `MarkdownDocument` update frontmatter values without changing the body or delimiter format. ``FrontMatterConversion`` parses and serializes the neutral value model; `YAMLConversion` remains available for YAML-specific interoperability. + +Comments are not part of the neutral value model. Parsing and rendering either YAML or TOML does not guarantee that frontmatter comments survive, so callers should avoid comments in frontmatter that will be mutated. ## Missing and Null Values -A missing key and a key with a YAML null value are distinct states. Callers that render frontmatter for user-facing output should preserve that distinction when it matters to downstream tools. +A missing key and a key with a YAML null value are distinct states. TOML has no null value and serialization reports the exact unsupported key path. Callers that render frontmatter for user-facing output should preserve the distinction when it matters to downstream tools. ## Errors -Invalid YAML and mappings with unsupported keys are reported as thrown errors rather than fatal failures. +Invalid YAML, invalid TOML, and values unsupported by the selected format are reported as thrown errors rather than fatal failures. diff --git a/Sources/MarkdownUtilitiesCore/Documentation.docc/MarkdownTypes.md b/Sources/MarkdownUtilitiesCore/Documentation.docc/MarkdownTypes.md index aabfa13..e8b86aa 100644 --- a/Sources/MarkdownUtilitiesCore/Documentation.docc/MarkdownTypes.md +++ b/Sources/MarkdownUtilitiesCore/Documentation.docc/MarkdownTypes.md @@ -4,13 +4,13 @@ Assess complete Markdown records against reusable structural contracts. ## Records and Conformance -A `MarkdownRecord` contains canonical Markdown text and optional identity, revision, and external context. A `MarkdownDocument` is the parsed content view produced from valid text. Type assessment accepts the record so invalid YAML can be returned as a structured diagnostic instead of preventing the resource from being represented. +A `MarkdownRecord` contains canonical Markdown text and optional identity, revision, and external context. A `MarkdownDocument` is the parsed content view produced from valid text. Type assessment accepts the record so invalid YAML or TOML can be returned as a structured diagnostic instead of preventing the resource from being represented. Conformance is structural and non-exclusive. One record can conform to `Book`, `Document`, and `Publishable` at the same time. Requirements produce errors and affect conformance. Recommendations produce advisories without making the record fail. Types have three domains: -- `frontmatter` validates schema-visible YAML values against every listed JSON Schema; +- `frontmatter` validates schema-visible YAML or TOML values against every listed JSON Schema; - `body` evaluates Markdown AST predicates such as headings, hierarchy, and sections; and - `context` evaluates external facts such as a normalized logical path. @@ -18,7 +18,7 @@ Types have three domains: ## Define and Assess a Type -Type definitions use the same model whether decoded from YAML or JSON. Filesystem-backed definitions use the compound extensions `.mdtype.yaml`, `.mdtype.yml`, or `.mdtype.json`. A type contract version is an opaque nonempty string; Semantic Versioning is recommended but not enforced. +Type definitions use the same model whether decoded from YAML, JSON, or TOML. Filesystem-backed definitions use the compound extensions `.mdtype.yaml`, `.mdtype.yml`, `.mdtype.json`, or `.mdtype.toml`. A type contract version is an opaque nonempty string; Semantic Versioning is recommended but not enforced. ```swift let definition = MarkdownTypeDefinition( diff --git a/Sources/MarkdownUtilitiesCore/Documentation.docc/RecordIdentity.md b/Sources/MarkdownUtilitiesCore/Documentation.docc/RecordIdentity.md index 6c4ea33..22fe69d 100644 --- a/Sources/MarkdownUtilitiesCore/Documentation.docc/RecordIdentity.md +++ b/Sources/MarkdownUtilitiesCore/Documentation.docc/RecordIdentity.md @@ -36,7 +36,7 @@ Slug validation is selected explicitly: - `unicode` accepts lowercase Unicode letters and digits separated by non-adjacent hyphens or underscores; and - `preserve` accepts ASCII letters and digits separated by single hyphens while preserving authored case. -Missing and null values are reported as missing identities. Malformed YAML, invalid formats, arrays, objects, booleans, non-integral numbers, and lossy conversions produce structured invalid-identity diagnostics. Identity status is independent from Markdown type conformance. +Missing and null values are reported as missing identities. Malformed YAML or TOML, invalid formats, arrays, objects, booleans, non-integral numbers, and lossy conversions produce structured invalid-identity diagnostics. Identity status is independent from Markdown type conformance. ## Stability and Collisions diff --git a/Sources/MarkdownUtilitiesCore/Documentation.docc/WorkingWithMarkdownDocuments.md b/Sources/MarkdownUtilitiesCore/Documentation.docc/WorkingWithMarkdownDocuments.md index 6a5da86..9da32ab 100644 --- a/Sources/MarkdownUtilitiesCore/Documentation.docc/WorkingWithMarkdownDocuments.md +++ b/Sources/MarkdownUtilitiesCore/Documentation.docc/WorkingWithMarkdownDocuments.md @@ -1,10 +1,10 @@ # Working with Markdown Documents -Create a `MarkdownDocument` when you need structured access to Markdown content whose YAML can be parsed. +Create a `MarkdownDocument` when you need structured access to Markdown content whose YAML or TOML frontmatter can be parsed. ## Overview -`MarkdownDocument` accepts raw Markdown text and separates an optional YAML frontmatter block from the document body. Frontmatter is parsed into a Yams mapping so callers can inspect and mutate structured values, while the body remains available as plain text for extraction, formatting, and conversion workflows. +`MarkdownDocument` accepts raw Markdown text and separates optional YAML (`---`) or TOML (`+++`) frontmatter from the document body. Frontmatter is parsed into the format-neutral ``FrontMatter`` mapping so callers can inspect and mutate structured values, while the body remains available as plain text for extraction, formatting, and conversion workflows. When callers need syntax-aware access to the body, `MarkdownDocument.parseAST()` parses the body with MarkdownSyntax and returns a fresh syntax tree for each call. @@ -22,6 +22,6 @@ Use this workflow before applying frontmatter, section, heading, table-of-conten A `MarkdownDocument` is a parsed interpretation of text. Its frontmatter, body, and AST all come from that text. It deliberately has no identity, path, revision, database table, or object-store key. -A `MarkdownRecord` is the canonical, addressable resource. It owns the original Markdown string plus optional identity, revision, and external `MarkdownRecordContext`. A record can therefore exist when its YAML is invalid and no `MarkdownDocument` can be initialized. +A `MarkdownRecord` is the canonical, addressable resource. It owns the original Markdown string plus optional identity, revision, and external `MarkdownRecordContext`. A record can therefore exist when its frontmatter is invalid and no `MarkdownDocument` can be initialized. Use a record when assessing types or rules. Parse a document directly when the operation requires only successfully parsed content. diff --git a/Sources/MarkdownUtilitiesCore/Explore/ExploreDocument.swift b/Sources/MarkdownUtilitiesCore/Explore/ExploreDocument.swift index 69838f9..13a9f0d 100644 --- a/Sources/MarkdownUtilitiesCore/Explore/ExploreDocument.swift +++ b/Sources/MarkdownUtilitiesCore/Explore/ExploreDocument.swift @@ -11,7 +11,7 @@ public struct ExploreDocument: Sendable { /// Original Markdown source lines split on newline boundaries. public let sourceLines: [String] - /// Optional YAML frontmatter block from the original source. + /// Optional YAML or TOML frontmatter block from the original source. public let frontmatter: ExploreFrontmatter? /// Optional document preamble before the first heading. @@ -82,13 +82,15 @@ public struct ExploreDocument: Sendable { } private static func detectFrontmatter(in sourceLines: [String]) throws -> ExploreFrontmatter? { - guard sourceLines.first == "---" else { + guard let opening = sourceLines.first, + let format = FrontMatterFormat.allCases.first(where: { $0.delimiter == opening }) + else { return nil } - for index in 1.. String { - try YAMLConversion.serialize(frontMatter) - } // MARK: - Future Format Conversions (Placeholders) // Uncomment and implement as needed: diff --git a/Sources/MarkdownUtilitiesCore/FormatConversion/PlainText/PlainTextOptions.swift b/Sources/MarkdownUtilitiesCore/FormatConversion/PlainText/PlainTextOptions.swift index f008ec2..b44d98d 100644 --- a/Sources/MarkdownUtilitiesCore/FormatConversion/PlainText/PlainTextOptions.swift +++ b/Sources/MarkdownUtilitiesCore/FormatConversion/PlainText/PlainTextOptions.swift @@ -17,7 +17,7 @@ import Foundation public struct PlainTextOptions: ConversionOptions, Sendable { // MARK: - ConversionOptions Conformance - /// Whether to include YAML frontmatter in the plain text output + /// Whether to include YAML or TOML frontmatter in the plain text output /// /// When `true`, frontmatter is preserved as a YAML block at the beginning. /// When `false`, frontmatter is excluded from the output. @@ -67,7 +67,7 @@ public struct PlainTextOptions: ConversionOptions, Sendable { /// Creates plain text conversion options with specified settings. /// /// - Parameters: - /// - includeFrontmatter: Include YAML frontmatter (default: false) + /// - includeFrontmatter: Include YAML or TOML frontmatter (default: false) /// - blockSeparator: Newlines between blocks (default: 2) /// - preserveLineBreaks: Preserve line breaks as newlines (default: true) /// - extractImageAltText: Extract image alt text (default: true) diff --git a/Sources/MarkdownUtilitiesCore/FormatConversion/Protocols/ConversionOptions.swift b/Sources/MarkdownUtilitiesCore/FormatConversion/Protocols/ConversionOptions.swift index 53f3ac5..2c821b9 100644 --- a/Sources/MarkdownUtilitiesCore/FormatConversion/Protocols/ConversionOptions.swift +++ b/Sources/MarkdownUtilitiesCore/FormatConversion/Protocols/ConversionOptions.swift @@ -8,7 +8,7 @@ import Foundation /// Conforming types should provide configuration options specific to their /// target format while maintaining the base requirement for frontmatter handling. public protocol ConversionOptions: Sendable { - /// Whether to include YAML frontmatter in the converted output + /// Whether to include YAML or TOML frontmatter in the converted output /// /// When `true`, frontmatter will be preserved in the output. /// When `false`, frontmatter will be excluded from the conversion. diff --git a/Sources/MarkdownUtilitiesCore/Formatting/MarkdownDocument+Formatting.swift b/Sources/MarkdownUtilitiesCore/Formatting/MarkdownDocument+Formatting.swift index 951b7f2..17d1af9 100644 --- a/Sources/MarkdownUtilitiesCore/Formatting/MarkdownDocument+Formatting.swift +++ b/Sources/MarkdownUtilitiesCore/Formatting/MarkdownDocument+Formatting.swift @@ -64,6 +64,10 @@ extension MarkdownDocument { result = TableNormalizer.normalize(result, maxWidth: options.tableMaxWidth) } - return MarkdownDocument(frontMatter: frontMatter, body: result) + return MarkdownDocument( + frontMatter: frontMatter, + body: result, + frontMatterFormat: frontMatterFormat + ) } } diff --git a/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatter.swift b/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatter.swift new file mode 100644 index 0000000..74c3d7f --- /dev/null +++ b/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatter.swift @@ -0,0 +1,204 @@ +import Foundation +import TOML +import Yams + +/// The serialization format used by a Markdown frontmatter block. +public enum FrontMatterFormat: String, CaseIterable, Codable, Equatable, Sendable { + case yaml + case toml + + /// The complete line used to delimit this format in Markdown. + public var delimiter: String { + switch self { + case .yaml: "---" + case .toml: "+++" + } + } +} + +/// An ordered, format-neutral frontmatter mapping. +public struct FrontMatter: Equatable, Sendable { + private var entries: [(key: String, value: FrontMatterValue)] + + public init() { + entries = [] + } + + public init(_ values: [String: FrontMatterValue]) { + entries = values.map { (key: $0.key, value: $0.value) } + } + + public init(_ entries: [(String, FrontMatterValue)]) { + self.entries = [] + for (key, value) in entries { + self[key] = value + } + } + + public var isEmpty: Bool { entries.isEmpty } + public var count: Int { entries.count } + public var keys: [String] { entries.map(\.key) } + + public subscript(key: String) -> FrontMatterValue? { + get { entries.first(where: { $0.key == key })?.value } + set { + if let index = entries.firstIndex(where: { $0.key == key }) { + if let newValue { + entries[index].value = newValue + } else { + entries.remove(at: index) + } + } else if let newValue { + entries.append((key, newValue)) + } + } + } + + public mutating func sort( + by areInIncreasingOrder: ((key: String, value: FrontMatterValue), (key: String, value: FrontMatterValue)) -> Bool + ) { + entries.sort(by: areInIncreasingOrder) + } + + public var dictionary: [String: FrontMatterValue] { + Dictionary(uniqueKeysWithValues: entries.map { ($0.key, $0.value) }) + } + + public static func == (lhs: FrontMatter, rhs: FrontMatter) -> Bool { + guard lhs.entries.count == rhs.entries.count else { return false } + return zip(lhs.entries, rhs.entries).allSatisfy { left, right in + left.key == right.key && left.value == right.value + } + } +} + +extension FrontMatter: Sequence { + public func makeIterator() -> IndexingIterator<[(key: String, value: FrontMatterValue)]> { + entries.makeIterator() + } +} + +extension FrontMatter: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: FrontMatterCodingKey.self) + entries = try container.allKeys.map { key in + (key.stringValue, try container.decode(FrontMatterValue.self, forKey: key)) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: FrontMatterCodingKey.self) + for (key, value) in entries { + try container.encode(value, forKey: FrontMatterCodingKey(key)) + } + } +} + +/// A format-neutral frontmatter value, including TOML's native date/time types. +public enum FrontMatterValue: Equatable, Sendable { + case null + case boolean(Bool) + case integer(Int64) + case number(Double) + case string(String) + case offsetDateTime(Date) + case localDateTime(LocalDateTime) + case localDate(LocalDate) + case localTime(LocalTime) + case array([FrontMatterValue]) + case object(FrontMatter) + + /// The string payload when this value is a string. + public var stringValue: String? { + guard case .string(let value) = self else { return nil } + return value + } + + public var int: Int? { + guard case .integer(let value) = self else { return nil } + return Int(exactly: value) + } + + public var float: Double? { + switch self { + case .number(let value): value + case .integer(let value): Double(value) + default: nil + } + } + + public var bool: Bool? { + guard case .boolean(let value) = self else { return nil } + return value + } + + public var sequence: [FrontMatterValue]? { + guard case .array(let value) = self else { return nil } + return value + } + + public var mapping: FrontMatter? { + guard case .object(let value) = self else { return nil } + return value + } +} + +extension FrontMatterValue: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self = .null + } else if let value = try? container.decode(Bool.self) { + self = .boolean(value) + } else if let value = try? container.decode(Int64.self) { + self = .integer(value) + } else if let value = try? container.decode(Double.self) { + self = .number(value) + } else if let value = try? container.decode(String.self) { + self = .string(value) + } else if let value = try? container.decode(LocalDateTime.self) { + self = .localDateTime(value) + } else if let value = try? container.decode(LocalDate.self) { + self = .localDate(value) + } else if let value = try? container.decode(LocalTime.self) { + self = .localTime(value) + } else if let value = try? container.decode(Date.self) { + self = .offsetDateTime(value) + } else if let value = try? container.decode([FrontMatterValue].self) { + self = .array(value) + } else if let value = try? container.decode(FrontMatter.self) { + self = .object(value) + } else { + throw DecodingError.dataCorruptedError( + in: container, + debugDescription: "Unsupported frontmatter value" + ) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .null: try container.encodeNil() + case .boolean(let value): try container.encode(value) + case .integer(let value): try container.encode(value) + case .number(let value): try container.encode(value) + case .string(let value): try container.encode(value) + case .offsetDateTime(let value): try container.encode(value) + case .localDateTime(let value): try container.encode(value) + case .localDate(let value): try container.encode(value) + case .localTime(let value): try container.encode(value) + case .array(let value): try container.encode(value) + case .object(let value): try container.encode(value) + } + } +} + +private struct FrontMatterCodingKey: CodingKey { + let stringValue: String + let intValue: Int? = nil + + init(_ string: String) { stringValue = string } + init?(stringValue: String) { self.stringValue = stringValue } + init?(intValue: Int) { return nil } +} diff --git a/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterConversion.swift b/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterConversion.swift new file mode 100644 index 0000000..67b400f --- /dev/null +++ b/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterConversion.swift @@ -0,0 +1,207 @@ +import Foundation +import TOML +import Yams + +/// Errors raised while parsing or serializing format-neutral frontmatter. +public enum FrontMatterConversionError: Error, Equatable, LocalizedError { + case invalidTOML(String) + case unsupportedTOMLValue(path: String, value: String) + case integerOutOfRange(String) + case invalidYAMLValue(String) + + public var errorDescription: String? { + switch self { + case .invalidTOML(let message): + "Invalid TOML frontmatter: \(message)" + case .unsupportedTOMLValue(let path, let value): + "TOML cannot represent \(value) at \(path)" + case .integerOutOfRange(let value): + "Frontmatter integer is outside the supported 64-bit range: \(value)" + case .invalidYAMLValue(let value): + "Unsupported YAML frontmatter value: \(value)" + } + } +} + +/// Converts frontmatter between YAML, TOML, and shared dynamic values. +public enum FrontMatterConversion { + public static func parse(_ source: String, format: FrontMatterFormat) throws -> FrontMatter { + switch format { + case .yaml: + return try fromYAMLMapping(YAMLConversion.parse(source)) + case .toml: + do { + let decoder = TOMLDecoder() + return try decoder.decode(FrontMatter.self, from: source) + } catch { + throw FrontMatterConversionError.invalidTOML(String(describing: error)) + } + } + } + + public static func serialize(_ frontMatter: FrontMatter, format: FrontMatterFormat) throws -> String { + switch format { + case .yaml: + return try YAMLConversion.serialize(try toYAMLMapping(frontMatter)) + case .toml: + try validateTOML(frontMatter) + let encoder = TOMLEncoder() + encoder.outputFormatting = .sortedKeys + let data = try encoder.encode(frontMatter) + guard let string = String(data: data, encoding: .utf8) else { + throw FrontMatterConversionError.invalidTOML("Encoder returned non-UTF-8 data") + } + return string.hasSuffix("\n") ? string : string + "\n" + } + } + + public static func foundationValue(_ frontMatter: FrontMatter) -> [String: Any] { + frontMatter.dictionary.mapValues(foundationValue) + } + + public static func foundationValue(_ value: FrontMatterValue) -> Any { + switch value { + case .null: NSNull() + case .boolean(let value): value + case .integer(let value): value + case .number(let value): value + case .string(let value): value + case .offsetDateTime(let value): ISO8601DateFormatter().string(from: value) + case .localDateTime(let value): format(value) + case .localDate(let value): format(value) + case .localTime(let value): format(value) + case .array(let values): values.map(foundationValue) + case .object(let value): foundationValue(value) + } + } + + public static func fromFoundationValue(_ value: Any) throws -> FrontMatterValue { + if value is NSNull { return .null } + if let value = value as? Bool { return .boolean(value) } + if let value = value as? Int { return .integer(Int64(value)) } + if let value = value as? Int64 { return .integer(value) } + if let value = value as? Double { return .number(value) } + if let value = value as? Float { return .number(Double(value)) } + if let value = value as? String { return .string(value) } + if let value = value as? Date { return .offsetDateTime(value) } + if let values = value as? [Any] { + return .array(try values.map(fromFoundationValue)) + } + if let values = value as? [String: Any] { + var result = FrontMatter() + for (key, value) in values { + result[key] = try fromFoundationValue(value) + } + return .object(result) + } + throw FrontMatterConversionError.unsupportedTOMLValue( + path: "value", + value: String(describing: type(of: value)) + ) + } + + /// Serializes an arbitrary structured value as a TOML document. + /// Non-object roots use the stable `value` envelope required by TOML. + public static func serializeTOMLValue(_ value: Any) throws -> String { + let converted = try fromFoundationValue(value) + let document: FrontMatter + if case .object(let object) = converted { + document = object + } else { + document = FrontMatter(["value": converted]) + } + return try serialize(document, format: .toml) + } + + public static func fromYAMLMapping(_ mapping: Yams.Node.Mapping) throws -> FrontMatter { + var result = FrontMatter() + for (keyNode, valueNode) in mapping { + guard let key = keyNode.string else { + throw YAMLConversionError.nonStringKey(String(describing: keyNode)) + } + result[key] = try fromYAMLNode(valueNode) + } + return result + } + + public static func toYAMLMapping(_ frontMatter: FrontMatter) throws -> Yams.Node.Mapping { + Yams.Node.Mapping(try frontMatter.map { entry in + (.scalar(.init(entry.key)), try toYAMLNode(entry.value)) + }) + } + + private static func fromYAMLNode(_ node: Yams.Node) throws -> FrontMatterValue { + if let mapping = node.mapping { return .object(try fromYAMLMapping(mapping)) } + if let sequence = node.sequence { return .array(try sequence.map(fromYAMLNode)) } + if node.tag == Tag(.null) { return .null } + if let value = node.bool { return .boolean(value) } + if let value = node.int { return .integer(Int64(value)) } + if let value = node.float { return .number(value) } + if let value = node.string { return .string(value) } + throw FrontMatterConversionError.invalidYAMLValue(String(describing: node)) + } + + private static func toYAMLNode(_ value: FrontMatterValue) throws -> Yams.Node { + switch value { + case .null: .scalar(.init("", Tag(.null))) + case .boolean(let value): .scalar(.init(value ? "true" : "false")) + case .integer(let value): .scalar(.init(String(value))) + case .number(let value): .scalar(.init(String(value))) + case .string(let value): .scalar(.init(value)) + case .offsetDateTime(let value): .scalar(.init(ISO8601DateFormatter().string(from: value))) + case .localDateTime(let value): .scalar(.init(format(value))) + case .localDate(let value): .scalar(.init(format(value))) + case .localTime(let value): .scalar(.init(format(value))) + case .array(let values): .sequence(.init(try values.map(toYAMLNode))) + case .object(let value): .mapping(try toYAMLMapping(value)) + } + } + + private static func validateTOML(_ frontMatter: FrontMatter) throws { + for (key, value) in frontMatter { + try validateTOML(value, path: key) + } + } + + private static func validateTOML(_ value: FrontMatterValue, path: String) throws { + switch value { + case .null: + throw FrontMatterConversionError.unsupportedTOMLValue(path: path, value: "null") + case .array(let values): + for (index, value) in values.enumerated() { + try validateTOML(value, path: "\(path)[\(index)]") + } + case .object(let object): + for (key, value) in object { + try validateTOML(value, path: "\(path).\(key)") + } + default: + break + } + } + + private static func format(_ value: LocalDateTime) -> String { + var result = String( + format: "%04d-%02d-%02dT%02d:%02d:%02d", + value.year, value.month, value.day, value.hour, value.minute, value.second + ) + if value.nanosecond > 0 { result += fractionalSeconds(value.nanosecond) } + return result + } + + private static func format(_ value: LocalDate) -> String { + String(format: "%04d-%02d-%02d", value.year, value.month, value.day) + } + + private static func format(_ value: LocalTime) -> String { + var result = String(format: "%02d:%02d:%02d", value.hour, value.minute, value.second) + if value.nanosecond > 0 { result += fractionalSeconds(value.nanosecond) } + return result + } + + private static func fractionalSeconds(_ nanosecond: Int) -> String { + let digits = String(format: "%09d", nanosecond) + .replacingOccurrences(of: "0+$", with: "", options: .regularExpression) + return ".\(digits)" + } +} diff --git a/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterParser.swift b/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterParser.swift index 838e316..b83f35e 100644 --- a/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterParser.swift +++ b/Sources/MarkdownUtilitiesCore/FrontMatter/FrontMatterParser.swift @@ -8,49 +8,48 @@ import Parsing /// Parser for separating frontmatter from markdown body content. /// -/// This parser detects YAML frontmatter delimited by `---` markers and separates -/// it from the body content without parsing the YAML itself. +/// This parser detects YAML (`---`) or TOML (`+++`) frontmatter and separates it +/// from the body content without parsing the structured data itself. struct FrontMatterParser: Parsing.Parser { typealias Input = Substring - typealias Output = (rawFrontMatter: String, body: String) + typealias Output = (rawFrontMatter: String, body: String, format: FrontMatterFormat?) /// Parses input into structured Markdown data. /// /// See for workflow details. func parse(_ input: inout Substring) throws -> Output { - // Check if input starts with frontmatter delimiter (Unix line endings only) - if input.starts(with: "---\n") { - var workingInput = input - do { - // Try to extract just the frontmatter (up to and including closing delimiter) - let rawFrontMatter = try frontMatterOnlyParser.parse(&workingInput) - // Whatever remains in workingInput is the body - let body = String(workingInput) - input = "" - return (rawFrontMatter, body) - } catch { - // No closing delimiter found - treat entire content as body with empty frontmatter - let body = String(input) - input = "" - return ("", body) - } - } else { - // No opening delimiter - empty frontmatter, entire content is body - let body = String(input) + let originalSource = String(input) + guard let format = FrontMatterFormat.allCases.first(where: { + originalSource.hasPrefix("\($0.delimiter)\n") + }) else { input = "" - return ("", body) + return ("", originalSource, nil) } - } - - /// Parser that extracts only the frontmatter content (between delimiters) - private var frontMatterOnlyParser: some Parsing.Parser { - Parse { - "---\n" // Opening delimiter with newline - PrefixUpTo("---").map { String($0) } // Content until closing delimiter - "---" // Closing delimiter - Optionally { "\n" } // Optional newline after closing + let bytes = Array(originalSource.utf8) + let delimiter = Array(format.delimiter.utf8) + let contentStart = delimiter.count + 1 + let closingStart = stride(from: contentStart, to: bytes.count, by: 1).first { index in + let startsLine = index == contentStart || bytes[index - 1] == 0x0A + return startsLine + && index + delimiter.count <= bytes.count + && bytes[index..<(index + delimiter.count)].elementsEqual(delimiter) } - .map { (frontMatter, _) in - frontMatter + guard let closingStart else { + input = "" + return ("", originalSource, nil) + } + + let rawEnd = closingStart > contentStart && bytes[closingStart - 1] == 0x0A + ? closingStart - 1 + : closingStart + let rawFrontMatter = String(decoding: bytes[contentStart.. for workflow details. @@ -25,20 +24,16 @@ extension MarkdownDocument { /// it returns just the body. /// /// - Returns: The reconstructed markdown document - /// - Throws: If YAML serialization fails + /// - Throws: If serialization in the selected frontmatter format fails public func render() throws -> String { - // Only add delimiters if there's actual frontmatter content - if frontMatter.isEmpty { - return body - } + guard !frontMatter.isEmpty else { return body } + let format = frontMatterFormat ?? .yaml - // Serialize frontmatter back to YAML - let yamlString = try YAMLConversion.serialize(frontMatter) + let serialized = try FrontMatterConversion.serialize(frontMatter, format: format) - // Add delimiters around frontmatter return """ - --- - \(yamlString)--- + \(format.delimiter) + \(serialized)\(format.delimiter) \(body) """ } diff --git a/Sources/MarkdownUtilitiesCore/FrontMatter/MarkdownDocument+FrontMatterMutation.swift b/Sources/MarkdownUtilitiesCore/FrontMatter/MarkdownDocument+FrontMatterMutation.swift index 743dc9d..9725cf1 100644 --- a/Sources/MarkdownUtilitiesCore/FrontMatter/MarkdownDocument+FrontMatterMutation.swift +++ b/Sources/MarkdownUtilitiesCore/FrontMatter/MarkdownDocument+FrontMatterMutation.swift @@ -1,5 +1,4 @@ import Foundation -import Yams /// Adds frontmatter behavior to ``MarkdownDocument``. /// /// See for workflow details. @@ -7,21 +6,20 @@ extension MarkdownDocument { /// Get value for key from frontmatter /// /// - Parameter key: The frontmatter key to retrieve - /// - Returns: The Yams.Node value if the key exists, nil otherwise - public func getValue(forKey key: String) -> Yams.Node? { + /// - Returns: The format-neutral value if the key exists, nil otherwise + public func getValue(forKey key: String) -> FrontMatterValue? { return frontMatter[key] } /// Set string value for key in frontmatter /// - /// This method converts the provided string value to a YAML scalar node - /// and assigns it to the specified key in the frontmatter mapping. + /// This method stores the provided value as a string in the frontmatter mapping. /// /// - Parameters: /// - value: The string value to set /// - key: The frontmatter key public mutating func setValue(_ value: String, forKey key: String) { - frontMatter[key] = Yams.Node.scalar(.init(value)) + frontMatter[key] = .string(value) } /// Groups CreateKeyError cases and related behavior. /// @@ -45,7 +43,7 @@ extension MarkdownDocument { guard !hasKey(key) else { throw CreateKeyError.keyAlreadyExists } - frontMatter[key] = Yams.Node("", Tag(.null)) + frontMatter[key] = .null } /// Check if key exists in frontmatter @@ -113,26 +111,16 @@ extension MarkdownDocument { /// - method: The sorting method to use (alphabetical or by length) /// - reverse: Whether to reverse the sorting order (default: false) public mutating func sortKeys(by method: SortMethod = .alphabetical, reverse: Bool = false) { - let sorted: [(Yams.Node, Yams.Node)] - switch method { case .alphabetical: - sorted = frontMatter.sorted { lhs, rhs in - guard let lhsKey = lhs.key.string, let rhsKey = rhs.key.string else { - return false - } - return reverse ? lhsKey > rhsKey : lhsKey < rhsKey + frontMatter.sort { lhs, rhs in + reverse ? lhs.key > rhs.key : lhs.key < rhs.key } case .length: - sorted = frontMatter.sorted { lhs, rhs in - guard let lhsKey = lhs.key.string, let rhsKey = rhs.key.string else { - return false - } - return reverse ? lhsKey.count > rhsKey.count : lhsKey.count < rhsKey.count + frontMatter.sort { lhs, rhs in + reverse ? lhs.key.count > rhs.key.count : lhs.key.count < rhs.key.count } } - - frontMatter = Yams.Node.Mapping(sorted) } /// Sorting method for frontmatter keys diff --git a/Sources/MarkdownUtilitiesCore/FrontMatter/WrappedFrontMatter.swift b/Sources/MarkdownUtilitiesCore/FrontMatter/WrappedFrontMatter.swift index e4cb34b..d238ae3 100644 --- a/Sources/MarkdownUtilitiesCore/FrontMatter/WrappedFrontMatter.swift +++ b/Sources/MarkdownUtilitiesCore/FrontMatter/WrappedFrontMatter.swift @@ -1,7 +1,7 @@ import Foundation import Parsing -/// A host-language envelope used to contain YAML frontmatter in a text file. +/// A host-language envelope used to contain YAML or TOML frontmatter in a text file. public struct FrontMatterSyntax: Equatable, Sendable { /// The stable name used in diagnostics and prompts. public let name: String @@ -101,8 +101,14 @@ public struct FrontMatterSyntax: Equatable, Sendable { /// A complete wrapped frontmatter block located in one exact source snapshot. public struct WrappedFrontMatterBlock: Equatable, Sendable { - /// The raw YAML between the two `---` marker lines. - public let rawYAML: String + /// The raw metadata between the format marker lines. + public let rawFrontMatter: String + + /// The serialization format used by the wrapped block. + public let format: FrontMatterFormat + + /// Compatibility spelling for YAML-only callers. + public var rawYAML: String { rawFrontMatter } /// The source range spanning both host wrapper lines. public let range: Range @@ -120,7 +126,7 @@ public struct WrappedFrontMatterScan: Equatable, Sendable { public let additionalOpeningLines: [Int] } -/// Finds complete delimiter-wrapped YAML frontmatter blocks in LF text. +/// Finds complete delimiter-wrapped YAML or TOML frontmatter blocks in LF text. public struct WrappedFrontMatterParser: Sendable { /// The host syntax recognized by this parser. public let syntax: FrontMatterSyntax @@ -132,7 +138,7 @@ public struct WrappedFrontMatterParser: Sendable { /// Scans the whole snapshot, returning the first block and later opening lines. /// - /// The parser recognizes LF input and requires wrapper delimiters and both YAML + /// The parser recognizes LF input and requires wrapper and frontmatter /// markers to occupy complete physical lines. Incomplete candidates are treated /// as absent. Returned ranges are valid only for the supplied snapshot. /// @@ -146,7 +152,9 @@ public struct WrappedFrontMatterParser: Sendable { while openingIndex < lines.count { guard lines[openingIndex].text == syntax.openingWrapper, openingIndex + 1 < lines.count, - lines[openingIndex + 1].text == "---" + let format = FrontMatterFormat.allCases.first(where: { + $0.delimiter == lines[openingIndex + 1].text + }) else { openingIndex += 1 continue @@ -155,14 +163,15 @@ public struct WrappedFrontMatterParser: Sendable { var yamlClosingIndex = openingIndex + 2 var matchedBlock: WrappedFrontMatterBlock? while yamlClosingIndex + 1 < lines.count { - if lines[yamlClosingIndex].text == "---", + if lines[yamlClosingIndex].text == format.delimiter, lines[yamlClosingIndex + 1].text == syntax.closingWrapper { let range = lines[openingIndex].start.. String? { + private func parseCompleteEnvelope( + _ candidate: String, + format: FrontMatterFormat + ) -> String? { var input = Substring(candidate) let parser = Parse { syntax.openingWrapper - "\n---\n" - PrefixUpTo("---\n\(syntax.closingWrapper)").map(String.init) - "---\n" + "\n\(format.delimiter)\n" + PrefixUpTo("\(format.delimiter)\n\(syntax.closingWrapper)").map(String.init) + "\(format.delimiter)\n" syntax.closingWrapper End() } diff --git a/Sources/MarkdownUtilitiesCore/HeadingAdjustment/MarkdownDocument+HeadingAdjustment.swift b/Sources/MarkdownUtilitiesCore/HeadingAdjustment/MarkdownDocument+HeadingAdjustment.swift index 8977fba..1a4f86d 100644 --- a/Sources/MarkdownUtilitiesCore/HeadingAdjustment/MarkdownDocument+HeadingAdjustment.swift +++ b/Sources/MarkdownUtilitiesCore/HeadingAdjustment/MarkdownDocument+HeadingAdjustment.swift @@ -123,22 +123,13 @@ extension MarkdownDocument { /// - body: The body content /// - Returns: Complete markdown content with frontmatter (if non-empty) and body private func reconstructFullDocument( - frontMatter: Yams.Node.Mapping, + frontMatter: FrontMatter, body: String ) throws -> String { - // If frontmatter is empty, return just the body - guard !frontMatter.isEmpty else { - return body - } - - // Serialize frontmatter to YAML - let yamlContent = try YAMLConversion.serialize(frontMatter) - - // Reconstruct with frontmatter delimiters - return """ - --- - \(yamlContent)--- - \(body) - """ + try MarkdownDocument( + frontMatter: frontMatter, + body: body, + frontMatterFormat: frontMatterFormat + ).render() } } diff --git a/Sources/MarkdownUtilitiesCore/MarkdownDocument.swift b/Sources/MarkdownUtilitiesCore/MarkdownDocument.swift index 7b88c98..7eb4b62 100644 --- a/Sources/MarkdownUtilitiesCore/MarkdownDocument.swift +++ b/Sources/MarkdownUtilitiesCore/MarkdownDocument.swift @@ -6,7 +6,6 @@ import Foundation import MarkdownSyntax import Parsing -import Yams /// A parsed representation of Markdown content. /// @@ -14,12 +13,15 @@ import Yams /// frontmatter, body text, and a derived Markdown AST. It has no persistent /// identity, logical path, revision, or storage context. Use ``MarkdownRecord`` /// for canonical, addressable content that must remain representable even when -/// its YAML is invalid. +/// its frontmatter is invalid. public struct MarkdownDocument: @unchecked Sendable { - /// The YAML frontmatter as a parsed mapping. + /// The YAML or TOML frontmatter as a format-neutral parsed mapping. /// /// This is an empty mapping if the document has no frontmatter. - public var frontMatter: Yams.Node.Mapping + public var frontMatter: FrontMatter + + /// The serialization format of the physical frontmatter block, or `nil` when absent. + public var frontMatterFormat: FrontMatterFormat? /// The body content of the document (everything after frontmatter, or entire document if no frontmatter). /// @@ -29,18 +31,19 @@ public struct MarkdownDocument: @unchecked Sendable { /// Initialize a markdown document by parsing the content to separate frontmatter from body. /// - /// This initializer uses `FrontMatterParser` to detect and separate YAML frontmatter - /// delimited by `---` markers. The frontmatter is immediately parsed into a `Yams.Node.Mapping`, - /// and the body contains everything after the closing delimiter. + /// This initializer detects YAML (`---`) or TOML (`+++`) frontmatter, parses it + /// into ``FrontMatter``, and retains the source format for rendering. /// /// - Parameter content: The markdown content to parse - /// - Throws: `YAMLConversionError` if the frontmatter exists but is invalid YAML or not a mapping + /// - Throws: If frontmatter is invalid or its root is not a mapping public init(content: String) throws { let parser = FrontMatterParser() var input = Substring(content) - let (rawFrontMatter, body) = try parser.parse(&input) + let (rawFrontMatter, body, format) = try parser.parse(&input) - self.frontMatter = try YAMLConversion.parse(rawFrontMatter) + self.frontMatter = try format.map { try FrontMatterConversion.parse(rawFrontMatter, format: $0) } + ?? FrontMatter() + self.frontMatterFormat = format self.body = body } @@ -50,10 +53,15 @@ public struct MarkdownDocument: @unchecked Sendable { /// avoiding the YAML serialize/parse round-trip that `init(content:)` performs. /// /// - Parameters: - /// - frontMatter: The parsed YAML frontmatter mapping (empty mapping if none) + /// - frontMatter: The parsed format-neutral frontmatter mapping (empty if none) /// - body: The markdown body text - public init(frontMatter: Yams.Node.Mapping, body: String) { + public init( + frontMatter: FrontMatter, + body: String, + frontMatterFormat: FrontMatterFormat? = nil + ) { self.frontMatter = frontMatter + self.frontMatterFormat = frontMatterFormat ?? (frontMatter.isEmpty ? nil : .yaml) self.body = body } diff --git a/Sources/MarkdownUtilitiesCore/Rules/MarkdownRuleChecker.swift b/Sources/MarkdownUtilitiesCore/Rules/MarkdownRuleChecker.swift index 5717dd0..1ff55d6 100644 --- a/Sources/MarkdownUtilitiesCore/Rules/MarkdownRuleChecker.swift +++ b/Sources/MarkdownUtilitiesCore/Rules/MarkdownRuleChecker.swift @@ -527,7 +527,10 @@ public struct MarkdownRuleChecker: Sendable { ) let typeRegistry = try MarkdownTypeRegistry(definitions: [definition]) return try MarkdownTypeChecker(registry: typeRegistry).assess(record, as: definition.name).diagnostics - .filter { $0.code != "record.frontmatter.invalid-yaml" } + .filter { + $0.code != "record.frontmatter.invalid-yaml" + && $0.code != "record.frontmatter.invalid-toml" + } } private func isPathCandidate( diff --git a/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionExtraction.swift b/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionExtraction.swift index f384ef6..07d1446 100644 --- a/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionExtraction.swift +++ b/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionExtraction.swift @@ -170,22 +170,13 @@ extension MarkdownDocument { /// - body: The body content /// - Returns: Complete markdown content with frontmatter (if non-empty) and body private func reconstructFullDocument( - frontMatter: Yams.Node.Mapping, + frontMatter: FrontMatter, body: String ) throws -> String { - // If frontmatter is empty, return just the body - guard !frontMatter.isEmpty else { - return body - } - - // Serialize frontmatter to YAML - let yamlContent = try YAMLConversion.serialize(frontMatter) - - // Reconstruct with frontmatter delimiters - return """ - --- - \(yamlContent)--- - \(body) - """ + try MarkdownDocument( + frontMatter: frontMatter, + body: body, + frontMatterFormat: frontMatterFormat + ).render() } } diff --git a/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionInsertion.swift b/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionInsertion.swift index cd188d0..8cc9854 100644 --- a/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionInsertion.swift +++ b/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionInsertion.swift @@ -220,17 +220,11 @@ extension MarkdownDocument { } private func rebuildDocumentAfterInsertion(withBody newBody: String) throws -> MarkdownDocument { - guard !frontMatter.isEmpty else { - return try MarkdownDocument(content: newBody) - } - - let yamlContent = try YAMLConversion.serialize(frontMatter) - let fullContent = """ - --- - \(yamlContent)--- - \(newBody) - """ - return try MarkdownDocument(content: fullContent) + MarkdownDocument( + frontMatter: frontMatter, + body: newBody, + frontMatterFormat: frontMatterFormat + ) } } diff --git a/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionReplacement.swift b/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionReplacement.swift index c245e70..cae3850 100644 --- a/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionReplacement.swift +++ b/Sources/MarkdownUtilitiesCore/SectionExtraction/MarkdownDocument+SectionReplacement.swift @@ -117,18 +117,10 @@ extension MarkdownDocument { /// Rebuilds the document with a new body, preserving frontmatter. private func rebuildDocument(withBody newBody: String) throws -> MarkdownDocument { - guard !frontMatter.isEmpty else { - return try MarkdownDocument(content: newBody) - } - - let yamlContent = try YAMLConversion.serialize(frontMatter) - - let fullContent = """ - --- - \(yamlContent)--- - \(newBody) - """ - - return try MarkdownDocument(content: fullContent) + MarkdownDocument( + frontMatter: frontMatter, + body: newBody, + frontMatterFormat: frontMatterFormat + ) } } diff --git a/Sources/MarkdownUtilitiesCore/SectionReordering/MarkdownDocument+SectionReordering.swift b/Sources/MarkdownUtilitiesCore/SectionReordering/MarkdownDocument+SectionReordering.swift index 91eedd9..81d99d3 100644 --- a/Sources/MarkdownUtilitiesCore/SectionReordering/MarkdownDocument+SectionReordering.swift +++ b/Sources/MarkdownUtilitiesCore/SectionReordering/MarkdownDocument+SectionReordering.swift @@ -183,19 +183,13 @@ extension MarkdownDocument { /// Reconstructs the full markdown document with frontmatter and body. private func reconstructFullDocument( - frontMatter: Yams.Node.Mapping, + frontMatter: FrontMatter, body: String ) throws -> String { - guard !frontMatter.isEmpty else { - return body - } - - let yamlContent = try YAMLConversion.serialize(frontMatter) - - return """ - --- - \(yamlContent)--- - \(body) - """ + try MarkdownDocument( + frontMatter: frontMatter, + body: body, + frontMatterFormat: frontMatterFormat + ).render() } } diff --git a/Sources/MarkdownUtilitiesCore/Types/MarkdownRecordAnalyzer.swift b/Sources/MarkdownUtilitiesCore/Types/MarkdownRecordAnalyzer.swift index 119acc1..6ce8c4b 100644 --- a/Sources/MarkdownUtilitiesCore/Types/MarkdownRecordAnalyzer.swift +++ b/Sources/MarkdownUtilitiesCore/Types/MarkdownRecordAnalyzer.swift @@ -113,7 +113,8 @@ package enum MarkdownRecordAnalyzer { parts = RecordParts( rawFrontmatter: parsed.rawFrontMatter, body: parsed.body, - hasFrontmatter: containsFrontmatterBlock(record.content), + hasFrontmatter: parsed.format != nil, + format: parsed.format, diagnostics: [] ) } catch { @@ -127,7 +128,7 @@ package enum MarkdownRecordAnalyzer { headings: supportsMarkdownStructure ? await analyzeHeadings(in: record.content, when: requirements) : [], - parseDiagnostics: [parseDiagnostic(error.localizedDescription)], + parseDiagnostics: [parseDiagnostic(error.localizedDescription, format: nil)], supportsMarkdownStructure: supportsMarkdownStructure, unavailableFrontmatterExtension: nil ) @@ -142,9 +143,10 @@ package enum MarkdownRecordAnalyzer { [multipleBlocksDiagnostic(line: line)] } ?? [] parts = RecordParts( - rawFrontmatter: scan.firstBlock?.rawYAML ?? "", + rawFrontmatter: scan.firstBlock?.rawFrontMatter ?? "", body: body, hasFrontmatter: scan.firstBlock != nil, + format: scan.firstBlock?.format, diagnostics: diagnostics ) case .unmapped(let fileExtension): @@ -167,8 +169,11 @@ package enum MarkdownRecordAnalyzer { if parts.hasFrontmatter { do { - let mapping = try YAMLConversion.parse(parts.rawFrontmatter) - let dynamicValue = try YAMLConversion.safeNodeToSwiftValue(.mapping(mapping)) + let frontMatter = try FrontMatterConversion.parse( + parts.rawFrontmatter, + format: parts.format ?? .yaml + ) + let dynamicValue = FrontMatterConversion.foundationValue(frontMatter) guard case .object(var object) = try JSONValue(any: dynamicValue) else { throw YAMLConversionError.notAMapping } @@ -179,7 +184,7 @@ package enum MarkdownRecordAnalyzer { } userFrontmatter = object.isEmpty ? nil : object } catch { - diagnostics.append(parseDiagnostic(error.localizedDescription)) + diagnostics.append(parseDiagnostic(error.localizedDescription, format: parts.format)) } } @@ -202,6 +207,7 @@ package enum MarkdownRecordAnalyzer { var rawFrontmatter: String var body: String var hasFrontmatter: Bool + var format: FrontMatterFormat? var diagnostics: [MarkdownDiagnostic] } @@ -255,10 +261,9 @@ package enum MarkdownRecordAnalyzer { } private static func containsFrontmatterBlock(_ content: String) -> Bool { - guard content.starts(with: "---\n") else { return false } - let remaining = content.dropFirst(4) - return remaining == "---" || remaining.hasPrefix("---\n") || remaining.contains("\n---\n") - || remaining.hasSuffix("\n---") + let parser = FrontMatterParser() + var input = Substring(content) + return (try? parser.parse(&input).format) != nil } private static func parseTypeHints( @@ -291,13 +296,17 @@ package enum MarkdownRecordAnalyzer { return (hints, diagnostics) } - private static func parseDiagnostic(_ message: String) -> MarkdownDiagnostic { - MarkdownDiagnostic( - code: "record.frontmatter.invalid-yaml", + private static func parseDiagnostic( + _ message: String, + format: FrontMatterFormat? + ) -> MarkdownDiagnostic { + let formatName = format?.rawValue.uppercased() ?? "frontmatter" + return MarkdownDiagnostic( + code: format == .toml ? "record.frontmatter.invalid-toml" : "record.frontmatter.invalid-yaml", severity: .error, domain: .frontmatter, location: "frontmatter", - message: "Invalid YAML: \(message)" + message: "Invalid \(formatName): \(message)" ) } diff --git a/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeDefinitionDecoder.swift b/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeDefinitionDecoder.swift index 7156fee..7bdd963 100644 --- a/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeDefinitionDecoder.swift +++ b/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeDefinitionDecoder.swift @@ -5,6 +5,7 @@ import Yams public enum MarkdownTypeDefinitionFormat: String, Codable, Equatable, Sendable { case yaml case json + case toml } /// Decodes YAML and JSON type definitions into the same portable model. @@ -73,6 +74,14 @@ public enum MarkdownTypeDefinitionDecoder { } catch { throw MarkdownTypeDefinitionError.invalidSerialization(error.localizedDescription) } + case .toml: + do { + return FrontMatterConversion.foundationValue( + try FrontMatterConversion.parse(content, format: .toml) + ) + } catch { + throw MarkdownTypeDefinitionError.invalidSerialization(error.localizedDescription) + } } } diff --git a/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeFixer.swift b/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeFixer.swift index 41c958d..349107a 100644 --- a/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeFixer.swift +++ b/Sources/MarkdownUtilitiesCore/Types/MarkdownTypeFixer.swift @@ -50,7 +50,11 @@ public enum MarkdownTypeFixer { } else if canAppendWithoutReformatting(frontmatterEdits, original: original) { prefix = try appending(frontmatterEdits, to: original.prefix) } else { - prefix = try renderFrontmatter(frontmatter, preserveEmpty: ensureFrontmatter) + prefix = try renderFrontmatter( + frontmatter, + preserveEmpty: ensureFrontmatter, + format: original.format ?? .yaml + ) } var updated = record @@ -62,12 +66,21 @@ public enum MarkdownTypeFixer { let parser = FrontMatterParser() var input = Substring(content) let parts = try parser.parse(&input) - let hasFrontmatter = content.starts(with: "---\n") + let hasFrontmatter = parts.format != nil guard hasFrontmatter else { - return ParsedRecordContent(frontmatter: [:], prefix: "", body: parts.body, hasFrontmatter: false) + return ParsedRecordContent( + frontmatter: [:], + prefix: "", + body: parts.body, + hasFrontmatter: false, + format: nil + ) } - let mapping = try YAMLConversion.parse(parts.rawFrontMatter) - let value = try JSONValue(any: YAMLConversion.safeNodeToSwiftValue(.mapping(mapping))) + let frontMatter = try FrontMatterConversion.parse( + parts.rawFrontMatter, + format: parts.format ?? .yaml + ) + let value = try JSONValue(any: FrontMatterConversion.foundationValue(frontMatter)) let prefix = parts.body.isEmpty ? content : String(content.dropLast(parts.body.count)) @@ -75,7 +88,8 @@ public enum MarkdownTypeFixer { frontmatter: value.objectValue ?? [:], prefix: prefix, body: parts.body, - hasFrontmatter: true + hasFrontmatter: true, + format: parts.format ) } @@ -83,7 +97,7 @@ public enum MarkdownTypeFixer { _ edits: [(path: [String], value: JSONValue)], original: ParsedRecordContent ) -> Bool { - guard original.hasFrontmatter else { return false } + guard original.hasFrontmatter, original.format == .yaml else { return false } var seen = Set(original.frontmatter.keys) for edit in edits { guard edit.path.count == 1, let key = edit.path.first, seen.insert(key).inserted else { @@ -151,19 +165,20 @@ public enum MarkdownTypeFixer { private static func renderFrontmatter( _ frontmatter: [String: JSONValue], - preserveEmpty: Bool + preserveEmpty: Bool, + format: FrontMatterFormat ) throws -> String { guard frontmatter.isEmpty == false else { - return preserveEmpty ? "---\n---\n" : "" + return preserveEmpty ? "\(format.delimiter)\n\(format.delimiter)\n" : "" + } + let converted = try FrontMatterConversion.fromFoundationValue( + JSONValue.object(frontmatter).foundationValue + ) + guard case .object(let object) = converted else { + throw MarkdownTypeFixerError.invalidFrontmatterBoundary } - let yaml = try Yams.dump( - object: JSONValue.object(frontmatter).foundationValue, - sortKeys: false, - sequenceStyle: .block, - mappingStyle: .block, - newLineScalarStyle: .plain - ).trimmingCharacters(in: .newlines) - return "---\n\(yaml)\n---\n" + let serialized = try FrontMatterConversion.serialize(object, format: format) + return "\(format.delimiter)\n\(serialized)\(format.delimiter)\n" } private struct ParsedRecordContent { @@ -171,6 +186,7 @@ public enum MarkdownTypeFixer { var prefix: String var body: String var hasFrontmatter: Bool + var format: FrontMatterFormat? } } diff --git a/Sources/MarkdownUtilitiesCore/Wikilink/MarkdownDocument+Wikilink.swift b/Sources/MarkdownUtilitiesCore/Wikilink/MarkdownDocument+Wikilink.swift index e53d1e1..daaad79 100644 --- a/Sources/MarkdownUtilitiesCore/Wikilink/MarkdownDocument+Wikilink.swift +++ b/Sources/MarkdownUtilitiesCore/Wikilink/MarkdownDocument+Wikilink.swift @@ -3,10 +3,9 @@ // MarkdownUtilities // -import Yams /// Adds wikilink behavior to ``MarkdownDocument``. extension MarkdownDocument { - /// Scans both the YAML frontmatter and the document body for wikilinks. + /// Scans both YAML or TOML frontmatter and the document body for wikilinks. /// /// Frontmatter wikilinks appear first (in the order they are encountered /// while walking the YAML tree), followed by body wikilinks in document order. @@ -29,33 +28,32 @@ extension MarkdownDocument { WikilinkScanner.scan(body) } - /// Scans only the YAML frontmatter for wikilinks. + /// Scans only the YAML or TOML frontmatter for wikilinks. /// /// Recursively walks all scalar values in the frontmatter mapping and scans /// each string for wikilinks. public func frontMatterWikilinks() -> [Wikilink] { var results: [Wikilink] = [] - collectWikilinks(from: .mapping(frontMatter), into: &results) + for (_, value) in frontMatter { + collectWikilinks(from: value, into: &results) + } return results } /// Recursively walks a YAML node tree, scanning all scalar string values for wikilinks. - private func collectWikilinks(from node: Yams.Node, into results: inout [Wikilink]) { - switch node { - case .scalar(let scalar): - results.append(contentsOf: WikilinkScanner.scan(scalar.string)) - - case .mapping(let mapping): + private func collectWikilinks(from value: FrontMatterValue, into results: inout [Wikilink]) { + switch value { + case .string(let string): + results.append(contentsOf: WikilinkScanner.scan(string)) + case .object(let mapping): for (_, value) in mapping { collectWikilinks(from: value, into: &results) } - - case .sequence(let sequence): + case .array(let sequence): for item in sequence { collectWikilinks(from: item, into: &results) } - - case .alias: + default: break } } diff --git a/Sources/md-utils/Commands/Body.swift b/Sources/md-utils/Commands/Body.swift index 46b728c..0b1ff35 100644 --- a/Sources/md-utils/Commands/Body.swift +++ b/Sources/md-utils/Commands/Body.swift @@ -13,7 +13,7 @@ import PathKit /// /// See for workflow details. extension CLIEntry { - /// Extracts Markdown body content after removing YAML frontmatter. + /// Extracts Markdown body content after removing YAML or TOML frontmatter. /// /// See for workflow details. struct Body: AsyncParsableCommand { @@ -21,7 +21,7 @@ extension CLIEntry { commandName: "body", abstract: "Extract the body content without frontmatter", discussion: """ - Extract the body content from Markdown files, excluding YAML frontmatter. + Extract the body content from Markdown files, excluding YAML or TOML frontmatter. Output can be in Markdown format (preserves formatting) or plain text (strips all Markdown formatting). diff --git a/Sources/md-utils/Commands/ExtractSection.swift b/Sources/md-utils/Commands/ExtractSection.swift index 3dd1fdd..592178a 100644 --- a/Sources/md-utils/Commands/ExtractSection.swift +++ b/Sources/md-utils/Commands/ExtractSection.swift @@ -194,20 +194,7 @@ extension CLIEntry { /// Reconstructs the full document including frontmatter if present. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - // If frontmatter is empty, return just the body - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - // Serialize frontmatter - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - // Reconstruct with frontmatter delimiters - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/Commands/FormatCommand.swift b/Sources/md-utils/Commands/FormatCommand.swift index f70c3aa..b345734 100644 --- a/Sources/md-utils/Commands/FormatCommand.swift +++ b/Sources/md-utils/Commands/FormatCommand.swift @@ -127,9 +127,7 @@ extension CLIEntry { } /// Reconstructs formatted Markdown output from parsed document parts. private func reconstructOutput(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { return doc.body } - let yaml = try YAMLConversion.serialize(doc.frontMatter) - return "---\n\(yaml)---\n\(doc.body)" + try doc.render() } } } diff --git a/Sources/md-utils/ConvertCommands/ToCSV.swift b/Sources/md-utils/ConvertCommands/ToCSV.swift index 9cfcf8e..c1f075c 100644 --- a/Sources/md-utils/ConvertCommands/ToCSV.swift +++ b/Sources/md-utils/ConvertCommands/ToCSV.swift @@ -18,7 +18,7 @@ extension CLIEntry.ConvertCommands { commandName: "to-csv", abstract: "Convert Markdown files with frontmatter to CSV format", discussion: """ - Converts a directory of Markdown files with YAML frontmatter into a single CSV file. + Converts a directory of Markdown files with YAML or TOML frontmatter into a single CSV file. Each YAML key becomes a column, and each file becomes a row. EXAMPLES: diff --git a/Sources/md-utils/ConvertCommands/ToText.swift b/Sources/md-utils/ConvertCommands/ToText.swift index 2d5a2bc..0cb4eab 100644 --- a/Sources/md-utils/ConvertCommands/ToText.swift +++ b/Sources/md-utils/ConvertCommands/ToText.swift @@ -82,7 +82,7 @@ extension CLIEntry.ConvertCommands { @Flag( name: .long, - help: "Include YAML frontmatter in the plain text output" + help: "Include YAML or TOML frontmatter in the plain text output" ) var includeFrontmatter: Bool = false diff --git a/Sources/md-utils/Documentation.docc/FrontmatterCommands.md b/Sources/md-utils/Documentation.docc/FrontmatterCommands.md index 25678cc..8afea4d 100644 --- a/Sources/md-utils/Documentation.docc/FrontmatterCommands.md +++ b/Sources/md-utils/Documentation.docc/FrontmatterCommands.md @@ -1,12 +1,14 @@ # Frontmatter Commands -Read, search, and mutate YAML frontmatter from the `frontmatter` command group. +Read and mutate YAML or TOML frontmatter from the `frontmatter` command group. ## Overview -The `frontmatter` command group, also available as `fm`, provides CRUD operations for Markdown YAML frontmatter. Commands can operate on one file, several files, or directories resolved through the shared global path options. +The `frontmatter` command group, also available as `fm`, provides CRUD operations for Markdown frontmatter. YAML uses `---` delimiter lines and TOML uses `+++`. Existing blocks retain their format; creation-capable commands accept `--frontmatter-format yaml|toml`, with YAML as the default. Commands can operate on one file, several files, or directories resolved through the shared global path options. -Common operations include reading values, setting values, checking for keys, removing or renaming keys, replacing or completely removing an entire frontmatter block, dumping frontmatter in multiple formats, searching with JMESPath, checking uniqueness, sorting keys, touching empty keys, and mutating array values. +Common operations include reading values, setting values, checking for keys, removing or renaming keys, replacing or completely removing an entire frontmatter block, dumping frontmatter in multiple formats, checking uniqueness, sorting keys, and mutating array values. `fm search` is intentionally YAML-only. TOML has no null value, so `fm touch` cannot add an empty TOML key and reports an error instead. + +Do not put comments in frontmatter that will be mutated by `md-utils`. YAML and TOML are parsed into a format-neutral value model and serialized again, so comments are not guaranteed to survive. ## Removing Complete Frontmatter @@ -49,7 +51,9 @@ literals from shell command substitution. ## Output Semantics -Commands that report values preserve the distinction between a missing key and a key whose YAML value is null. Machine-readable formats should be preferred when that distinction matters. +Commands that report values preserve the distinction between a missing key and a YAML null value. TOML cannot represent null. Machine-readable formats should be preferred when that distinction matters. + +`--format toml` is available anywhere the shared structured-output format is supported. Because TOML requires a document root to be a table, scalar and array roots are emitted under a stable `value` key. `fm dump --format raw` emits the source format, and `--include-delimiters` uses the matching `---` or `+++` lines. `fm dump` is read-only and automatically selects Markdown, plain-text, and mapped non-Markdown files. A sole explicit file outputs its mapping directly. diff --git a/Sources/md-utils/Documentation.docc/RulesValidationCommands.md b/Sources/md-utils/Documentation.docc/RulesValidationCommands.md index a0b3a87..a4fa0cd 100644 --- a/Sources/md-utils/Documentation.docc/RulesValidationCommands.md +++ b/Sources/md-utils/Documentation.docc/RulesValidationCommands.md @@ -38,7 +38,7 @@ Mapped extensions reuse the shipped wrapped-frontmatter syntaxes: - `lua-block`: Lua files - `markdown-text`: `.txt`, only with `--include-non-md` -Wrapped YAML supports frontmatter field predicates, JMESPath, `$md-utils` type +Wrapped YAML or TOML supports frontmatter field predicates, JMESPath, `$md-utils` type hints, and JSON Schema validation. Raw body predicates and counts operate on the host source after the wrapper is removed; they can therefore match comments or string literals. Markdown headings, sections, heading relationships, required @@ -49,7 +49,7 @@ requires frontmatter for one of those files reports that no syntax mapping exist ## Supported Checks -- `frontmatterSchema`: validates parsed YAML frontmatter against a JSON Schema file. +- `frontmatterSchema`: validates parsed YAML or TOML frontmatter against a JSON Schema file. - `requiredHeading`: requires an exact Markdown heading text in a Markdown body. - `maxBodyLines`: limits Markdown body line count. - `maxBodyWords`: limits Markdown body word count. @@ -84,4 +84,4 @@ Supported file metadata matcher operators are `pathRegex`, `filenameEquals`, `ex ## Failure Behavior -Invalid YAML frontmatter is reported as an error for matched rules because frontmatter predicates and schema checks cannot proceed. Files without required frontmatter fail when the matched `frontmatterSchema` check requires frontmatter, and are skipped for optional frontmatter schema checks. +Invalid YAML or TOML frontmatter is reported as an error for matched rules because frontmatter predicates and schema checks cannot proceed. Files without required frontmatter fail when the matched `frontmatterSchema` check requires frontmatter, and are skipped for optional frontmatter schema checks. diff --git a/Sources/md-utils/Documentation.docc/TypesCommands.md b/Sources/md-utils/Documentation.docc/TypesCommands.md index 3f336c3..2a2c1cf 100644 --- a/Sources/md-utils/Documentation.docc/TypesCommands.md +++ b/Sources/md-utils/Documentation.docc/TypesCommands.md @@ -4,14 +4,15 @@ Create, inspect, assess, identify, verify, and repair typed Markdown records. ## Definitions -Project definitions are YAML or JSON files under `.md-utils/types/`. Definition filenames must end in `.mdtype.yaml`, `.mdtype.yml`, or `.mdtype.json`. Other YAML and JSON files in the directory are ignored. Add a definition scaffold with: +Project definitions are YAML, JSON, or TOML files under `.md-utils/types/`. Definition filenames must end in `.mdtype.yaml`, `.mdtype.yml`, `.mdtype.json`, or `.mdtype.toml`. Other structured-data files in the directory are ignored. Add a definition scaffold with: ```bash md-utils types add Book --version 1.0.0 md-utils types add Publishable --version draft-3 --format json +md-utils types add Article --version 1.0.0 --format toml ``` -These commands create `.md-utils/types/` when needed, then add `.md-utils/types/book.mdtype.yaml` and `.md-utils/types/publishable.mdtype.json`. The declared `name` inside each definition remains the stable type identity; the filename only identifies the file as a Markdown type definition. +These commands create `.md-utils/types/` when needed and use the compound extension matching the selected format. The declared `name` inside each definition remains the stable type identity; the filename only identifies the file as a Markdown type definition. Use `types list`, `types describe Book`, and `types doctor` to inspect compiled definitions. `types schema` prints the JSON Schema for definition format version `1`. diff --git a/Sources/md-utils/ExploreCommands/Explore.swift b/Sources/md-utils/ExploreCommands/Explore.swift index 44a5a5e..846548a 100644 --- a/Sources/md-utils/ExploreCommands/Explore.swift +++ b/Sources/md-utils/ExploreCommands/Explore.swift @@ -91,7 +91,7 @@ extension CLIEntry { @Flag( name: [.customShort("F"), .customLong("expand-frontmatter")], - help: "Expand YAML frontmatter when present" + help: "Expand YAML or TOML frontmatter when present" ) var expandFrontmatter: Bool = false diff --git a/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift b/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift index e72bd9a..268549a 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayAppend.swift @@ -51,6 +51,9 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { @Option(name: .shortAndLong, help: "The value to append to the array") var value: String + @Option(name: .long, help: "Frontmatter format to create or convert to (yaml, toml)") + var frontmatterFormat: FrontMatterFormat? + @Flag(name: .long, help: "Skip if value already exists in array") var skipDuplicates: Bool = false @@ -83,6 +86,7 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { includeNonMarkdown: includeNonMD ) var doc = parsed.document + if let frontmatterFormat { doc.frontMatterFormat = frontmatterFormat } // Get array (creates empty if doesn't exist, errors if not an array) let sequence = try ArrayHelpers.getOrCreateArrayKey(key, in: doc, path: path) @@ -102,7 +106,7 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { // Append value let updatedSequence = ArrayHelpers.append(value: value, to: sequence) - doc.frontMatter[key] = .sequence(updatedSequence) + doc.frontMatter[key] = .array(updatedSequence) // Write back try FrontMatterCLIMutator.write(doc, parsed: parsed, to: path) diff --git a/Sources/md-utils/FrontMatterCommands/ArrayCommands.swift b/Sources/md-utils/FrontMatterCommands/ArrayCommands.swift index d6079fa..79f51ea 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayCommands.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayCommands.swift @@ -18,7 +18,7 @@ extension CLIEntry.FrontMatterCommands { commandName: "array", abstract: "Array manipulation commands for frontmatter", discussion: NonMarkdownFrontMatterHelp.appending(to: """ - Manipulate arrays in YAML frontmatter with various subcommands. + Manipulate arrays in YAML or TOML frontmatter with various subcommands. SUBCOMMANDS: contains Check if arrays contain specific values diff --git a/Sources/md-utils/FrontMatterCommands/ArrayContains.swift b/Sources/md-utils/FrontMatterCommands/ArrayContains.swift index d5869c9..2346bb2 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayContains.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayContains.swift @@ -9,7 +9,6 @@ import ArgumentParser import Foundation import MarkdownUtilitiesCore import PathKit -import Yams /// Adds Markdown document behavior to ``CLIEntry.FrontMatterCommands``. /// /// See for workflow details. @@ -106,7 +105,7 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { searchedCount += 1 // 2. Check if key exists and is an array (skip if not) - let sequence: Yams.Node.Sequence + let sequence: [FrontMatterValue] do { sequence = try ArrayHelpers.validateArrayKey(key, in: doc, path: path) } catch { diff --git a/Sources/md-utils/FrontMatterCommands/ArrayHelpers.swift b/Sources/md-utils/FrontMatterCommands/ArrayHelpers.swift index 74d6191..7e7a208 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayHelpers.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayHelpers.swift @@ -8,7 +8,6 @@ import Foundation import MarkdownUtilitiesCore import PathKit -import Yams /// Shared utilities for array manipulation commands enum ArrayHelpers { @@ -24,7 +23,7 @@ enum ArrayHelpers { _ key: String, in doc: MarkdownDocument, path: Path - ) throws -> Yams.Node.Sequence { + ) throws -> [FrontMatterValue] { guard doc.hasKey(key) else { throw ArrayError.keyNotFound(key: key, path: path.string) } @@ -33,7 +32,7 @@ enum ArrayHelpers { throw ArrayError.cannotRetrieveValue(key: key, path: path.string) } - guard case .sequence(let sequence) = node else { + guard case .array(let sequence) = node else { throw ArrayError.notAnArray(key: key, path: path.string) } @@ -51,10 +50,10 @@ enum ArrayHelpers { _ key: String, in doc: MarkdownDocument, path: Path - ) throws -> Yams.Node.Sequence { + ) throws -> [FrontMatterValue] { // If key doesn't exist, return empty sequence guard doc.hasKey(key) else { - return Yams.Node.Sequence() + return [] } guard let node = doc.getValue(forKey: key) else { @@ -62,7 +61,7 @@ enum ArrayHelpers { } // If key exists but is not an array, throw error - guard case .sequence(let sequence) = node else { + guard case .array(let sequence) = node else { throw ArrayError.notAnArray(key: key, path: path.string) } @@ -77,7 +76,7 @@ enum ArrayHelpers { /// - Returns: True if the value is found, false otherwise static func containsValue( _ searchValue: String, - in sequence: Yams.Node.Sequence, + in sequence: [FrontMatterValue], caseInsensitive: Bool ) -> Bool { let compareValue = caseInsensitive ? searchValue.lowercased() : searchValue @@ -86,11 +85,9 @@ enum ArrayHelpers { let element = sequence[i] // Only compare scalar (string) values - guard case .scalar(let scalar) = element else { + guard case .string(let elementString) = element else { continue } - - let elementString = scalar.string let compareElement = caseInsensitive ? elementString.lowercased() : elementString if compareElement == compareValue { @@ -108,10 +105,10 @@ enum ArrayHelpers { /// - Returns: A new sequence with the value appended static func append( value: String, - to sequence: Yams.Node.Sequence - ) -> Yams.Node.Sequence { + to sequence: [FrontMatterValue] + ) -> [FrontMatterValue] { var newSequence = sequence - newSequence.append(.scalar(.init(value))) + newSequence.append(.string(value)) return newSequence } @@ -122,10 +119,10 @@ enum ArrayHelpers { /// - Returns: A new sequence with the value prepended static func prepend( value: String, - to sequence: Yams.Node.Sequence - ) -> Yams.Node.Sequence { + to sequence: [FrontMatterValue] + ) -> [FrontMatterValue] { var newSequence = sequence - newSequence.insert(.scalar(.init(value)), at: 0) + newSequence.insert(.string(value), at: 0) return newSequence } @@ -137,9 +134,9 @@ enum ArrayHelpers { /// - Returns: A new sequence with the first occurrence removed, or nil if value not found static func removeFirst( value: String, - from sequence: Yams.Node.Sequence, + from sequence: [FrontMatterValue], caseInsensitive: Bool - ) -> Yams.Node.Sequence? { + ) -> [FrontMatterValue]? { let compareValue = caseInsensitive ? value.lowercased() : value var newSequence = sequence @@ -147,11 +144,9 @@ enum ArrayHelpers { let element = newSequence[i] // Only compare scalar (string) values - guard case .scalar(let scalar) = element else { + guard case .string(let elementString) = element else { continue } - - let elementString = scalar.string let compareElement = caseInsensitive ? elementString.lowercased() : elementString if compareElement == compareValue { diff --git a/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift b/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift index 4603be4..6a4cac0 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayPrepend.swift @@ -52,6 +52,9 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { @Option(name: .shortAndLong, help: "The value to prepend to the array") var value: String + @Option(name: .long, help: "Frontmatter format to create or convert to (yaml, toml)") + var frontmatterFormat: FrontMatterFormat? + @Flag(name: .long, help: "Skip if value already exists in array") var skipDuplicates: Bool = false @@ -84,6 +87,7 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { includeNonMarkdown: includeNonMD ) var doc = parsed.document + if let frontmatterFormat { doc.frontMatterFormat = frontmatterFormat } // Get array (creates empty if doesn't exist, errors if not an array) let sequence = try ArrayHelpers.getOrCreateArrayKey(key, in: doc, path: path) @@ -103,7 +107,7 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { // Prepend value let updatedSequence = ArrayHelpers.prepend(value: value, to: sequence) - doc.frontMatter[key] = .sequence(updatedSequence) + doc.frontMatter[key] = .array(updatedSequence) // Write back try FrontMatterCLIMutator.write(doc, parsed: parsed, to: path) diff --git a/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift b/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift index 4951abb..bd85e67 100644 --- a/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift +++ b/Sources/md-utils/FrontMatterCommands/ArrayRemove.swift @@ -88,7 +88,7 @@ extension CLIEntry.FrontMatterCommands.ArrayCommands { continue } - doc.frontMatter[key] = .sequence(updatedSequence) + doc.frontMatter[key] = .array(updatedSequence) // Write back try FrontMatterCLIMutator.write(doc, parsed: parsed, to: path) diff --git a/Sources/md-utils/FrontMatterCommands/Dump.swift b/Sources/md-utils/FrontMatterCommands/Dump.swift index 9d8023a..bd22025 100644 --- a/Sources/md-utils/FrontMatterCommands/Dump.swift +++ b/Sources/md-utils/FrontMatterCommands/Dump.swift @@ -20,7 +20,7 @@ extension CLIEntry.FrontMatterCommands { commandName: "dump", abstract: "Dump entire frontmatter in specified format", discussion: NonMarkdownFrontMatterHelp.appendingForDump(to: """ - Outputs the complete frontmatter from files in various formats: JSON, YAML, raw, or plist. + Outputs complete frontmatter as JSON, YAML, TOML, raw source, or plist. Supports multiple files and directory processing with recursive mode. @@ -80,10 +80,10 @@ extension CLIEntry.FrontMatterCommands { @OptionGroup var options: GlobalOptions - @Option(name: .shortAndLong, help: "Output format (json, yaml, raw, plist)") + @Option(name: .shortAndLong, help: "Output format (json, yaml, toml, raw, plist)") var format: OutputFormat = .json - @Flag(name: .long, help: "Include --- delimiters in YAML/raw output") + @Flag(name: .long, help: "Include format-appropriate delimiters in YAML/TOML/raw output") var includeDelimiters: Bool = false @Flag(name: .long, help: "Use cat-style headers (==> path <==) instead of collection output for multiple files") @@ -108,14 +108,14 @@ extension CLIEntry.FrontMatterCommands { do { let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: true) - if includeDelimiters && (format == .yaml || format == .raw) { - Swift.print("---") + if includeDelimiters, let delimiter = outputDelimiter(for: doc) { + Swift.print(delimiter) } - try print(node: .mapping(doc.frontMatter), format: format) + try print(frontMatter: doc.frontMatter, format: format, sourceFormat: doc.frontMatterFormat) - if includeDelimiters && (format == .yaml || format == .raw) { - Swift.print("---") + if includeDelimiters, let delimiter = outputDelimiter(for: doc) { + Swift.print(delimiter) } } catch { CLIStyle.writeError("\(CLIStyle.path(file.string)): \(error.localizedDescription)") @@ -133,14 +133,14 @@ extension CLIEntry.FrontMatterCommands { do { let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: true) - if includeDelimiters && (format == .yaml || format == .raw) { - Swift.print("---") + if includeDelimiters, let delimiter = outputDelimiter(for: doc) { + Swift.print(delimiter) } - try print(node: .mapping(doc.frontMatter), format: format) + try print(frontMatter: doc.frontMatter, format: format, sourceFormat: doc.frontMatterFormat) - if includeDelimiters && (format == .yaml || format == .raw) { - Swift.print("---") + if includeDelimiters, let delimiter = outputDelimiter(for: doc) { + Swift.print(delimiter) } } catch { CLIStyle.writeError("\(CLIStyle.path(file.string)): \(error.localizedDescription)") @@ -175,11 +175,7 @@ extension CLIEntry.FrontMatterCommands { continue } - let node = Yams.Node.mapping(parsed.document.frontMatter) - - guard var dict = try YAMLConversion.safeNodeToSwiftValue(node) as? [String: Any] else { - continue - } + var dict = FrontMatterConversion.foundationValue(parsed.document.frontMatter) dict["$path"] = file.string frontMatter.append(dict) @@ -198,5 +194,14 @@ extension CLIEntry.FrontMatterCommands { } if hasErrors { throw ExitCode.failure } } + + private func outputDelimiter(for document: MarkdownDocument) -> String? { + switch format { + case .yaml: FrontMatterFormat.yaml.delimiter + case .toml: FrontMatterFormat.toml.delimiter + case .raw: (document.frontMatterFormat ?? .yaml).delimiter + case .json, .plist: nil + } + } } } diff --git a/Sources/md-utils/FrontMatterCommands/FrontMatterCommands.swift b/Sources/md-utils/FrontMatterCommands/FrontMatterCommands.swift index 1d7a2f2..cbae82f 100644 --- a/Sources/md-utils/FrontMatterCommands/FrontMatterCommands.swift +++ b/Sources/md-utils/FrontMatterCommands/FrontMatterCommands.swift @@ -12,11 +12,15 @@ extension CLIEntry { struct FrontMatterCommands: AsyncParsableCommand { static let configuration = CommandConfiguration( commandName: "frontmatter", - abstract: "Manipulate YAML frontmatter in Markdown and mapped text files", + abstract: "Manipulate YAML or TOML frontmatter in Markdown and mapped text files", discussion: NonMarkdownFrontMatterHelp.appending(to: """ - Provides CRUD operations for YAML frontmatter in Markdown files and in + Provides CRUD operations for YAML or TOML frontmatter in Markdown files and in non-Markdown text files with shipped syntax mappings. + Existing blocks preserve their format. Creation-capable commands accept + --frontmatter-format yaml|toml and default to YAML. Comments in either + format are not guaranteed to survive mutation. + By default, directory and multi-file operations remain Markdown-only. Use --include-non-md on supported commands to include mapped files. """), diff --git a/Sources/md-utils/FrontMatterCommands/FrontMatterJMESPath.swift b/Sources/md-utils/FrontMatterCommands/FrontMatterJMESPath.swift index 3e5f963..3a90698 100644 --- a/Sources/md-utils/FrontMatterCommands/FrontMatterJMESPath.swift +++ b/Sources/md-utils/FrontMatterCommands/FrontMatterJMESPath.swift @@ -6,7 +6,6 @@ import Foundation import JMESPath import MarkdownUtilitiesCore -import Yams /// Shared JMESPath support for frontmatter commands. enum FrontMatterJMESPath { @@ -17,7 +16,10 @@ enum FrontMatterJMESPath { /// Converts parsed frontmatter into the Foundation representation expected by JMESPath. static func object(from document: MarkdownDocument) throws -> Any { - try YAMLConversion.safeNodeToSwiftValue(.mapping(document.frontMatter)) + guard document.frontMatterFormat != .toml else { + throw FrontMatterCommandError(message: "TOML frontmatter is not supported by fm search") + } + return FrontMatterConversion.foundationValue(document.frontMatter) } /// Extracts the useful description from the JMESPath package's wrapped errors. diff --git a/Sources/md-utils/FrontMatterCommands/Get.swift b/Sources/md-utils/FrontMatterCommands/Get.swift index c23fb8f..f593865 100644 --- a/Sources/md-utils/FrontMatterCommands/Get.swift +++ b/Sources/md-utils/FrontMatterCommands/Get.swift @@ -18,7 +18,7 @@ extension CLIEntry.FrontMatterCommands { commandName: "get", abstract: "Get a frontmatter value by key", discussion: NonMarkdownFrontMatterHelp.appending(to: """ - Retrieves the value of a specified key from YAML frontmatter. + Retrieves the value of a specified key from YAML or TOML frontmatter. If the key doesn't exist, the command exits with an error code. When processing multiple files, the filename is included in the output. @@ -83,7 +83,7 @@ extension CLIEntry.FrontMatterCommands { if let value = doc.getValue(forKey: key) { // Key found — include "value" (NSNull if YAML value is null) - let jsonValue = try YAMLConversion.safeNodeToSwiftValue(value) + let jsonValue = FrontMatterConversion.foundationValue(value) results.append(["path": file.string, "value": jsonValue]) } else { // Key missing — omit "value" key; absence is the signal @@ -137,29 +137,22 @@ extension CLIEntry.FrontMatterCommands { } } - /// Format a Yams.Node value for display - private func formatNodeValue(_ node: Yams.Node, format: OutputFormat) -> String { - // Handle scalar values - if let string = node.string { - return string - } - - // Handle numbers - if let int = node.int { - return String(int) - } - - if let float = node.float { - return String(float) - } - - // Handle booleans - if let bool = node.bool { - return String(bool) - } - - // Handle arrays - if let sequence = node.sequence { + /// Formats a frontmatter value for display. + private func formatNodeValue(_ node: FrontMatterValue, format: OutputFormat) -> String { + switch node { + case .null: + return "null" + case .string(let value): + return value + case .integer(let value): + return String(value) + case .number(let value): + return String(value) + case .boolean(let value): + return String(value) + case .offsetDateTime, .localDateTime, .localDate, .localTime: + return String(describing: FrontMatterConversion.foundationValue(node)) + case .array(let sequence): let items = sequence.map { formatNodeValue($0, format: .inline) } switch format { @@ -172,18 +165,12 @@ extension CLIEntry.FrontMatterCommands { "\(index + 1). \(item)" }.joined(separator: "\n") } - } - - // Handle mappings/objects - if let mapping = node.mapping { + case .object(let mapping): let pairs = mapping.map { key, value in - "\(formatNodeValue(key, format: .inline)): \(formatNodeValue(value, format: .inline))" + "\(key): \(formatNodeValue(value, format: .inline))" } return "{\(pairs.joined(separator: ", "))}" } - - // Fallback - return String(describing: node) } } } diff --git a/Sources/md-utils/FrontMatterCommands/List.swift b/Sources/md-utils/FrontMatterCommands/List.swift index 3a22913..851b25e 100644 --- a/Sources/md-utils/FrontMatterCommands/List.swift +++ b/Sources/md-utils/FrontMatterCommands/List.swift @@ -18,7 +18,7 @@ extension CLIEntry.FrontMatterCommands { commandName: "list", abstract: "List all keys in frontmatter", discussion: NonMarkdownFrontMatterHelp.appending(to: """ - Lists all keys present in the YAML frontmatter of Markdown files. + Lists all keys present in the YAML or TOML frontmatter of Markdown files. When processing multiple files, each file's keys are prefixed with the filename. Keys are listed one per line in alphabetical order. @@ -47,9 +47,7 @@ extension CLIEntry.FrontMatterCommands { let doc = try FrontMatterCLIReader.document(at: file, includeNonMarkdown: includeNonMD) // Extract keys from frontmatter - let keys = Array(doc.frontMatter.keys) - .compactMap { $0.string } - .sorted() + let keys = doc.frontMatter.keys.sorted() // Print keys if keys.isEmpty { diff --git a/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift b/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift index df6de82..f49ba76 100644 --- a/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift +++ b/Sources/md-utils/FrontMatterCommands/NonMarkdownFrontMatterSupport.swift @@ -27,8 +27,8 @@ enum NonMarkdownFrontMatterHelp { FRONTMATTER ON NON-MD FILES A supported non-Markdown file uses the wrapper mapped from its extension. A complete wrapped frontmatter block must contain, on separate complete LF - lines: the mapped opening wrapper, an opening ---, a YAML mapping, a closing - ---, and the matching mapped closing wrapper. For example: + lines: the mapped opening wrapper, matching YAML --- or TOML +++ delimiter + lines, a mapping, and the matching mapped closing wrapper. For example: /* --- @@ -36,6 +36,12 @@ enum NonMarkdownFrontMatterHelp { --- */ + /* + +++ + title = "Example" + +++ + */ + The block may occur anywhere, though placement near the beginning is recommended. Incomplete blocks are treated as absent; multiple complete blocks are invalid. Plain .txt uses ordinary Markdown-style frontmatter. @@ -62,13 +68,13 @@ enum NonMarkdownFrontMatterHelp { /// Describes how a selected file represents frontmatter. /// -/// Markdown and opted-in plain-text files use ordinary leading `---` markers. +/// Markdown and opted-in plain-text files use ordinary leading format markers. /// Other supported text files use the shipped wrapper mapped from their extension. enum FrontMatterFileSyntax: Equatable { /// Ordinary Markdown-style frontmatter. case markdown - /// YAML frontmatter enclosed by a host-language wrapper. + /// YAML or TOML frontmatter enclosed by a host-language wrapper. case wrapped(FrontMatterSyntax) /// Resolves the frontmatter representation for a selected file. @@ -122,7 +128,7 @@ struct ParsedFrontMatterFile { var hasFrontMatterBlock: Bool { switch syntax { case .markdown: - return document.body != source + return document.frontMatterFormat != nil case .wrapped: return wrappedBlock != nil } @@ -130,13 +136,13 @@ struct ParsedFrontMatterFile { /// Parses frontmatter from a source snapshot using its selected representation. /// - /// Later wrapped blocks are located but their YAML is never converted or merged. + /// Later wrapped blocks are located but their frontmatter is never converted or merged. /// /// - Parameters: /// - source: The complete LF text snapshot. /// - syntax: The representation selected for the file. /// - Returns: Parsed frontmatter tied to `source`. - /// - Throws: A YAML conversion error when the first complete block is invalid. + /// - Throws: A conversion error when the first complete block is invalid. static func parse(source: String, syntax: FrontMatterFileSyntax) throws -> ParsedFrontMatterFile { switch syntax { case .markdown: @@ -149,11 +155,18 @@ struct ParsedFrontMatterFile { ) case .wrapped(let wrapper): let scan = WrappedFrontMatterParser(syntax: wrapper).parse(source) - let mapping = try YAMLConversion.parse(scan.firstBlock?.rawYAML ?? "") + let format = scan.firstBlock?.format + let frontMatter = try format.map { + try FrontMatterConversion.parse(scan.firstBlock?.rawFrontMatter ?? "", format: $0) + } ?? FrontMatter() return ParsedFrontMatterFile( source: source, syntax: syntax, - document: MarkdownDocument(frontMatter: mapping, body: source), + document: MarkdownDocument( + frontMatter: frontMatter, + body: source, + frontMatterFormat: format + ), wrappedBlock: scan.firstBlock, additionalOpeningLines: scan.additionalOpeningLines ) @@ -167,14 +180,15 @@ struct ParsedFrontMatterFile { /// /// - Parameter updatedDocument: The document containing the updated mapping. /// - Returns: Complete updated file text. - /// - Throws: A YAML serialization error. + /// - Throws: A frontmatter serialization error. func rendering(_ updatedDocument: MarkdownDocument) throws -> String { switch syntax { case .markdown: return try updatedDocument.render() case .wrapped(let wrapper): - let yaml = try YAMLConversion.serialize(updatedDocument.frontMatter) - let renderedBlock = "\(wrapper.openingWrapper)\n---\n\(yaml)---\n\(wrapper.closingWrapper)" + let format = updatedDocument.frontMatterFormat ?? .yaml + let serialized = try FrontMatterConversion.serialize(updatedDocument.frontMatter, format: format) + let renderedBlock = "\(wrapper.openingWrapper)\n\(format.delimiter)\n\(serialized)\(format.delimiter)\n\(wrapper.closingWrapper)" guard let wrappedBlock else { return "\(renderedBlock)\n\n\(source)" } diff --git a/Sources/md-utils/FrontMatterCommands/Replace.swift b/Sources/md-utils/FrontMatterCommands/Replace.swift index 330e8d6..644b04b 100644 --- a/Sources/md-utils/FrontMatterCommands/Replace.swift +++ b/Sources/md-utils/FrontMatterCommands/Replace.swift @@ -41,6 +41,7 @@ extension CLIEntry.FrontMatterCommands { SUPPORTED FORMATS: - json: JavaScript Object Notation - yaml: YAML Ain't Markup Language + - toml: Tom's Obvious, Minimal Language - plist: Apple PropertyList XML VALIDATION: @@ -73,9 +74,12 @@ extension CLIEntry.FrontMatterCommands { @Option(name: .long, help: "Path to file containing new frontmatter") var fromFile: String? - @Option(name: [.short, .long], help: "Data format (json, yaml, plist)") + @Option(name: [.short, .long], help: "Data format (json, yaml, toml, plist)") var format: OutputFormat = .json + @Option(name: .long, help: "Frontmatter format to create or convert to (yaml, toml)") + var frontmatterFormat: FrontMatterFormat? + @Flag(name: [.customShort("y"), .long], help: "Skip confirmation prompt") var yes: Bool = false @@ -108,17 +112,23 @@ extension CLIEntry.FrontMatterCommands { } // Parse to mapping based on format - let newFrontMatter: Yams.Node.Mapping + let newFrontMatter: FrontMatter do { switch format { case .json: - newFrontMatter = try YAMLConversion.parseJSON(dataString) + newFrontMatter = try FrontMatterConversion.fromYAMLMapping( + YAMLConversion.parseJSON(dataString) + ) case .yaml, .raw: - newFrontMatter = try YAMLConversion.parse(dataString) + newFrontMatter = try FrontMatterConversion.parse(dataString, format: .yaml) + case .toml: + newFrontMatter = try FrontMatterConversion.parse(dataString, format: .toml) case .plist: - newFrontMatter = try YAMLConversion.parsePlist(dataString) + newFrontMatter = try FrontMatterConversion.fromYAMLMapping( + YAMLConversion.parsePlist(dataString) + ) } - } catch let error as YAMLConversionError { + } catch { throw ValidationError(error.localizedDescription) } @@ -146,7 +156,7 @@ extension CLIEntry.FrontMatterCommands { /// Replaces frontmatter in one Markdown file. /// /// See for workflow details. - private func replaceInFile(path: Path, newFrontMatter: Yams.Node.Mapping) throws { + private func replaceInFile(path: Path, newFrontMatter: FrontMatter) throws { let parsed = try FrontMatterCLIMutator.parsedFile( at: path, includeNonMarkdown: includeNonMD @@ -177,6 +187,7 @@ extension CLIEntry.FrontMatterCommands { } var doc = parsed.document + if let frontmatterFormat { doc.frontMatterFormat = frontmatterFormat } // Replace frontmatter (direct assignment) doc.frontMatter = newFrontMatter diff --git a/Sources/md-utils/FrontMatterCommands/Search.swift b/Sources/md-utils/FrontMatterCommands/Search.swift index a08326f..d9d8071 100644 --- a/Sources/md-utils/FrontMatterCommands/Search.swift +++ b/Sources/md-utils/FrontMatterCommands/Search.swift @@ -21,7 +21,8 @@ extension CLIEntry.FrontMatterCommands { discussion: NonMarkdownFrontMatterHelp.appending(to: """ Search for files whose frontmatter matches a JMESPath expression. - The query is evaluated against each file's YAML frontmatter. + The query is evaluated against each file's YAML frontmatter. TOML + frontmatter is intentionally outside the scope of this command. Files where the expression evaluates to true (or truthy) are included. Directories are searched recursively. @@ -106,6 +107,10 @@ extension CLIEntry.FrontMatterCommands { /// /// See for workflow details. mutating func run() async throws { + guard format != .toml else { + throw ValidationError("TOML output is not supported by fm search") + } + // Convert path strings to Path objects let paths = pathStrings.isEmpty ? [Path.current] : pathStrings.map { Path($0) } // Compile the JMESPath expression once @@ -147,6 +152,8 @@ extension CLIEntry.FrontMatterCommands { case .plist: // For search results (file paths), use plist format try printAny(matchingFiles, format: .plist) + case .toml: + break } } diff --git a/Sources/md-utils/FrontMatterCommands/Set.swift b/Sources/md-utils/FrontMatterCommands/Set.swift index 5e04c9f..beada93 100644 --- a/Sources/md-utils/FrontMatterCommands/Set.swift +++ b/Sources/md-utils/FrontMatterCommands/Set.swift @@ -34,6 +34,9 @@ extension CLIEntry.FrontMatterCommands { @Option(name: .long, help: "The value to set") var value: String + @Option(name: .long, help: "Frontmatter format to create or convert to (yaml, toml)") + var frontmatterFormat: FrontMatterFormat? + @Flag(name: .long, help: "Process mapped non-Markdown files") var includeNonMD = false @@ -66,6 +69,7 @@ extension CLIEntry.FrontMatterCommands { ) var doc = parsed.document + if let frontmatterFormat { doc.frontMatterFormat = frontmatterFormat } doc.setValue(value, forKey: key) diff --git a/Sources/md-utils/FrontMatterCommands/Touch.swift b/Sources/md-utils/FrontMatterCommands/Touch.swift index a1733da..71d0687 100644 --- a/Sources/md-utils/FrontMatterCommands/Touch.swift +++ b/Sources/md-utils/FrontMatterCommands/Touch.swift @@ -39,6 +39,9 @@ extension CLIEntry.FrontMatterCommands { ) var keys: String + @Option(name: .long, help: "Frontmatter format to create or convert to (yaml, toml)") + var frontmatterFormat: FrontMatterFormat? + @Flag(name: .long, help: "Process mapped non-Markdown files") var includeNonMD = false @@ -73,6 +76,7 @@ extension CLIEntry.FrontMatterCommands { includeNonMarkdown: includeNonMD ) var doc = parsed.document + if let frontmatterFormat { doc.frontMatterFormat = frontmatterFormat } let missingKeys = keyList.filter { doc.hasKey($0) == false } guard missingKeys.isEmpty == false else { continue } try FrontMatterCLIMutator.authorizeCreationIfNeeded( diff --git a/Sources/md-utils/FrontMatterCommands/Unique.swift b/Sources/md-utils/FrontMatterCommands/Unique.swift index 441474b..50da4e5 100644 --- a/Sources/md-utils/FrontMatterCommands/Unique.swift +++ b/Sources/md-utils/FrontMatterCommands/Unique.swift @@ -16,7 +16,7 @@ extension CLIEntry.FrontMatterCommands { commandName: "unique", abstract: "Check that a frontmatter value is unique across files", discussion: NonMarkdownFrontMatterHelp.appending(to: """ - Evaluates one JMESPath expression against each file's YAML frontmatter and + Evaluates one JMESPath expression against each file's YAML or TOML frontmatter and checks that the selected scalar value is unique. COLLECTION MODE: @@ -80,7 +80,7 @@ extension CLIEntry.FrontMatterCommands { @Flag(name: .long, help: "Process mapped non-Markdown files") var includeNonMD = false - @Option(name: .long, help: "Output format: text, json, or yaml") + @Option(name: .long, help: "Output format: text, json, yaml, or toml") var format: UniqueOutputFormat = .text mutating func run() async throws { @@ -130,6 +130,7 @@ enum UniqueOutputFormat: String, ExpressibleByArgument { case text case json case yaml + case toml } enum UniqueMode: String { @@ -440,6 +441,8 @@ enum UniqueRenderer { return try YAMLConversion.anyToJSON(report.foundationObject, options: [.prettyPrinted, .sortedKeys]) case .yaml: return try YAMLConversion.anyToYAML(report.foundationObject) + case .toml: + return try FrontMatterConversion.serializeTOMLValue(report.foundationObject) } } diff --git a/Sources/md-utils/HeadingCommands/DemoteHeading.swift b/Sources/md-utils/HeadingCommands/DemoteHeading.swift index c54ede8..3fa9e00 100644 --- a/Sources/md-utils/HeadingCommands/DemoteHeading.swift +++ b/Sources/md-utils/HeadingCommands/DemoteHeading.swift @@ -105,20 +105,7 @@ extension CLIEntry { /// Reconstructs the full document including frontmatter if present. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - // If frontmatter is empty, return just the body - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - // Serialize frontmatter - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - // Reconstruct with frontmatter delimiters - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/HeadingCommands/PromoteHeading.swift b/Sources/md-utils/HeadingCommands/PromoteHeading.swift index 7ed9e8a..f8ca7ee 100644 --- a/Sources/md-utils/HeadingCommands/PromoteHeading.swift +++ b/Sources/md-utils/HeadingCommands/PromoteHeading.swift @@ -105,20 +105,7 @@ extension CLIEntry { /// Reconstructs the full document including frontmatter if present. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - // If frontmatter is empty, return just the body - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - // Serialize frontmatter - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - // Reconstruct with frontmatter delimiters - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/OKFCommands/OKFSupport.swift b/Sources/md-utils/OKFCommands/OKFSupport.swift index 1b76721..4f676d5 100644 --- a/Sources/md-utils/OKFCommands/OKFSupport.swift +++ b/Sources/md-utils/OKFCommands/OKFSupport.swift @@ -126,10 +126,10 @@ enum OKFValidator { return [issueError(relativePath, path: "frontmatter", message: "invalid YAML: \(error.localizedDescription)")] } - guard let typeNode = document.frontMatter[Yams.Node("type")] else { + guard let typeNode = document.frontMatter["type"] else { return [issueError(relativePath, path: "frontmatter.type", message: "missing required field \"type\"")] } - guard let typeValue = typeNode.string else { + guard let typeValue = typeNode.stringValue else { return [issueError(relativePath, path: "frontmatter.type", message: "must be a string")] } guard !typeValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { @@ -201,11 +201,11 @@ enum OKFAnalyzer { guard frontmatterPresence(in: content).hasFrontmatter else { continue } guard let document = try? MarkdownDocument(content: content) else { continue } - if let type = document.frontMatter[Yams.Node("type")]?.string?.trimmingCharacters(in: .whitespacesAndNewlines), !type.isEmpty { + if let type = document.frontMatter["type"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines), !type.isEmpty { typeCounts[type, default: 0] += 1 } - let missing = recommendedFields.filter { document.frontMatter[Yams.Node($0)] == nil } + let missing = recommendedFields.filter { document.frontMatter[$0] == nil } if !missing.isEmpty { missingRecommendedFields[relative] = missing advisoryIssues.append(OKFValidationIssue( @@ -216,7 +216,7 @@ enum OKFAnalyzer { )) } - if let timestamp = document.frontMatter[Yams.Node("timestamp")]?.string, + if let timestamp = document.frontMatter["timestamp"]?.stringValue, !isISO8601Timestamp(timestamp) { advisoryIssues.append(OKFValidationIssue( severity: .warning, @@ -639,9 +639,9 @@ enum OKFTypeSetter { private static func matchesFilter(document: MarkdownDocument, key: String?, contains value: String?) -> Bool { guard let key, let value else { return true } - guard let node = document.frontMatter[Yams.Node(key)] else { return false } - guard case .sequence(let sequence) = node else { return false } - return sequence.contains { $0.string == value } + guard let node = document.frontMatter[key] else { return false } + guard case .array(let sequence) = node else { return false } + return sequence.contains { $0.stringValue == value } } } diff --git a/Sources/md-utils/OutputFormat.swift b/Sources/md-utils/OutputFormat.swift index 2e802d8..09abd4f 100644 --- a/Sources/md-utils/OutputFormat.swift +++ b/Sources/md-utils/OutputFormat.swift @@ -10,10 +10,13 @@ import Foundation import MarkdownUtilitiesCore import Yams +extension FrontMatterFormat: ExpressibleByArgument {} + /// Output format options for CLI commands enum OutputFormat: String, CaseIterable, ExpressibleByArgument { case json case yaml + case toml case raw case plist @@ -21,6 +24,7 @@ enum OutputFormat: String, CaseIterable, ExpressibleByArgument { switch self { case .json: return "json (default)" case .yaml: return "yaml" + case .toml: return "toml" case .raw: return "raw" case .plist: return "plist" } @@ -39,6 +43,9 @@ func print(node: Yams.Node, format: OutputFormat) throws { case .plist: let plistString = try YAMLConversion.nodeToPlist(node) Swift.print(plistString) + case .toml: + let value = try YAMLConversion.safeNodeToSwiftValue(node) + Swift.print(try FrontMatterConversion.serializeTOMLValue(value), terminator: "") } } @@ -54,5 +61,33 @@ func printAny(_ any: Any, format: OutputFormat) throws { case .plist: let plistString = try YAMLConversion.anyToPlist(any) Swift.print(plistString) + case .toml: + Swift.print(try FrontMatterConversion.serializeTOMLValue(any), terminator: "") + } +} + +/// Prints format-neutral frontmatter in the requested representation. +func print( + frontMatter: FrontMatter, + format: OutputFormat, + sourceFormat: FrontMatterFormat? = nil +) throws { + switch format { + case .json: + Swift.print(try YAMLConversion.anyToJSON( + FrontMatterConversion.foundationValue(frontMatter), + options: [.prettyPrinted, .sortedKeys] + )) + case .yaml: + Swift.print(try FrontMatterConversion.serialize(frontMatter, format: .yaml), terminator: "") + case .toml: + Swift.print(try FrontMatterConversion.serialize(frontMatter, format: .toml), terminator: "") + case .raw: + Swift.print( + try FrontMatterConversion.serialize(frontMatter, format: sourceFormat ?? .yaml), + terminator: "" + ) + case .plist: + Swift.print(try YAMLConversion.anyToPlist(FrontMatterConversion.foundationValue(frontMatter))) } } diff --git a/Sources/md-utils/Resources/SKILL.md b/Sources/md-utils/Resources/SKILL.md index e7f2ccc..34d3034 100644 --- a/Sources/md-utils/Resources/SKILL.md +++ b/Sources/md-utils/Resources/SKILL.md @@ -1,7 +1,7 @@ --- name: markdown-utilities description: >- - Parse, manipulate, and analyze Markdown files using the `md-utils` CLI. Supports YAML frontmatter CRUD (get/set/search/array ops), structured document exploration, heading manipulation, section extraction and reordering, table of contents generation, wikilink analysis, line extraction, and format conversion. Handles batch operations across files and directories. Use when working with Markdown files to: read or write frontmatter, inspect lengthy document structure, restructure documents, search files by metadata using JMESPath, generate a TOC, extract sections or line ranges, check wikilinks, or convert to plain text or CSV. More reliable than grep/regex for structured Markdown operations. + Parse, manipulate, and analyze Markdown files using the `md-utils` CLI. Supports YAML and TOML frontmatter CRUD and array operations, YAML-only JMESPath search, structured document exploration, heading manipulation, section extraction and reordering, table of contents generation, wikilink analysis, line extraction, and format conversion. Handles batch operations across files and directories. Use when working with Markdown files to: read or write frontmatter, inspect lengthy document structure, restructure documents, search YAML files by metadata using JMESPath, generate a TOC, extract sections or line ranges, check wikilinks, or convert to plain text or CSV. More reliable than grep/regex for structured Markdown operations. --- # Markdown Utilities @@ -12,7 +12,7 @@ The `md-utils` CLI provides structured operations on Markdown files. Add `--help | Command | Purpose | |---------|---------| -| `md-utils fm` | YAML frontmatter: get, set, search, remove blocks, uniqueness checks, array ops, dump | +| `md-utils fm` | YAML/TOML frontmatter: get, set, remove blocks, uniqueness checks, array ops, dump; YAML-only search | | `md-utils explore` | Progressively inspect large Markdown files by tree, heading, and line | | `md-utils toc` | Generate table of contents | | `md-utils headings` | Promote or demote heading levels | @@ -40,6 +40,9 @@ md-utils toc docs/*.md # Get a frontmatter value md-utils fm get --key title post.md +# Create TOML frontmatter in a document that has none +md-utils fm set --key title --value "TOML Note" --frontmatter-format toml post.md + # Find files with a specific tag md-utils fm array contains --key tags --value swift posts/ diff --git a/Sources/md-utils/RulesCommands/RulesSupport.swift b/Sources/md-utils/RulesCommands/RulesSupport.swift index fa23e0c..62c1b3f 100644 --- a/Sources/md-utils/RulesCommands/RulesSupport.swift +++ b/Sources/md-utils/RulesCommands/RulesSupport.swift @@ -1432,12 +1432,19 @@ enum RulesValidatorRunner { let assessment = try checker.assess(analyzed, against: compiled) guard assessment.status != .notApplicable else { continue } let ruleName = compiled.definition.name - let errors = (assessment.applicabilityDiagnostics + assessment.diagnostics).map { + let errors = (assessment.applicabilityDiagnostics + assessment.diagnostics).map { diagnostic in RuleValidationErrorDetail( - path: $0.location, - message: $0.code == "record.frontmatter.invalid-yaml" - ? $0.message.replacingOccurrences(of: "Invalid YAML:", with: "invalid YAML:") - : $0.message + path: diagnostic.location, + message: { + switch diagnostic.code { + case "record.frontmatter.invalid-yaml": + return diagnostic.message.replacingOccurrences(of: "Invalid YAML:", with: "invalid YAML:") + case "record.frontmatter.invalid-toml": + return diagnostic.message.replacingOccurrences(of: "Invalid TOML:", with: "invalid TOML:") + default: + return diagnostic.message + } + }() ) } let status: RuleValidationResult.Status diff --git a/Sources/md-utils/SectionCommands/InsertSection.swift b/Sources/md-utils/SectionCommands/InsertSection.swift index 85d980c..5809b85 100644 --- a/Sources/md-utils/SectionCommands/InsertSection.swift +++ b/Sources/md-utils/SectionCommands/InsertSection.swift @@ -185,16 +185,7 @@ extension CLIEntry { } private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/SectionCommands/MoveSectionDown.swift b/Sources/md-utils/SectionCommands/MoveSectionDown.swift index fc0c5c2..11fe654 100644 --- a/Sources/md-utils/SectionCommands/MoveSectionDown.swift +++ b/Sources/md-utils/SectionCommands/MoveSectionDown.swift @@ -147,17 +147,7 @@ extension CLIEntry { /// /// See for workflow details. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/SectionCommands/MoveSectionTo.swift b/Sources/md-utils/SectionCommands/MoveSectionTo.swift index dfaa41e..af8f12f 100644 --- a/Sources/md-utils/SectionCommands/MoveSectionTo.swift +++ b/Sources/md-utils/SectionCommands/MoveSectionTo.swift @@ -152,17 +152,7 @@ extension CLIEntry { /// /// See for workflow details. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/SectionCommands/MoveSectionUp.swift b/Sources/md-utils/SectionCommands/MoveSectionUp.swift index 42cad5f..6531a88 100644 --- a/Sources/md-utils/SectionCommands/MoveSectionUp.swift +++ b/Sources/md-utils/SectionCommands/MoveSectionUp.swift @@ -147,17 +147,7 @@ extension CLIEntry { /// /// See for workflow details. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/SectionCommands/RemoveSection.swift b/Sources/md-utils/SectionCommands/RemoveSection.swift index 10aff05..4587e01 100644 --- a/Sources/md-utils/SectionCommands/RemoveSection.swift +++ b/Sources/md-utils/SectionCommands/RemoveSection.swift @@ -99,16 +99,7 @@ extension CLIEntry { } private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/SectionCommands/SetSection.swift b/Sources/md-utils/SectionCommands/SetSection.swift index d8551bb..fde4909 100644 --- a/Sources/md-utils/SectionCommands/SetSection.swift +++ b/Sources/md-utils/SectionCommands/SetSection.swift @@ -181,17 +181,7 @@ extension CLIEntry { /// /// See for workflow details. private func reconstructDocument(_ doc: MarkdownDocument) throws -> String { - guard !doc.frontMatter.isEmpty else { - return doc.body - } - - let yamlContent = try YAMLConversion.serialize(doc.frontMatter) - - return """ - --- - \(yamlContent)--- - \(doc.body) - """ + try doc.render() } } } diff --git a/Sources/md-utils/TypesCommands/TypesSubcommands.swift b/Sources/md-utils/TypesCommands/TypesSubcommands.swift index a1c383b..33e3abd 100644 --- a/Sources/md-utils/TypesCommands/TypesSubcommands.swift +++ b/Sources/md-utils/TypesCommands/TypesSubcommands.swift @@ -18,7 +18,7 @@ extension CLIEntry.TypesCommands { @Option(name: .long, help: "Type contract version; Semantic Versioning is recommended") var version = "0.1.0" - @Option(name: .long, help: "Definition format: yaml or json") + @Option(name: .long, help: "Definition format: yaml, json, or toml") var format: TypesDefinitionFormat = .yaml @Option(name: .long, help: "Override the generated definition file", completion: .file(), transform: { Path($0) }) @@ -46,7 +46,7 @@ extension CLIEntry.TypesCommands { abstract: "List project Markdown type definitions" ) - @Option(name: .long, help: "Output format: text, markdown, json, or yaml") + @Option(name: .long, help: "Output format: text, markdown, json, yaml, or toml") var format: TypesOutputFormat = .text @Option(name: .long, help: "Project root directory", completion: .directory, transform: { Path($0) }) @@ -72,7 +72,7 @@ extension CLIEntry.TypesCommands { @Argument(help: "Type name to describe") var name: String - @Option(name: .long, help: "Output format: text, markdown, json, or yaml") + @Option(name: .long, help: "Output format: text, markdown, json, yaml, or toml") var format: TypesOutputFormat = .text @Option(name: .long, help: "Project root directory", completion: .directory, transform: { Path($0) }) @@ -371,7 +371,7 @@ extension CLIEntry.TypesCommands { abstract: "Print the md-utils Markdown type-definition JSON Schema" ) - @Option(name: .long, help: "Output format: json or yaml") + @Option(name: .long, help: "Output format: json, yaml, or toml") var format: TypesDefinitionFormat = .json mutating func run() async throws { @@ -382,7 +382,15 @@ extension CLIEntry.TypesCommands { guard let data = content.data(using: .utf8) else { throw ValidationError("Bundled type schema is not UTF-8") } - print(try Yams.dump(object: JSONSerialization.jsonObject(with: data), sortKeys: true)) + let object = try JSONSerialization.jsonObject(with: data) + switch format { + case .yaml: + print(try Yams.dump(object: object, sortKeys: true)) + case .toml: + print(try FrontMatterConversion.serializeTOMLValue(object), terminator: "") + case .json: + break + } } } } diff --git a/Sources/md-utils/TypesCommands/TypesSupport.swift b/Sources/md-utils/TypesCommands/TypesSupport.swift index 2e60559..3e358c4 100644 --- a/Sources/md-utils/TypesCommands/TypesSupport.swift +++ b/Sources/md-utils/TypesCommands/TypesSupport.swift @@ -10,11 +10,13 @@ enum TypesOutputFormat: String, ExpressibleByArgument { case markdown case json case yaml + case toml } enum TypesDefinitionFormat: String, ExpressibleByArgument { case yaml case json + case toml } enum TypesProject { @@ -58,6 +60,8 @@ enum TypesProject { [".mdtype.yaml", ".mdtype.yml"] case .json: [".mdtype.json"] + case .toml: + [".mdtype.toml"] } guard allowedSuffixes.contains(where: destinationName.hasSuffix) else { let expectedSuffixes = allowedSuffixes.joined(separator: " or ") @@ -121,6 +125,16 @@ enum TypesProject { throw ValidationError("Failed to encode type definition") } return string + "\n" + case .toml: + let object: [String: Any] = [ + "md-utils-type-schema": "1", + "name": name, + "version": version, + "frontmatter": ["schemas": []], + "body": ["requirements": [], "recommendations": []], + "context": ["requirements": [], "recommendations": []], + ] + return try FrontMatterConversion.serializeTOMLValue(object) } } @@ -210,7 +224,7 @@ enum TypesRenderer { case .markdown: guard definitions.isEmpty == false else { return "No Markdown types found." } return definitions.map { "- `\($0.name.rawValue)` \($0.version)" }.joined(separator: "\n") - case .json, .yaml: + case .json, .yaml, .toml: return try serialize(definitions.map(definitionObject), format: format) } } @@ -252,7 +266,7 @@ enum TypesRenderer { - Context requirements: \(definition.context.requirements.count) - Context recommendations: \(definition.context.recommendations.count) """ - case .json, .yaml: + case .json, .yaml, .toml: return try serialize(definitionObject(definition), format: format) } } @@ -264,7 +278,7 @@ enum TypesRenderer { includeOK: Bool, includeAdvisories: Bool = true ) throws -> String { - if format == .json || format == .yaml { + if format == .json || format == .yaml || format == .toml { let objects = results.map { result in assessmentObject(result.assessment, path: relativePath(from: root, to: result.file)) } @@ -292,7 +306,7 @@ enum TypesRenderer { root: Path, includeAll: Bool ) throws -> String { - if format == .json || format == .yaml { + if format == .json || format == .yaml || format == .toml { let objects: [[String: Any]] = results.map { file, assessments in [ "path": relativePath(from: root, to: file), @@ -322,7 +336,7 @@ enum TypesRenderer { "hints": hints.filter { includeConfirmed || $0.status != .confirmed }.map(hintObject), ] } - if format == .json || format == .yaml { + if format == .json || format == .yaml || format == .toml { return try serialize(objects, format: format) } return results.map { file, hints in @@ -476,8 +490,10 @@ enum TypesRenderer { return string case .yaml: return try Yams.dump(object: object, sortKeys: true) + case .toml: + return try FrontMatterConversion.serializeTOMLValue(object) case .text, .markdown: - throw ValidationError("Structured serialization requires json or yaml output") + throw ValidationError("Structured serialization requires json, yaml, or toml output") } } } diff --git a/Tests/MarkdownUtilitiesCoreTests/Formatting/FormattingTests.swift b/Tests/MarkdownUtilitiesCoreTests/Formatting/FormattingTests.swift index 4d4d750..ade754e 100644 --- a/Tests/MarkdownUtilitiesCoreTests/Formatting/FormattingTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/Formatting/FormattingTests.swift @@ -437,8 +437,7 @@ struct MarkdownDocumentFormattingTests { let result = try await doc.format(options: options) #expect(!result.frontMatter.isEmpty) // Frontmatter title preserved - let titleKey = Yams.Node(stringLiteral: "title") - #expect(result.frontMatter[titleKey] != nil) + #expect(result.frontMatter["title"] != nil) #expect(result.body.contains("- item one")) } @@ -459,7 +458,7 @@ struct MarkdownDocumentFormattingTests { let options = FormattingOptions(normalizeTables: true) let result = try await doc.format(options: options) - let serialized = try YAMLConversion.serialize(result.frontMatter) + let serialized = try FrontMatterConversion.serialize(result.frontMatter, format: .yaml) // Block-style: tags should be a block sequence, not flow style [swift, yaml] #expect(serialized.contains("- swift")) #expect(!serialized.contains("[swift")) diff --git a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterEdgeCasesTests.swift b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterEdgeCasesTests.swift index 8063ffa..38d6a7e 100644 --- a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterEdgeCasesTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterEdgeCasesTests.swift @@ -18,7 +18,7 @@ struct FrontMatterEdgeCasesTests { """ let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.body == "") #expect(doc.hasFrontMatter == true) } @@ -33,7 +33,7 @@ struct FrontMatterEdgeCasesTests { """ let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") // Body should include the whitespace #expect(doc.body.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) } @@ -50,7 +50,7 @@ struct FrontMatterEdgeCasesTests { """ let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.body == "---\n---\nContent") } @@ -81,7 +81,7 @@ struct FrontMatterEdgeCasesTests { // Parser accepts mixed line endings in frontmatter content // YAML parser should handle \r characters - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.body == "Body") } @@ -90,7 +90,7 @@ struct FrontMatterEdgeCasesTests { let content = "---\ntitle: Test\n---Body" let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.body == "Body") } @@ -164,8 +164,8 @@ struct FrontMatterEdgeCasesTests { let content = "---\n\(largeFM)---\nBody" let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") - #expect(doc.frontMatter["key0"]?.string == "value0") + #expect(doc.frontMatter["title"]?.stringValue == "Test") + #expect(doc.frontMatter["key0"]?.stringValue == "value0") #expect(doc.body == "Body") #expect(doc.hasFrontMatter == true) } @@ -183,8 +183,8 @@ struct FrontMatterEdgeCasesTests { let doc = try MarkdownDocument(content: content) #expect(doc.hasFrontMatter == true) - #expect(doc.frontMatter["title"]?.string == "Test with quotes") - #expect(doc.frontMatter["emoji"]?.string == "🚀") + #expect(doc.frontMatter["title"]?.stringValue == "Test with quotes") + #expect(doc.frontMatter["emoji"]?.stringValue == "🚀") #expect(doc.body == "Body") } @@ -200,8 +200,8 @@ struct FrontMatterEdgeCasesTests { let doc = try MarkdownDocument(content: content) #expect(doc.hasFrontMatter == true) - #expect(doc.frontMatter["title"]?.string == "日本語") - #expect(doc.frontMatter["chinese"]?.string == "中文") + #expect(doc.frontMatter["title"]?.stringValue == "日本語") + #expect(doc.frontMatter["chinese"]?.stringValue == "中文") } @Test @@ -211,7 +211,7 @@ struct FrontMatterEdgeCasesTests { // PrefixUpTo("---") matches the first 3 hyphens of "----" // This is reasonable behavior - treats first 3 hyphens as closing delimiter - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.body == "-\nBody") } } diff --git a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterMutationTests.swift b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterMutationTests.swift index fbffd66..af4cac2 100644 --- a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterMutationTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterMutationTests.swift @@ -21,7 +21,7 @@ struct FrontMatterMutationTests { let doc = try MarkdownDocument(content: content) let value = doc.getValue(forKey: "title") - let stringValue = try #require(value?.string) + let stringValue = try #require(value?.stringValue) #expect(stringValue == "Test Document") } @@ -70,7 +70,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: "Just body") doc.setValue("Test", forKey: "title") - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") } @Test @@ -84,8 +84,8 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.setValue("New Value", forKey: "author") - #expect(doc.frontMatter["author"]?.string == "New Value") - #expect(doc.frontMatter["title"]?.string == "Original") + #expect(doc.frontMatter["author"]?.stringValue == "New Value") + #expect(doc.frontMatter["title"]?.stringValue == "Original") } @Test @@ -100,8 +100,8 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.setValue("Updated", forKey: "title") - #expect(doc.frontMatter["title"]?.string == "Updated") - #expect(doc.frontMatter["author"]?.string == "Jane") + #expect(doc.frontMatter["title"]?.stringValue == "Updated") + #expect(doc.frontMatter["author"]?.stringValue == "Jane") } @Test @@ -129,7 +129,7 @@ struct FrontMatterMutationTests { try doc.createNewKeyWithNullValue("title") #expect(doc.hasKey("title") == true) - #expect(doc.frontMatter["title"] == Yams.Node("", Tag(.null))) + #expect(doc.frontMatter["title"] == .null) } @Test @@ -145,7 +145,7 @@ struct FrontMatterMutationTests { #expect(doc.hasKey("newkey") == true) #expect(doc.hasKey("existing") == true) - #expect(doc.frontMatter["existing"]?.string == "value") + #expect(doc.frontMatter["existing"]?.stringValue == "value") } @Test @@ -188,9 +188,9 @@ struct FrontMatterMutationTests { let reparsed = try MarkdownDocument(content: rendered) #expect(reparsed.hasKey("title") == true) - #expect(reparsed.frontMatter["title"] == Yams.Node("", Tag(.null))) + #expect(reparsed.frontMatter["title"] == .null) #expect(reparsed.hasKey("author") == true) - #expect(reparsed.frontMatter["author"] == Yams.Node("", Tag(.null))) + #expect(reparsed.frontMatter["author"] == .null) #expect(reparsed.body == "Original body") let rerendered = try reparsed.render() #expect(rerendered == rendered) @@ -249,7 +249,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.removeValue(forKey: "author") - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.frontMatter["author"] == nil) #expect(doc.frontMatter["count"]?.int == 42) } @@ -265,7 +265,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.removeValue(forKey: "nonexistent") - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.frontMatter.count == 1) } @@ -326,7 +326,7 @@ struct FrontMatterMutationTests { // Update one doc.setValue("published", forKey: "status") - #expect(doc.getValue(forKey: "status")?.string == "published") + #expect(doc.getValue(forKey: "status")?.stringValue == "published") // Remove one doc.removeValue(forKey: "author") @@ -344,8 +344,8 @@ struct FrontMatterMutationTests { let rendered = try doc.render() let reparsed = try MarkdownDocument(content: rendered) - #expect(reparsed.getValue(forKey: "title")?.string == "Test") - #expect(reparsed.getValue(forKey: "author")?.string == "Jane") + #expect(reparsed.getValue(forKey: "title")?.stringValue == "Test") + #expect(reparsed.getValue(forKey: "author")?.stringValue == "Jane") #expect(reparsed.body == "Original body") } @@ -366,9 +366,9 @@ struct FrontMatterMutationTests { #expect(doc.hasKey("author") == false) #expect(doc.hasKey("creator") == true) - #expect(doc.getValue(forKey: "creator")?.string == "Jane Doe") + #expect(doc.getValue(forKey: "creator")?.stringValue == "Jane Doe") // Other keys should remain unchanged - #expect(doc.getValue(forKey: "title")?.string == "Test Document") + #expect(doc.getValue(forKey: "title")?.stringValue == "Test Document") #expect(doc.getValue(forKey: "count")?.int == 42) } @@ -463,8 +463,8 @@ struct FrontMatterMutationTests { #expect(reparsed.hasKey("old_key") == false) #expect(reparsed.hasKey("new_key") == true) - #expect(reparsed.getValue(forKey: "new_key")?.string == "Some Value") - #expect(reparsed.getValue(forKey: "other")?.string == "Another Value") + #expect(reparsed.getValue(forKey: "new_key")?.stringValue == "Some Value") + #expect(reparsed.getValue(forKey: "other")?.stringValue == "Another Value") #expect(reparsed.body == "Body content") } @@ -483,13 +483,13 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.sortKeys() - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["author", "title", "zebra"]) // Verify values are preserved - #expect(doc.getValue(forKey: "zebra")?.string == "last") - #expect(doc.getValue(forKey: "title")?.string == "middle") - #expect(doc.getValue(forKey: "author")?.string == "first") + #expect(doc.getValue(forKey: "zebra")?.stringValue == "last") + #expect(doc.getValue(forKey: "title")?.stringValue == "middle") + #expect(doc.getValue(forKey: "author")?.stringValue == "first") } @Test @@ -505,7 +505,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.sortKeys(by: .alphabetical, reverse: true) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["zebra", "title", "author"]) } @@ -523,14 +523,14 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.sortKeys(by: .length) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["a", "mid", "short", "very_long_key_name"]) // Verify values are preserved - #expect(doc.getValue(forKey: "very_long_key_name")?.string == "value1") - #expect(doc.getValue(forKey: "short")?.string == "value2") - #expect(doc.getValue(forKey: "mid")?.string == "value3") - #expect(doc.getValue(forKey: "a")?.string == "value4") + #expect(doc.getValue(forKey: "very_long_key_name")?.stringValue == "value1") + #expect(doc.getValue(forKey: "short")?.stringValue == "value2") + #expect(doc.getValue(forKey: "mid")?.stringValue == "value3") + #expect(doc.getValue(forKey: "a")?.stringValue == "value4") } @Test @@ -546,7 +546,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.sortKeys(by: .length, reverse: true) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["abc", "ab", "a"]) } @@ -567,7 +567,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.sortKeys() - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["author", "metadata", "zebra"]) // Verify array was preserved @@ -625,7 +625,7 @@ struct FrontMatterMutationTests { var doc = try MarkdownDocument(content: content) doc.sortKeys() - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["active", "count", "disabled", "price"]) #expect(doc.getValue(forKey: "count")?.int == 42) @@ -650,11 +650,11 @@ struct FrontMatterMutationTests { let rendered = try doc.render() let reparsed = try MarkdownDocument(content: rendered) - let keys = Array(reparsed.frontMatter.keys).compactMap { $0.string } + let keys = reparsed.frontMatter.keys #expect(keys == ["a", "m", "z"]) - #expect(reparsed.getValue(forKey: "a")?.string == "first") - #expect(reparsed.getValue(forKey: "m")?.string == "middle") - #expect(reparsed.getValue(forKey: "z")?.string == "last") + #expect(reparsed.getValue(forKey: "a")?.stringValue == "first") + #expect(reparsed.getValue(forKey: "m")?.stringValue == "middle") + #expect(reparsed.getValue(forKey: "z")?.stringValue == "last") #expect(reparsed.body == "Body content") } } diff --git a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterParsingTests.swift b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterParsingTests.swift index dc48061..5054b11 100644 --- a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterParsingTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterParsingTests.swift @@ -27,7 +27,7 @@ struct FrontMatterParsingTests { let doc = try MarkdownDocument(content: content) // Verify parsed values - #expect(doc.frontMatter["title"]?.string == "Test Document") + #expect(doc.frontMatter["title"]?.stringValue == "Test Document") #expect(doc.frontMatter["count"]?.int == 42) #expect(doc.frontMatter["tags"]?.sequence?.count == 2) } @@ -145,7 +145,7 @@ struct FrontMatterParsingTests { #expect(doc.frontMatter["nested"] != nil) #expect(doc.frontMatter["list"]?.sequence?.count == 3) - #expect(doc.frontMatter["metadata"]?.mapping?["author"]?.string == "Jane") + #expect(doc.frontMatter["metadata"]?.mapping?["author"]?.stringValue == "Jane") } @Test diff --git a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterSeparationTests.swift b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterSeparationTests.swift index 9ff625a..804944b 100644 --- a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterSeparationTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/FrontMatterSeparationTests.swift @@ -22,8 +22,8 @@ struct FrontMatterSeparationTests { """ let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") - #expect(doc.frontMatter["author"]?.string == "Jane") + #expect(doc.frontMatter["title"]?.stringValue == "Test") + #expect(doc.frontMatter["author"]?.stringValue == "Jane") #expect(doc.body == "Body content here") } @@ -61,7 +61,7 @@ struct FrontMatterSeparationTests { """ let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") + #expect(doc.frontMatter["title"]?.stringValue == "Test") #expect(doc.body == "Some text\n---\nMore text") } @@ -111,8 +111,8 @@ struct FrontMatterSeparationTests { """ let doc = try MarkdownDocument(content: content) - #expect(doc.frontMatter["title"]?.string == "Test") - #expect(doc.frontMatter["author"]?.string == "John") + #expect(doc.frontMatter["title"]?.stringValue == "Test") + #expect(doc.frontMatter["author"]?.stringValue == "John") #expect(doc.body == "") } diff --git a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/TOMLFrontMatterTests.swift b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/TOMLFrontMatterTests.swift new file mode 100644 index 0000000..3c4b7af --- /dev/null +++ b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/TOMLFrontMatterTests.swift @@ -0,0 +1,139 @@ +import MarkdownUtilitiesCore +import Testing + +@Suite("TOML frontmatter") +struct TOMLFrontMatterTests { + @Test + func `parses and renders TOML values with TOML delimiters`() throws { + let content = """ + +++ + title = "TOML document" + draft = false + count = 3 + tags = ["swift", "toml"] + published = 2026-08-16 + + [author] + name = "Daniel" + +++ + # Body + """ + + let document = try MarkdownDocument(content: content) + + #expect(document.frontMatterFormat == .toml) + #expect(document.frontMatter["title"]?.stringValue == "TOML document") + #expect(document.frontMatter["draft"]?.bool == false) + #expect(document.frontMatter["count"]?.int == 3) + #expect(document.frontMatter["tags"]?.sequence?.compactMap(\.stringValue) == ["swift", "toml"]) + #expect(document.frontMatter["author"]?.mapping?["name"]?.stringValue == "Daniel") + guard case .localDate = document.frontMatter["published"] else { + Issue.record("Expected a TOML local date") + return + } + + let rendered = try document.render() + #expect(rendered.hasPrefix("+++\n")) + #expect(rendered.contains("published = 2026-08-16")) + #expect(rendered.contains("\n+++\n# Body")) + + let reparsed = try MarkdownDocument(content: rendered) + #expect(reparsed.frontMatterFormat == .toml) + #expect(reparsed.frontMatter["author"]?.mapping?["name"]?.stringValue == "Daniel") + } + + @Test + func `supports TOML date time types and arrays of tables`() throws { + let document = try MarkdownDocument(content: """ + +++ + offset = 1979-05-27T07:32:00Z + local_datetime = 1979-05-27T07:32:00 + local_date = 1979-05-27 + local_time = 07:32:00 + + [[products]] + name = "Hammer" + + [[products]] + name = "Nail" + +++ + Body + """) + + guard case .offsetDateTime = document.frontMatter["offset"] else { + Issue.record("Expected an offset date-time") + return + } + guard case .localDateTime = document.frontMatter["local_datetime"] else { + Issue.record("Expected a local date-time") + return + } + guard case .localDate = document.frontMatter["local_date"] else { + Issue.record("Expected a local date") + return + } + guard case .localTime = document.frontMatter["local_time"] else { + Issue.record("Expected a local time") + return + } + let products = try #require(document.frontMatter["products"]?.sequence) + #expect(products.count == 2) + #expect(products[0].mapping?["name"]?.stringValue == "Hammer") + #expect(products[1].mapping?["name"]?.stringValue == "Nail") + + let reparsed = try MarkdownDocument(content: document.render()) + #expect(reparsed.frontMatter["products"]?.sequence?.count == 2) + } + + @Test + func `recognizes empty TOML and does not accept mismatched delimiters`() throws { + let empty = try MarkdownDocument(content: "+++\n+++\nBody") + #expect(empty.frontMatterFormat == .toml) + #expect(empty.frontMatter.isEmpty) + #expect(empty.body == "Body") + + let mismatchedSource = "+++\ntitle = \"Example\"\n---\nBody" + let mismatched = try MarkdownDocument(content: mismatchedSource) + #expect(mismatched.frontMatterFormat == nil) + #expect(mismatched.body == mismatchedSource) + } + + @Test + func `TOML mutations preserve TOML and nested values`() throws { + var document = try MarkdownDocument(content: """ + +++ + title = "Before" + tags = ["one"] + +++ + Body + """) + + document.setValue("After", forKey: "title") + document.frontMatter["tags"] = .array([.string("one"), .string("two")]) + + let rendered = try document.render() + #expect(rendered.hasPrefix("+++\n")) + #expect(!rendered.contains("---")) + #expect(try MarkdownDocument(content: rendered).frontMatter["title"]?.stringValue == "After") + } + + @Test + func `TOML rejects null values with their key path`() throws { + let document = MarkdownDocument( + frontMatter: FrontMatter(["draft": .null]), + body: "Body", + frontMatterFormat: .toml + ) + + #expect(throws: FrontMatterConversionError.unsupportedTOMLValue(path: "draft", value: "null")) { + try document.render() + } + } + + @Test + func `malformed TOML reports a TOML conversion error`() { + #expect(throws: FrontMatterConversionError.self) { + try MarkdownDocument(content: "+++\nvalue = [\n+++\nBody") + } + } +} diff --git a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/WrappedFrontMatterParserTests.swift b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/WrappedFrontMatterParserTests.swift index 7ddb53d..4bc0ffc 100644 --- a/Tests/MarkdownUtilitiesCoreTests/FrontMatter/WrappedFrontMatterParserTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/FrontMatter/WrappedFrontMatterParserTests.swift @@ -49,6 +49,15 @@ struct WrappedFrontMatterParserTests { #expect(scan.additionalOpeningLines == [8]) } + @Test + func `discovers wrapped TOML frontmatter`() throws { + let source = "/*\n+++\ntitle = \"Example\"\n+++\n*/\nstruct Example {}\n" + let block = try #require(WrappedFrontMatterParser(syntax: .cBlock).parse(source).firstBlock) + + #expect(block.format == .toml) + #expect(block.rawFrontMatter == "title = \"Example\"\n") + } + @Test func `treats incomplete candidates as absent`() { let missingYAMLCloser = "/*\n---\ntitle: Example\n*/\n" diff --git a/Tests/MarkdownUtilitiesCoreTests/MarkdownASTTests.swift b/Tests/MarkdownUtilitiesCoreTests/MarkdownASTTests.swift index 5fd7b53..b6d5012 100644 --- a/Tests/MarkdownUtilitiesCoreTests/MarkdownASTTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/MarkdownASTTests.swift @@ -279,8 +279,8 @@ struct MarkdownASTIntegrationTests { // Verify frontmatter parsed correctly #expect(!doc.frontMatter.isEmpty) - let title = try #require(doc.frontMatter["title"]?.string) - let author = try #require(doc.frontMatter["author"]?.string) + let title = try #require(doc.frontMatter["title"]?.stringValue) + let author = try #require(doc.frontMatter["author"]?.stringValue) #expect(title == "My Document") #expect(author == "Test User") diff --git a/Tests/MarkdownUtilitiesCoreTests/Types/MarkdownTypeDefinitionTests.swift b/Tests/MarkdownUtilitiesCoreTests/Types/MarkdownTypeDefinitionTests.swift index 763d28d..275c594 100644 --- a/Tests/MarkdownUtilitiesCoreTests/Types/MarkdownTypeDefinitionTests.swift +++ b/Tests/MarkdownUtilitiesCoreTests/Types/MarkdownTypeDefinitionTests.swift @@ -46,6 +46,33 @@ struct MarkdownTypeDefinitionTests { #expect(yamlDefinition.version == "draft-3") } + @Test + func `Decode an equivalent TOML definition`() throws { + let toml = """ + "md-utils-type-schema" = "1" + name = "Book" + version = "draft-3" + + [frontmatter] + schemas = [] + + [body] + requirements = [] + recommendations = [] + + [context] + requirements = [] + recommendations = [] + """ + + let definition = try MarkdownTypeDefinitionDecoder.decode(toml, format: .toml) + + #expect(definition.name.rawValue == "Book") + #expect(definition.version == "draft-3") + #expect(definition.frontmatter.schemas.isEmpty) + #expect(definition.body.requirements.isEmpty) + } + @Test func `Infer required frontmatter when schemas are present`() async throws { let definition = MarkdownFrontmatterDefinition(schemas: [ diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayAppendTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayAppendTests.swift index 8f8b54e..abc7b74 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayAppendTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayAppendTests.swift @@ -13,6 +13,30 @@ import Yams @Suite("fm array append command") struct ArrayAppendTests { + @Test + func `fm array append preserves TOML frontmatter`() async throws { + let tempFile = try createTempFile(content: """ + +++ + tags = ["swift"] + +++ + Body + """, name: "toml.md") + defer { try? tempFile.delete() } + + let command_ = try CLIEntry.FrontMatterCommands.ArrayCommands.Append.parseAsRoot([ + "--key", "tags", + "--value", "cli", + tempFile.string, + ]) + var command = try #require(command_ as? CLIEntry.FrontMatterCommands.ArrayCommands.Append) + try await command.run() + + let content: String = try tempFile.read() + let document = try MarkdownDocument(content: content) + #expect(content.hasPrefix("+++\n")) + #expect(try extractArrayValues(from: document, key: "tags") == ["swift", "cli"]) + } + @Test func `fm array append adds value to end of array`() async throws { let testContent = """ @@ -202,12 +226,9 @@ struct ArrayAppendTests { private func extractArrayValues(from doc: MarkdownDocument, key: String) throws -> [String] { guard let node = doc.getValue(forKey: key), - case .sequence(let sequence) = node else { + case .array(let sequence) = node else { return [] } - return sequence.compactMap { node in - guard case .scalar(let scalar) = node else { return nil } - return scalar.string - } + return sequence.compactMap(\.stringValue) } } diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayPrependTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayPrependTests.swift index 0fb278e..8011165 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayPrependTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayPrependTests.swift @@ -173,12 +173,9 @@ struct ArrayPrependTests { private func extractArrayValues(from doc: MarkdownDocument, key: String) throws -> [String] { guard let node = doc.getValue(forKey: key), - case .sequence(let sequence) = node else { + case .array(let sequence) = node else { return [] } - return sequence.compactMap { node in - guard case .scalar(let scalar) = node else { return nil } - return scalar.string - } + return sequence.compactMap(\.stringValue) } } diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayRemoveTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayRemoveTests.swift index 8dd2435..e76aae2 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayRemoveTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/ArrayRemoveTests.swift @@ -200,12 +200,9 @@ struct ArrayRemoveTests { private func extractArrayValues(from doc: MarkdownDocument, key: String) throws -> [String] { guard let node = doc.getValue(forKey: key), - case .sequence(let sequence) = node else { + case .array(let sequence) = node else { return [] } - return sequence.compactMap { node in - guard case .scalar(let scalar) = node else { return nil } - return scalar.string - } + return sequence.compactMap(\.stringValue) } } diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift index 6d32dc8..aec0d45 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/DumpTests.swift @@ -13,6 +13,41 @@ import ArgumentParser @Suite("fm dump command") struct DumpTests { + @Test + func `single TOML file supports raw and TOML output with delimiters`() throws { + let tempFile = try createTempFile(content: "+++\ntitle = \"Example\"\n+++\nBody", name: "toml.md") + defer { try? tempFile.delete() } + + let raw = try CLIProcessTestHelper.run([ + "fm", "dump", tempFile.string, "--format", "raw", "--include-delimiters", + ]) + #expect(raw.status == 0, "Command failed: \(raw.standardError)") + #expect(raw.standardOutput.hasPrefix("+++\n")) + #expect(raw.standardOutput.contains("title = \"Example\"")) + + let toml = try CLIProcessTestHelper.run([ + "fm", "dump", tempFile.string, "--format", "toml", + ]) + #expect(toml.status == 0, "Command failed: \(toml.standardError)") + #expect(toml.standardOutput.contains("title = \"Example\"")) + } + + @Test + func `TOML collection output remains a parseable document`() throws { + let tempDir = try createProjectTempDirectory(prefix: "md-utils-toml-dump") + defer { try? tempDir.delete() } + try (tempDir + "one.md").write("+++\ntitle = \"One\"\n+++\nBody") + try (tempDir + "two.md").write("+++\ntitle = \"Two\"\n+++\nBody") + + let result = try CLIProcessTestHelper.run([ + "fm", "dump", tempDir.string, "--format", "toml", + ]) + #expect(result.status == 0, "Command failed: \(result.standardError)") + let output = try FrontMatterConversion.parse(result.standardOutput, format: .toml) + #expect(output["frontMatter"]?.sequence?.count == 2) + #expect(output["noFrontMatter"]?.sequence?.isEmpty == true) + } + @Test func `single file outputs directly without array or path`() async throws { let tempFile = try createTempFile(content: """ diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/RenameTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/RenameTests.swift index 34c5fd3..9e38386 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/RenameTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/RenameTests.swift @@ -41,10 +41,10 @@ struct RenameTests { #expect(doc.hasKey("date") == false) #expect(doc.hasKey("created") == true) - #expect(doc.getValue(forKey: "created")?.string == "2024-01-15") + #expect(doc.getValue(forKey: "created")?.stringValue == "2024-01-15") // Other keys should remain unchanged - #expect(doc.getValue(forKey: "title")?.string == "Test Document") - #expect(doc.getValue(forKey: "author")?.string == "Jane Doe") + #expect(doc.getValue(forKey: "title")?.stringValue == "Test Document") + #expect(doc.getValue(forKey: "author")?.stringValue == "Jane Doe") } @Test @@ -75,7 +75,7 @@ struct RenameTests { #expect(doc.hasKey("status") == false) #expect(doc.hasKey("publish_status") == true) - #expect(doc.getValue(forKey: "publish_status")?.string == "draft") + #expect(doc.getValue(forKey: "publish_status")?.stringValue == "draft") } @Test @@ -137,8 +137,8 @@ struct RenameTests { // Verify file was not modified let content: String = try tempFile.read() let doc = try MarkdownDocument(content: content) - #expect(doc.getValue(forKey: "title")?.string == "Test") - #expect(doc.getValue(forKey: "author")?.string == "Jane") + #expect(doc.getValue(forKey: "title")?.stringValue == "Test") + #expect(doc.getValue(forKey: "author")?.stringValue == "Jane") } @Test @@ -253,11 +253,11 @@ struct RenameTests { #expect(doc1.hasKey("old_key") == false) #expect(doc1.hasKey("new_key") == true) - #expect(doc1.getValue(forKey: "new_key")?.string == "Value 1") + #expect(doc1.getValue(forKey: "new_key")?.stringValue == "Value 1") #expect(doc2.hasKey("old_key") == false) #expect(doc2.hasKey("new_key") == true) - #expect(doc2.getValue(forKey: "new_key")?.string == "Value 2") + #expect(doc2.getValue(forKey: "new_key")?.stringValue == "Value 2") } @Test @@ -296,13 +296,13 @@ struct RenameTests { #expect(doc1.hasKey("old_name") == false) #expect(doc1.hasKey("new_name") == true) - #expect(doc1.getValue(forKey: "new_name")?.string == "Test Value") - #expect(doc1.getValue(forKey: "other")?.string == "Keep This") + #expect(doc1.getValue(forKey: "new_name")?.stringValue == "Test Value") + #expect(doc1.getValue(forKey: "other")?.stringValue == "Keep This") #expect(doc2.hasKey("old_name") == false) #expect(doc2.hasKey("new_name") == true) - #expect(doc2.getValue(forKey: "new_name")?.string == "Test Value") - #expect(doc2.getValue(forKey: "other")?.string == "Keep This") + #expect(doc2.getValue(forKey: "new_name")?.stringValue == "Test Value") + #expect(doc2.getValue(forKey: "other")?.stringValue == "Keep This") } @Test @@ -365,7 +365,7 @@ struct RenameTests { #expect(doc.hasKey("before") == false) #expect(doc.hasKey("after") == true) - #expect(doc.getValue(forKey: "after")?.string == "value") + #expect(doc.getValue(forKey: "after")?.stringValue == "value") } // MARK: - Test Helpers diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/ReplaceTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/ReplaceTests.swift index 1a6ff3d..c60cba8 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/ReplaceTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/ReplaceTests.swift @@ -12,6 +12,28 @@ import MarkdownUtilitiesCore @Suite("fm replace command") struct ReplaceTests { + @Test + func `fm replace accepts TOML and can convert the physical format`() async throws { + let tempFile = try createTempFile(content: "---\nold: value\n---\nBody", name: "toml.md") + defer { try? tempFile.delete() } + + let command_ = try CLIEntry.FrontMatterCommands.Replace.parseAsRoot([ + "--data", "title = \"TOML Title\"\ncount = 2", + "--format", "toml", + "--frontmatter-format", "toml", + "--yes", + tempFile.string, + ]) + var command = try #require(command_ as? CLIEntry.FrontMatterCommands.Replace) + try await command.run() + + let content: String = try tempFile.read() + let document = try MarkdownDocument(content: content) + #expect(content.hasPrefix("+++\n")) + #expect(document.frontMatter["title"]?.stringValue == "TOML Title") + #expect(document.frontMatter["count"]?.int == 2) + } + @Test func `fm replace with inline JSON data`() async throws { let testContent = """ @@ -46,8 +68,8 @@ struct ReplaceTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "New Title") - #expect(doc.getValue(forKey: "status")?.string == "published") + #expect(doc.getValue(forKey: "title")?.stringValue == "New Title") + #expect(doc.getValue(forKey: "status")?.stringValue == "published") // Old keys should be gone #expect(doc.getValue(forKey: "author") == nil) #expect(doc.getValue(forKey: "draft") == nil) @@ -87,8 +109,8 @@ struct ReplaceTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "YAML Title") - #expect(doc.getValue(forKey: "category")?.string == "tutorial") + #expect(doc.getValue(forKey: "title")?.stringValue == "YAML Title") + #expect(doc.getValue(forKey: "category")?.stringValue == "tutorial") #expect(doc.getValue(forKey: "old_key") == nil) } @@ -129,9 +151,9 @@ struct ReplaceTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "From File") - #expect(doc.getValue(forKey: "author")?.string == "Test Author") - #expect(doc.getValue(forKey: "version")?.string == "2") + #expect(doc.getValue(forKey: "title")?.stringValue == "From File") + #expect(doc.getValue(forKey: "author")?.stringValue == "Test Author") + #expect(doc.getValue(forKey: "version")?.int == 2) } @Test @@ -221,13 +243,13 @@ struct ReplaceTests { let doc1 = try MarkdownDocument(content: try file1.read()) let doc2 = try MarkdownDocument(content: try file2.read()) - #expect(doc1.getValue(forKey: "category")?.string == "tutorial") - #expect(doc1.getValue(forKey: "status")?.string == "published") + #expect(doc1.getValue(forKey: "category")?.stringValue == "tutorial") + #expect(doc1.getValue(forKey: "status")?.stringValue == "published") #expect(doc1.getValue(forKey: "title") == nil) #expect(doc1.getValue(forKey: "type") == nil) - #expect(doc2.getValue(forKey: "category")?.string == "tutorial") - #expect(doc2.getValue(forKey: "status")?.string == "published") + #expect(doc2.getValue(forKey: "category")?.stringValue == "tutorial") + #expect(doc2.getValue(forKey: "status")?.stringValue == "published") #expect(doc2.getValue(forKey: "title") == nil) #expect(doc2.getValue(forKey: "type") == nil) } @@ -271,8 +293,8 @@ struct ReplaceTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "Plist Title") - #expect(doc.getValue(forKey: "version")?.string == "3") + #expect(doc.getValue(forKey: "title")?.stringValue == "Plist Title") + #expect(doc.getValue(forKey: "version")?.int == 3) } @Test @@ -407,8 +429,8 @@ struct ReplaceTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "Added Title") - #expect(doc.getValue(forKey: "draft")?.string == "false") + #expect(doc.getValue(forKey: "title")?.stringValue == "Added Title") + #expect(doc.getValue(forKey: "draft")?.bool == false) } // MARK: - Test Helpers diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/SearchTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/SearchTests.swift index d15e8a1..c51c154 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/SearchTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/SearchTests.swift @@ -7,12 +7,35 @@ import ArgumentParser import Foundation import PathKit import Testing +import MarkdownUtilitiesCore @testable import md_utils @Suite("fm search command") struct SearchTests { + @Test + func `search explicitly rejects TOML frontmatter`() throws { + let document = try MarkdownDocument(content: "+++\ntitle = \"Example\"\n+++\nBody") + + #expect(throws: Error.self) { + try FrontMatterJMESPath.object(from: document) + } + } + + @Test + func `search explicitly rejects TOML output`() async throws { + let command_ = try CLIEntry.FrontMatterCommands.Search.parseAsRoot([ + "title", + "--format", "toml", + ]) + var command = try #require(command_ as? CLIEntry.FrontMatterCommands.Search) + + await #expect(throws: Error.self) { + try await command.run() + } + } + // MARK: - Basic Equality Tests @Test diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/SetTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/SetTests.swift index 001135c..550daf3 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/SetTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/SetTests.swift @@ -13,6 +13,27 @@ import ArgumentParser @Suite("fm set command") struct SetTests { + @Test + func `fm set can create TOML frontmatter`() async throws { + let tempFile = try createTempFile(content: "Body", name: "toml.md") + defer { try? tempFile.delete() } + + let command_ = try CLIEntry.FrontMatterCommands.Set.parseAsRoot([ + "--key", "title", + "--value", "TOML Title", + "--frontmatter-format", "toml", + tempFile.string, + ]) + var command = try #require(command_ as? CLIEntry.FrontMatterCommands.Set) + try await command.run() + + let content: String = try tempFile.read() + let document = try MarkdownDocument(content: content) + #expect(content.hasPrefix("+++\n")) + #expect(document.frontMatterFormat == .toml) + #expect(document.frontMatter["title"]?.stringValue == "TOML Title") + } + @Test func `fm set creates new frontmatter key`() async throws { let testContent = "Just body" @@ -33,7 +54,7 @@ struct SetTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "Test Title") + #expect(doc.getValue(forKey: "title")?.stringValue == "Test Title") } @Test @@ -62,8 +83,8 @@ struct SetTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - #expect(doc.getValue(forKey: "title")?.string == "Updated Title") - #expect(doc.getValue(forKey: "author")?.string == "Jane") + #expect(doc.getValue(forKey: "title")?.stringValue == "Updated Title") + #expect(doc.getValue(forKey: "author")?.stringValue == "Jane") } @Test @@ -132,8 +153,8 @@ struct SetTests { let doc1 = try MarkdownDocument(content: try file1.read()) let doc2 = try MarkdownDocument(content: try file2.read()) - #expect(doc1.getValue(forKey: "category")?.string == "Tutorial") - #expect(doc2.getValue(forKey: "category")?.string == "Tutorial") + #expect(doc1.getValue(forKey: "category")?.stringValue == "Tutorial") + #expect(doc2.getValue(forKey: "category")?.stringValue == "Tutorial") } // MARK: - Invalid YAML Error Handling Tests @@ -180,7 +201,7 @@ struct SetTests { // Valid file should have been processed despite the error on the invalid file let updatedDoc = try MarkdownDocument(content: try validFile.read()) - #expect(updatedDoc.getValue(forKey: "status")?.string == "processed") + #expect(updatedDoc.getValue(forKey: "status")?.stringValue == "processed") // Invalid file should be unchanged (parse failed before write) let invalidFileContent: String = try invalidFile.read() diff --git a/Tests/md-utilsTests/Commands/FrontMatterCommands/SortKeysTests.swift b/Tests/md-utilsTests/Commands/FrontMatterCommands/SortKeysTests.swift index 19558c6..a3ff266 100644 --- a/Tests/md-utilsTests/Commands/FrontMatterCommands/SortKeysTests.swift +++ b/Tests/md-utilsTests/Commands/FrontMatterCommands/SortKeysTests.swift @@ -39,15 +39,15 @@ struct SortKeysTests { let doc = try MarkdownDocument(content: updatedContent) // Get keys in the order they appear - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["author", "date", "title", "zebra"]) // Verify values are preserved - #expect(doc.getValue(forKey: "title")?.string == "Test Document") - #expect(doc.getValue(forKey: "author")?.string == "Jane Doe") - #expect(doc.getValue(forKey: "date")?.string == "2024-01-15") - #expect(doc.getValue(forKey: "zebra")?.string == "last") + #expect(doc.getValue(forKey: "title")?.stringValue == "Test Document") + #expect(doc.getValue(forKey: "author")?.stringValue == "Jane Doe") + #expect(doc.getValue(forKey: "date")?.stringValue == "2024-01-15") + #expect(doc.getValue(forKey: "zebra")?.stringValue == "last") } @Test @@ -76,7 +76,7 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["zebra", "title", "author"]) } @@ -106,7 +106,7 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["a", "ab", "abc"]) } @@ -137,15 +137,15 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["a", "short", "mid_length", "very_long_key_name"]) // Verify values are preserved - #expect(doc.getValue(forKey: "very_long_key_name")?.string == "value1") - #expect(doc.getValue(forKey: "short")?.string == "value2") - #expect(doc.getValue(forKey: "mid_length")?.string == "value3") - #expect(doc.getValue(forKey: "a")?.string == "value4") + #expect(doc.getValue(forKey: "very_long_key_name")?.stringValue == "value1") + #expect(doc.getValue(forKey: "short")?.stringValue == "value2") + #expect(doc.getValue(forKey: "mid_length")?.stringValue == "value3") + #expect(doc.getValue(forKey: "a")?.stringValue == "value4") } @Test @@ -174,7 +174,7 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["abc", "ab", "a"]) } @@ -237,7 +237,7 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["author", "metadata", "zebra"]) // Verify array was preserved @@ -290,8 +290,8 @@ struct SortKeysTests { let doc1 = try MarkdownDocument(content: try file1.read()) let doc2 = try MarkdownDocument(content: try file2.read()) - let keys1 = Array(doc1.frontMatter.keys).compactMap { $0.string } - let keys2 = Array(doc2.frontMatter.keys).compactMap { $0.string } + let keys1 = doc1.frontMatter.keys + let keys2 = doc2.frontMatter.keys #expect(keys1 == ["a", "m", "z"]) #expect(keys2 == ["author", "date", "title"]) @@ -330,8 +330,8 @@ struct SortKeysTests { let doc1 = try MarkdownDocument(content: try file1.read()) let doc2 = try MarkdownDocument(content: try file2.read()) - let keys1 = Array(doc1.frontMatter.keys).compactMap { $0.string } - let keys2 = Array(doc2.frontMatter.keys).compactMap { $0.string } + let keys1 = doc1.frontMatter.keys + let keys2 = doc2.frontMatter.keys #expect(keys1 == ["a", "m", "z"]) #expect(keys2 == ["a", "m", "z"]) @@ -363,7 +363,7 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["active", "count", "disabled", "price"]) #expect(doc.getValue(forKey: "count")?.int == 42) @@ -397,7 +397,7 @@ struct SortKeysTests { let updatedContent: String = try tempFile.read() let doc = try MarkdownDocument(content: updatedContent) - let keys = Array(doc.frontMatter.keys).compactMap { $0.string } + let keys = doc.frontMatter.keys #expect(keys == ["a", "z"]) } diff --git a/Tests/md-utilsTests/Commands/TypesCommandsTests.swift b/Tests/md-utilsTests/Commands/TypesCommandsTests.swift index 51a7793..9bf173c 100644 --- a/Tests/md-utilsTests/Commands/TypesCommandsTests.swift +++ b/Tests/md-utilsTests/Commands/TypesCommandsTests.swift @@ -97,6 +97,24 @@ struct TypesCommandsTests { #expect(definition.version == "1.0.0") } + @Test + func `types project scaffolds and loads a TOML definition`() throws { + let project = Path(NSTemporaryDirectory()) + "types-command-toml-\(UUID().uuidString)" + defer { try? project.delete() } + + let destination = try TypesProject.addDefinition( + name: "Article", + version: "1.0.0", + format: .toml, + root: project, + output: nil + ) + let registry = try TypesProject.load(root: project) + + #expect(destination.string.hasSuffix(".md-utils/types/article.mdtype.toml")) + #expect(try #require(registry.definition(named: "Article")).version == "1.0.0") + } + @Test func `types project rejects an output filename without the mdtype suffix`() throws { let project = Path(NSTemporaryDirectory()) + "types-command-output-\(UUID().uuidString)" diff --git a/Tests/md-utilsTests/Fixtures/NonMDFrontmatter/command-parity-existing/expected/replace.swift b/Tests/md-utilsTests/Fixtures/NonMDFrontmatter/command-parity-existing/expected/replace.swift index d79a9cf..2b1607c 100644 --- a/Tests/md-utilsTests/Fixtures/NonMDFrontmatter/command-parity-existing/expected/replace.swift +++ b/Tests/md-utilsTests/Fixtures/NonMDFrontmatter/command-parity-existing/expected/replace.swift @@ -1,6 +1,6 @@ /* --- -replacement: yes +replacement: true --- */ diff --git a/docs/architecture.md b/docs/architecture.md index ed3e507..202aad9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,8 +21,8 @@ The full source and dependency classification is recorded in the [portability au **Location**: `Sources/MarkdownUtilitiesCore/MarkdownDocument.swift` Central data structure representing a Markdown document: -- Separates YAML frontmatter from body content -- Frontmatter parsed into `Yams.Node.Mapping` for structured access +- Separates YAML (`---`) or TOML (`+++`) frontmatter from body content +- Frontmatter parsed into the ordered, format-neutral `FrontMatter` model - Body available as `String` for text processing - Supports parsing body into Markdown AST via `parseAST()` method - AST parsing uses MarkdownSyntax library returning `Root` structure @@ -56,8 +56,8 @@ Server resources are explicit and opt-in. Configuration supports rule selection, The read snapshot scans canonical storage in bounded pages, applies path narrowing before parsing, reuses one record analysis across rule, type, and identity assessment, and publishes immutable generic envelopes with collision-safe primary-ID and logical-path lookup. The standalone server work in issue #77 will load its own configuration outside `.md-utils.json`, add Hummingbird 2, and register adapters from the plan and snapshot. The OpenAPI generator in issue #84 consumes the same accepted plan. **Frontmatter Handling:** -- Uses `FrontMatterParser` to separate `---` delimited YAML frontmatter -- Frontmatter parsed into `Yams.Node.Mapping` during initialization +- Uses `FrontMatterParser` to detect YAML and TOML delimiter blocks +- Frontmatter parsed into `FrontMatter` during initialization while retaining its source format - Gracefully handles documents with no frontmatter (empty mapping) **Markdown AST Parsing:** @@ -94,7 +94,7 @@ if let heading = ast.children.first as? Heading { - `convert to-text` - Convert Markdown to plain text - `convert to-csv` - Convert Markdown to CSV - `extract` (ExtractSection) - Extract a section from Markdown files by name or index -- `frontmatter` / `fm` (FrontMatterCommands) - Manipulate YAML frontmatter +- `frontmatter` / `fm` (FrontMatterCommands) - Manipulate YAML or TOML frontmatter - `fm get` - Retrieve frontmatter value by key - `fm set` - Set/update frontmatter value by key - `fm has` - Check if frontmatter key exists @@ -103,8 +103,8 @@ if let heading = ast.children.first as? Heading { - `fm rename` - Rename frontmatter key - `fm replace` / `fm r` - Replace entire frontmatter with new data - `fm list` - List all frontmatter keys - - `fm dump` - Dump entire frontmatter in specified format (JSON, YAML, raw, plist) - - `fm search` - Search for files matching a JMESPath query + - `fm dump` - Dump entire frontmatter in specified format (JSON, YAML, TOML, raw, plist) + - `fm search` - Search YAML frontmatter with a JMESPath query (TOML is out of scope) - `fm sort-keys` / `fm sk` - Sort frontmatter keys - `fm touch` - Add frontmatter keys without values - `fm array` - Array manipulation commands: @@ -152,7 +152,7 @@ By default, CLI commands performed on a directory: ### 1. Front Matter Parsing ✅ -YAML frontmatter separated and parsed into structured data. +YAML and TOML frontmatter are separated and parsed into structured data. ### 2. Markdown AST Parsing ✅ @@ -171,16 +171,16 @@ Body text parsed into Abstract Syntax Tree for programmatic manipulation. Full CRUD operations plus advanced features: - **Library**: `MarkdownDocument+FrontMatterMutation` extension with `getValue`, `setValue`, `hasKey`, `removeValue` -- **Format conversion**: `YAMLConversion` utilities for JSON, YAML, and PropertyList output +- **Format conversion**: `FrontMatterConversion` for YAML/TOML and shared JSON, YAML, TOML, and PropertyList output - **CLI**: `md-utils frontmatter` (alias `fm`) with subcommands: - Basic CRUD: `get`, `set`, `has`, `remove`, `rename`, `list`, `dump` - Advanced: `replace`, `search` (JMESPath queries), `sort-keys`, `touch` - Array operations: `array append`, `array contains`, `array prepend`, `array remove` - **Dump Feature**: Output entire frontmatter in multiple formats - - Formats: JSON (default), YAML, raw, PropertyList (XML) + - Formats: JSON (default), YAML, TOML, raw, PropertyList (XML) - Single file: direct output without wrapper - Multiple files: cat-style headers (==> path <==) with separation - - Optional YAML delimiters (---) via `--include-delimiters` + - Optional format-appropriate delimiters via `--include-delimiters` - Alias: `fm d` for quick access - Works on single files or batch operations across directories - Preserves body content and existing frontmatter structure @@ -288,6 +288,7 @@ The following features are **NOT YET IMPLEMENTED**: - **swift-argument-parser** (1.6.1+) - CLI argument parsing - **PathKit** (1.0.1+) - File path handling - **Yams** (6.1.0+) - YAML parsing and serialization +- **swift-toml** (2.0.0+) - TOML parsing and serialization - **jmespath.swift** (1.0.3+) - JMESPath query language for JSON (used by `fm search`) - **Rainbow** (4.2.1+) - ANSI styling for human-facing CLI output diff --git a/docs/common-use-cases.md b/docs/common-use-cases.md index f777b49..bfefa04 100644 --- a/docs/common-use-cases.md +++ b/docs/common-use-cases.md @@ -110,8 +110,17 @@ md-utils fm get --key tags --format numbered-list document.md #### Set frontmatter value ```bash md-utils fm set --key author --value "Jane Doe" document.md + +# Create a TOML block when the document has no frontmatter +md-utils fm set --key author --value "Jane Doe" --frontmatter-format toml document.md ``` +YAML frontmatter is delimited by `---`; TOML frontmatter is delimited by `+++`. +Mutations preserve the existing format. Creation-capable commands default to +YAML unless `--frontmatter-format toml` is supplied. Avoid comments inside either +format: parsing and reserialization do not guarantee that comments are preserved. +TOML cannot represent null, so `fm touch` is unavailable for TOML blocks. + ### Frontmatter in Non-Markdown Text Files One explicitly named supported file infers its shipped wrapper mapping. Wrapped @@ -174,9 +183,11 @@ md-utils fm list document.md #### Search files by frontmatter value ```bash -md-utils fm search --key status --value published posts/ +md-utils fm search 'status == `"published"`' posts/ ``` +`fm search` is YAML-only; TOML support is intentionally out of scope for that command. + ### Frontmatter Array Operations #### Add tag to frontmatter array @@ -201,9 +212,10 @@ md-utils fm array remove --key tags --value draft posts/*.md md-utils fm sort-keys document.md ``` -#### Dump all frontmatter as YAML +#### Dump all frontmatter as YAML or TOML ```bash -md-utils fm dump document.md +md-utils fm dump document.md --format yaml +md-utils fm dump document.md --format toml ``` ## Conversion Operations @@ -221,7 +233,7 @@ md-utils convert to-text document.md md-utils types add Book --version 1.0.0 ``` -Type definitions are stored under `.md-utils/types/` and use the compound extensions `.mdtype.yaml`, `.mdtype.yml`, or `.mdtype.json`. +Type definitions are stored under `.md-utils/types/` and use the compound extensions `.mdtype.yaml`, `.mdtype.yml`, `.mdtype.json`, or `.mdtype.toml`. ### Check and find conforming records diff --git a/docs/portability-audit.md b/docs/portability-audit.md index 64550cb..dda11d7 100644 --- a/docs/portability-audit.md +++ b/docs/portability-audit.md @@ -12,7 +12,7 @@ | Classification | Source files | Reason | |---|---|---| -| Portable Core | `MarkdownDocument.swift`; every Swift file under `Explore/`, `Formatting/`, `FrontMatter/`, `HeadingAdjustment/`, `Helpers/`, `SectionExtraction/`, `SectionReordering/`, and `TOC/` | Operates on strings, YAML nodes, or Markdown AST models without discovering host state. | +| Portable Core | `MarkdownDocument.swift`; every Swift file under `Explore/`, `Formatting/`, `FrontMatter/`, `HeadingAdjustment/`, `Helpers/`, `SectionExtraction/`, `SectionReordering/`, and `TOC/` | Operates on strings, format-neutral frontmatter values, or Markdown AST models without discovering host state. | | Portable Core | `FormatConversion/Protocols/`; `FormatConversion/Shared/`; `FormatConversion/PlainText/`; `FormatConversion/MarkdownDocument+FormatConversion.swift` | Pure conversion contracts and Markdown-to-text transformations. | | Portable Core | `Wikilink/Wikilink.swift`, `WikilinkAnchor.swift`, `WikilinkParser.swift`, `WikilinkScanner.swift`, and `MarkdownDocument+Wikilink.swift` | Parses supplied content without resolving against a filesystem. | | Portable Core | Every Swift file under `Types/` and `Rules/` | Operates on canonical record strings, explicit logical context, supplied definitions, and host-provided schema resources. It does not discover or mutate host state. | @@ -32,6 +32,7 @@ Directory paths in the table are relative to `Sources/MarkdownUtilitiesCore/`, ` | MarkdownSyntax and swift-cmark | Core | Verified by the Linux container build and Core tests. | MarkdownSyntax 1.3.0 and swift-cmark 0.7.1 compile and run under WASI, including GFM task lists and tables. | | swift-parsing | Core | Verified by the Linux container build and parser tests. | Verified while building and running the Core WASI smoke target. | | Yams and libYAML | Core | Verified by the Linux container build and frontmatter tests. | libYAML and Yams compile and run under WASI. Yams 6.2.0 requires the version-checked `DBL_DECIMAL_DIG` compatibility patch described in [WebAssembly Support](webassembly.md). | +| swift-toml and toml++ | Core | TOML parsing and serialization are covered by Core tests. | Built as part of the Core WASI workflow. The version-checked compatibility patch selects toml++'s exception-free parser because the WASI C++ runtime has exceptions disabled. | | JSONSchema | Core | Draft 2020-12 validation, external graph compilation, and the Linux Core build are verified. | Draft 2020-12 assessment runs under WASI. JSONSchema.swift 0.6.0 requires the version-checked WASI `NSNumber` compatibility patch described in [WebAssembly Support](webassembly.md). | | PathKit | Native only | Supported by the native package; excluded from Core. | Out of scope because it is not a Core dependency. | | ArgumentParser, JMESPath, Rainbow | CLI only | Outside the Core boundary. JMESPath is exposed to Core only through a serialized runtime capability provider; result truthiness remains a Core semantic. | Out of scope because the CLI is not a WebAssembly target. | diff --git a/docs/rfcs/0001-mdtype.md b/docs/rfcs/0001-mdtype.md index 4370df1..9beef59 100644 --- a/docs/rfcs/0001-mdtype.md +++ b/docs/rfcs/0001-mdtype.md @@ -85,7 +85,7 @@ A **document** is a successfully parsed content view derived from a record. It h ### Type definition -A **type definition** is a named, versioned structural contract serialized as YAML or JSON. +A **type definition** is a named, versioned structural contract serialized as YAML, JSON, or TOML. ### Requirement @@ -130,10 +130,11 @@ A definition filename MUST end with one of these compound extensions: - `.mdtype.yaml` - `.mdtype.yml` - `.mdtype.json` +- `.mdtype.toml` Suffix matching is case-insensitive. Other files under `.md-utils/types/` are ignored. The declared `name`, not the filename, is the type's identity. -The portable Core decoder does not perform discovery and can decode explicitly supplied YAML or JSON from any host. +The portable Core decoder does not perform discovery and can decode explicitly supplied YAML, JSON, or TOML from any host. ### Definition envelope diff --git a/docs/webassembly.md b/docs/webassembly.md index 835e703..d0139d3 100644 --- a/docs/webassembly.md +++ b/docs/webassembly.md @@ -59,11 +59,16 @@ The patches live in `scripts/wasm-patches/`. The build script verifies the exact CoreFoundation also requires the WASI signal and memory-mapping emulation definitions while compiling. The smoke target links the corresponding `wasi-emulated-signal` and `wasi-emulated-mman` libraries only on WASI. +swift-toml's toml++ parser is a C++ target, while the WASI SDK's C++ runtime has +exceptions disabled. The version-checked compatibility patch selects toml++'s +exception-free parse-result path and the build passes `-fno-exceptions` for WASI +C++ sources. Native behavior is unchanged. + ## Smoke Coverage `IntegrationTests/WasmCoreSmoke/` verifies representative behavior across the dependency boundary: -- YAML frontmatter parsing and floating-point serialization through Yams and libYAML; +- YAML frontmatter through Yams/libYAML and TOML frontmatter through swift-toml; - Markdown AST parsing of headings, task lists, and tables through MarkdownSyntax and swift-cmark; - Markdown rendering; and - draft 2020-12 JSON Schema and Markdown type assessment. diff --git a/scripts/build-wasm.sh b/scripts/build-wasm.sh index bb0d227..ccf5fc5 100755 --- a/scripts/build-wasm.sh +++ b/scripts/build-wasm.sh @@ -11,6 +11,7 @@ readonly SWIFT_WASM_SDK="${SWIFT_WASM_SDK:-swift-6.3.1-RELEASE_wasm}" # A package update must be evaluated before these guards are changed. readonly YAMS_REVISION="51b5127c7fb6ffac106ad6d199aaa33c5024895f" readonly JSONSCHEMA_REVISION="d14de4b2d9205068c9db89c00d097ca43c897000" +readonly SWIFT_TOML_REVISION="827506c90475e82d5a7f191f950fb3025cbdc0d6" # CI uses the default SDK store, while local verification can point Swift at an # isolated SDK installation through SWIFT_WASM_SDKS_PATH. @@ -44,14 +45,14 @@ apply_dependency_patch() { # A successful reverse check means this exact patch is already present. This # keeps repeated and incremental WASM builds idempotent. - if git -C "${checkout_directory}" apply --reverse --check "${patch_file}" 2>/dev/null; then + if git -C "${checkout_directory}" apply --unidiff-zero --reverse --check "${patch_file}" 2>/dev/null; then return 0 fi # Validate the entire diff before modifying the checkout. Context drift or a # partial prior edit fails here rather than leaving a half-applied patch. - git -C "${checkout_directory}" apply --check "${patch_file}" - git -C "${checkout_directory}" apply "${patch_file}" + git -C "${checkout_directory}" apply --unidiff-zero --check "${patch_file}" + git -C "${checkout_directory}" apply --unidiff-zero "${patch_file}" } cd "${REPOSITORY_ROOT}" @@ -77,6 +78,13 @@ apply_dependency_patch \ "${JSONSCHEMA_REVISION}" \ "${SCRIPT_DIRECTORY}wasm-patches/jsonschema-0.6.0-wasi.patch" \ "Sources/Validators.swift" +# The WASI SDK's C++ runtime has exceptions disabled. toml++ supports an +# exception-free parser result, so the C bridge uses that path on WASI. +apply_dependency_patch \ + ".build/checkouts/swift-toml/" \ + "${SWIFT_TOML_REVISION}" \ + "${SCRIPT_DIRECTORY}wasm-patches/swift-toml-2.0.0-wasi.patch" \ + "Sources/CTomlPlusPlus/ctoml.cpp" # swift-cmark's C sources require wasi-libc's signal and memory-mapping # compatibility shims. Package.swift links the matching emulation libraries. @@ -84,7 +92,8 @@ swift build \ "${swift_sdk_arguments[@]}" \ --target MarkdownUtilitiesCore \ -Xcc -D_WASI_EMULATED_SIGNAL \ - -Xcc -D_WASI_EMULATED_MMAN + -Xcc -D_WASI_EMULATED_MMAN \ + -Xcxx -fno-exceptions # Running the product through `swift run` executes it with the SDK's configured # WASM runtime and verifies real parsing behavior, not compilation alone. @@ -92,4 +101,5 @@ swift run \ "${swift_sdk_arguments[@]}" \ -Xcc -D_WASI_EMULATED_SIGNAL \ -Xcc -D_WASI_EMULATED_MMAN \ + -Xcxx -fno-exceptions \ MarkdownUtilitiesCoreWasmSmoke diff --git a/scripts/wasm-patches/swift-toml-2.0.0-wasi.patch b/scripts/wasm-patches/swift-toml-2.0.0-wasi.patch new file mode 100644 index 0000000..be8d14b --- /dev/null +++ b/scripts/wasm-patches/swift-toml-2.0.0-wasi.patch @@ -0,0 +1,46 @@ +diff --git a/Sources/CTomlPlusPlus/ctoml.cpp b/Sources/CTomlPlusPlus/ctoml.cpp +index e2158b7..8bb64be 100644 +--- a/Sources/CTomlPlusPlus/ctoml.cpp ++++ b/Sources/CTomlPlusPlus/ctoml.cpp +@@ -3,0 +4,3 @@ ++#if defined(__wasi__) ++#define TOML_EXCEPTIONS 0 ++#endif +@@ -43,0 +47,3 @@ struct CTomlTable ++ #if defined(__wasi__) ++ std::abort(); ++ #else +@@ -44,0 +51 @@ struct CTomlTable ++ #endif +@@ -57,0 +65,3 @@ struct CTomlTable ++ #if defined(__wasi__) ++ std::abort(); ++ #else +@@ -58,0 +69 @@ struct CTomlTable ++ #endif +@@ -200,0 +212,23 @@ extern "C" ++ #if defined(__wasi__) ++ CTomlTable* storage = new (std::nothrow) CTomlTable(); ++ if (!storage) ++ { ++ result.error_message = "Out of memory"; ++ return result; ++ } ++ result.handle = storage; ++ ++ std::string_view sv(input, length); ++ auto parsed = toml::parse(sv); ++ if (!parsed) ++ { ++ const auto& err = parsed.error(); ++ storage->error_message = std::string(err.description()); ++ result.error_message = storage->error_message.c_str(); ++ result.error_line = err.source().begin.line; ++ result.error_column = err.source().begin.column; ++ return result; ++ } ++ result.root = convert_table(parsed.table(), storage); ++ result.success = true; ++ #else +@@ -259,0 +294 @@ extern "C" ++ #endif diff --git a/skill/markdown-utilities/skills/markdown-utilities/SKILL.md b/skill/markdown-utilities/skills/markdown-utilities/SKILL.md index e7f2ccc..34d3034 100644 --- a/skill/markdown-utilities/skills/markdown-utilities/SKILL.md +++ b/skill/markdown-utilities/skills/markdown-utilities/SKILL.md @@ -1,7 +1,7 @@ --- name: markdown-utilities description: >- - Parse, manipulate, and analyze Markdown files using the `md-utils` CLI. Supports YAML frontmatter CRUD (get/set/search/array ops), structured document exploration, heading manipulation, section extraction and reordering, table of contents generation, wikilink analysis, line extraction, and format conversion. Handles batch operations across files and directories. Use when working with Markdown files to: read or write frontmatter, inspect lengthy document structure, restructure documents, search files by metadata using JMESPath, generate a TOC, extract sections or line ranges, check wikilinks, or convert to plain text or CSV. More reliable than grep/regex for structured Markdown operations. + Parse, manipulate, and analyze Markdown files using the `md-utils` CLI. Supports YAML and TOML frontmatter CRUD and array operations, YAML-only JMESPath search, structured document exploration, heading manipulation, section extraction and reordering, table of contents generation, wikilink analysis, line extraction, and format conversion. Handles batch operations across files and directories. Use when working with Markdown files to: read or write frontmatter, inspect lengthy document structure, restructure documents, search YAML files by metadata using JMESPath, generate a TOC, extract sections or line ranges, check wikilinks, or convert to plain text or CSV. More reliable than grep/regex for structured Markdown operations. --- # Markdown Utilities @@ -12,7 +12,7 @@ The `md-utils` CLI provides structured operations on Markdown files. Add `--help | Command | Purpose | |---------|---------| -| `md-utils fm` | YAML frontmatter: get, set, search, remove blocks, uniqueness checks, array ops, dump | +| `md-utils fm` | YAML/TOML frontmatter: get, set, remove blocks, uniqueness checks, array ops, dump; YAML-only search | | `md-utils explore` | Progressively inspect large Markdown files by tree, heading, and line | | `md-utils toc` | Generate table of contents | | `md-utils headings` | Promote or demote heading levels | @@ -40,6 +40,9 @@ md-utils toc docs/*.md # Get a frontmatter value md-utils fm get --key title post.md +# Create TOML frontmatter in a document that has none +md-utils fm set --key title --value "TOML Note" --frontmatter-format toml post.md + # Find files with a specific tag md-utils fm array contains --key tags --value swift posts/ diff --git a/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md b/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md index 6e97fa9..577d863 100644 --- a/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md +++ b/skill/markdown-utilities/skills/markdown-utilities/references/frontmatter.md @@ -1,5 +1,11 @@ # Frontmatter Operations Reference +YAML blocks use `---` delimiters and TOML blocks use `+++`. Mutations preserve +the existing format. Creation-capable commands accept `--frontmatter-format +yaml|toml` and default to YAML. Do not rely on YAML or TOML comments surviving a +mutation: both formats are parsed into structured values and serialized again. +TOML cannot represent null, so `fm touch` is unavailable for TOML blocks. + ## Basic CRUD ### Get a value @@ -12,6 +18,8 @@ md-utils fm get --key title document.md md-utils fm set --key author --value "Jane Doe" document.md # Batch: applies to all .md files in the directory md-utils fm set --key status --value published posts/ +# Create TOML frontmatter +md-utils fm set --key status --value published --frontmatter-format toml document.md ``` ### Check if key exists @@ -70,6 +78,9 @@ md-utils fm dump post.md # YAML format md-utils fm dump post.md --format yaml +# TOML format +md-utils fm dump post.md --format toml + # Multiple files: categorizes populated, absent, and empty frontmatter md-utils fm dump posts/ --format json @@ -83,7 +94,10 @@ md-utils fm dump posts/ --format yaml | yq '.frontMatter[].title' md-utils fm dump posts/ --cat-headers ``` -**Formats:** `json` (default), `yaml`, `raw`, `plist` +**Formats:** `json` (default), `yaml`, `toml`, `raw`, `plist` + +TOML requires a table at the document root, so scalar and array output uses a +stable `value` envelope. Raw dumps retain the source format and delimiter style. `fm dump` automatically processes Markdown, `.txt`, and mapped non-Markdown files; it does not accept `--include-non-md`. Multi-file output is an object: @@ -96,6 +110,8 @@ files; it does not accept `--include-non-md`. Multi-file output is an object: `fm search` filters files using a JMESPath expression evaluated against each file's frontmatter. Outputs matching file paths. +`fm search` supports YAML frontmatter only. TOML search is intentionally out of scope. + ```bash # Find files where status is "published" md-utils fm search "status == 'published'" posts/ From 089c0406230d9712806f1c5abf68bbf090c02233 Mon Sep 17 00:00:00 2001 From: Daniel Lyons <72824209+DandyLyons@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:06:42 -0500 Subject: [PATCH 3/3] Add YAML vs TOML docs --- .../md-utils/Documentation.docc/YAMLVsTOML.md | 55 +++++++++++++++++++ .../md-utils/Documentation.docc/md-utils.md | 1 + 2 files changed, 56 insertions(+) create mode 100644 Sources/md-utils/Documentation.docc/YAMLVsTOML.md diff --git a/Sources/md-utils/Documentation.docc/YAMLVsTOML.md b/Sources/md-utils/Documentation.docc/YAMLVsTOML.md new file mode 100644 index 0000000..d648c00 --- /dev/null +++ b/Sources/md-utils/Documentation.docc/YAMLVsTOML.md @@ -0,0 +1,55 @@ +# YAML and TOML in `md-utils` + +Understand where YAML, TOML and JSON are interchangeable and where their data models or CLI support differ. + +## Overview + +For most `md-utils` workflows, YAML and TOML are interchangeable. Both represent the JSON-like values used by frontmatter—objects, arrays, strings, numbers, and booleans—and `md-utils` converts both formats through the same format-neutral value model. + +YAML frontmatter uses `---` delimiters, while TOML uses `+++`: + +```markdown +--- +title: Example +tags: [swift, markdown] +--- +``` + +```markdown ++++ +title = "Example" +tags = ["swift", "markdown"] ++++ +``` + +Existing frontmatter keeps its format when mutated. Commands that create a block use YAML by default; pass `--frontmatter-format toml` to create TOML. An output option such as `--format toml` changes command output, not the stored frontmatter format. + +## Data-model differences + +| Edge case | YAML | TOML | `md-utils` behavior | +| --- | --- | --- | --- | +| Null | Has a null value | Has no null value | TOML serialization reports the unsupported key path. `fm touch`, which creates null values, cannot add a key to TOML frontmatter. | +| Document root | The language permits mappings, arrays, and scalars | A TOML document is a table | Frontmatter must be a top-level object in either format; non-object YAML frontmatter is rejected. For generic command output—not frontmatter—`--format toml` wraps an array or scalar root under a stable `value` key. | +| Mapping keys | Can use non-string keys (not supported by `md-utils`) | Keys are strings | The shared frontmatter model requires string keys. Unsupported YAML keys fail instead of being coerced. | +| Date and time | Usually interpreted as strings by `md-utils` | Has offset date-time, local date-time, local date, and local time values | TOML temporal values remain typed while processed as frontmatter. JSON and YAML output represent them as formatted strings. | +| Source syntax | Supports aliases, tags, block scalars, and flow styles | Supports dotted keys, inline tables, and arrays of tables | Mutations preserve values, the Markdown body, and the frontmatter format—not the original spelling, quoting, layout, or other format-specific syntax. | + +JSON is a useful common denominator, but it is not identical to either format. In particular, JSON has neither comments nor TOML's native temporal types, and TOML cannot represent YAML or JSON null values. + +## Comments + +YAML and TOML both allow comments. JSON does not, and neither does the `md-utils fm` data model. + +Once an `fm` command mutates frontmatter, however, `md-utils` parses and serializes the complete block. Comments—and syntax choices such as quoting or inline layout—are not guaranteed to survive. Avoid comments in frontmatter that will be managed with `md-utils fm`. Note: Read-only operations do not rewrite a file, and `fm dump --format raw` can return the original frontmatter text. + +## CLI-specific differences + +Most frontmatter CRUD, array, batch, rules, schema, and Markdown type workflows accept either format. The intentional exceptions are: + +- `fm search` accepts YAML frontmatter only and does not support `--format toml`. `fm unique` supports both formats when a JMESPath-based uniqueness check is sufficient. +- Open Knowledge Format commands use YAML because the OKF v0.1 specification defines YAML frontmatter. +- Project configuration files do not currently support TOML. That is a planned feature. See https://github.com/DandyLyons/md-utils/issues/105 + +Use `fm dump --format raw` when the source representation matters. Use `--format json`, `--format yaml`, or `--format toml` when downstream tools need a particular serialization, and use `--frontmatter-format yaml|toml` only when creating or explicitly converting a stored frontmatter block. + +See for command details. diff --git a/Sources/md-utils/Documentation.docc/md-utils.md b/Sources/md-utils/Documentation.docc/md-utils.md index 7f730c3..3d2ed8a 100644 --- a/Sources/md-utils/Documentation.docc/md-utils.md +++ b/Sources/md-utils/Documentation.docc/md-utils.md @@ -11,6 +11,7 @@ The executable target wraps the `MarkdownUtilities` library in a Swift Argument ### Frontmatter - +- ### Project Rules