diff --git a/.claude/agents/harvester.md b/.claude/agents/harvester.md index 32e3c87f94..3a6b8c47c4 100644 --- a/.claude/agents/harvester.md +++ b/.claude/agents/harvester.md @@ -28,7 +28,9 @@ other filing unnecessary. Only a comment no name can carry goes on to the rest. `REVIEW.md`, rewritten as a ban or duty per `REVIEW_COMMON.md`. Delete the comment. - **FACT** - it states how the system is or why its shape wins. Propose it for the architecture doc, as a present-tense statement per `ARCHITECTURE_COMMON.md`, naming the - section it joins. Delete the comment. + section it joins, the `{#anchor}` that section carries (or the one to mint), and the CITER - + the function the comment sat on, which gets `[arch(at="#")]` when the fact + lands. A FACT without a citer is not filed: it is KEEP or DROP. Delete the comment. - **KEEP** - a constraint true only at this code site, which filing to the architecture doc would bury. Compress to one line in place, spelled `//!` - a contract comment, legal on any visibility (a private `//!` caps at 3 lines under STYLE015). A comment already in the @@ -59,7 +61,8 @@ Your final message is the ledger, nothing else: 1. Counts first: N comments -> R RULE / F FACT / K KEEP / D DROP / RN RENAME / T TODO. 2. Every RULE, FACT, and TODO: the original comment (condensed to its point), the exact - proposed destination text, and the destination (REVIEW.md; arch doc + section; ledger). + proposed destination text, and the destination (REVIEW.md; arch doc + section + anchor + + citer, as `file:def name`; ledger). A FACT line missing its citer is an incomplete ledger. Every RENAME: the comment, current name -> proposed name, one line on what the new name carries. 3. KEEP entries only where you compressed: before -> after, one line each. diff --git a/.claude/hooks/LAWS.md b/.claude/hooks/LAWS.md new file mode 100644 index 0000000000..af1b2d3eeb --- /dev/null +++ b/.claude/hooks/LAWS.md @@ -0,0 +1,5 @@ +# LAWS - .claude/hooks + +| date | document | ruling | +|---|---|---| +| 2026-08-28 | README.md (the stdin-only rule now admits the files the payload names; arch_inject.py section) | Boris: "we will land the pr, make the hook, and try on next session. and i will ask u to asses it towards the end - i.e. if the noise is too much, if it was of any use" + "how do we prevent repeating ourselves like crazy?" - the [arch] section arrives on the first edit inside a cited function, once per anchor per session, memo reset on compaction | diff --git a/.claude/hooks/README.md b/.claude/hooks/README.md index baf62fb94c..4e49b5f261 100644 --- a/.claude/hooks/README.md +++ b/.claude/hooks/README.md @@ -1,8 +1,9 @@ # Claude Code hooks -Hooks for agent sessions in this repo. **A hook reads only its stdin payload - -no daslang, no config reads - and returns in tens of milliseconds.** It must -work when the das binary is down. +Hooks for agent sessions in this repo. **A hook reads only its stdin payload, the +file that payload names, and the documents that file cites - no daslang, no config +reads - and returns in tens of milliseconds.** It must work when the das binary is +down. There is deliberately no write-time comment hook: comments are working scaffolding - write them freely during a PR. The gate is `make_pr`'s comment @@ -19,6 +20,18 @@ use instead is `daslang utils/internal/pr-babysit/main.das -- --pr --watch`, run bare in a background Bash (exit code is the verdict). A single un-looped `gh pr checks ` from Bash passes. +## arch_inject.py - PostToolUse, `Edit|Write|MultiEdit` + +When an edit lands inside a `.das` function carrying `[arch(at=".md#")]`, +the hook returns the cited section as `additionalContext` - the 40 lines that govern the +function arrive when the function is touched, instead of a 300-line document being read +first. Once per anchor per session: a memo in the temp directory +(`claude-arch-memo-.json`) keeps each anchor's section hash, so twelve citers +of one section cost one injection and an edited section re-injects. `--reset` (registered +on `PreCompact` and `SessionEnd`) drops the memo, because after a compaction the earlier +copy has left the context. Sections cap at 60 lines; `arch_of` returns the rest. Reads and +`arch_of` stay quiet - the moment that matters is writing. + ## Registration The tracked `.claude/settings.json` registers hooks for every checkout; diff --git a/.claude/hooks/arch_inject.py b/.claude/hooks/arch_inject.py new file mode 100644 index 0000000000..14ecbb0970 --- /dev/null +++ b/.claude/hooks/arch_inject.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""PostToolUse hook (Edit|Write|MultiEdit): when the edit lands inside a function that carries +[arch(at=".md#")], hand the cited section to the model - once per anchor per +session (a memo keyed by anchor and section hash), so twelve citers of one section cost one +injection. `--reset` (PreCompact / SessionEnd) drops the memo: after a compaction the earlier +copy left the context, so the next edit re-injects. Reads only stdin plus the files it names.""" + +import hashlib +import json +import os +import re +import sys +import tempfile + +CITE = re.compile(r'arch\s*\(\s*at\s*=\s*"([^"#]+)#([^"]+)"') +DEF = re.compile(r'^\s*def\b') +HEADING = re.compile(r'^(#{1,6})\s+.*\{#([^}\s]+)\}\s*$') +MAX_LINES = 60 + + +def memo_path(session_id): + return os.path.join(tempfile.gettempdir(), "claude-arch-memo-%s.json" % (session_id or "nosession")) + + +def load_memo(path): + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return {} + + +def save_memo(path, memo): + try: + with open(path, "w", encoding="utf-8") as f: + json.dump(memo, f) + except Exception: + pass + + +def read_text(path): + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + return f.read() + except Exception: + return "" + + +def citations_above(lines, def_index): + """The [arch] citations in the annotation block directly above a def line.""" + found = [] + i = def_index - 1 + while i >= 0: + t = lines[i].strip() + if t == "" or t.startswith("//"): + i -= 1 + continue + if t.startswith("[") or t.endswith("]") or t.endswith(","): + found.extend(CITE.findall(t)) + i -= 1 + continue + break + return found + + +def enclosing_citations(text, offset): + """Citations of the function whose body holds `offset` (or the annotation block the edit sits in).""" + lines = text.split("\n") + line_no = text.count("\n", 0, max(offset, 0)) + for i in range(min(line_no, len(lines) - 1), -1, -1): + if DEF.match(lines[i]): + return citations_above(lines, i) + # an edit on the annotation block itself + if CITE.search(lines[i]) and i >= line_no: + return CITE.findall(lines[i]) + return [] + + +def all_citations(text): + return CITE.findall(text) + + +def section_of(doc_text, anchor): + lines = doc_text.split("\n") + start = -1 + level = 0 + for i, line in enumerate(lines): + m = HEADING.match(line.rstrip("\r")) + if m and m.group(2) == anchor: + start = i + level = len(m.group(1)) + break + if start < 0: + return "" + stop = len(lines) + for i in range(start + 1, len(lines)): + m = re.match(r'^(#{1,6})\s', lines[i]) + if m and len(m.group(1)) <= level: + stop = i + break + body = [l.rstrip("\r") for l in lines[start:stop]] + while len(body) > 1 and body[-1].strip() == "": + body.pop() + if len(body) > MAX_LINES: + body = body[:MAX_LINES] + ["... (section continues - `arch_of` returns it whole)"] + return "\n".join(body) + + +def main(): + raw = sys.stdin.read() + try: + payload = json.loads(raw) if raw.strip() else {} + except Exception: + return 0 + session = payload.get("session_id", "") + path = memo_path(session) + if "--reset" in sys.argv: + try: + os.remove(path) + except Exception: + pass + return 0 + tool = payload.get("tool_name", "") + tin = payload.get("tool_input", {}) or {} + file_path = tin.get("file_path", "") or "" + if tool not in ("Edit", "Write", "MultiEdit") or not file_path.endswith(".das"): + return 0 + text = read_text(file_path) + if not text or "arch" not in text: + return 0 + cites = [] + if tool == "Write": + cites = all_citations(text) + else: + edits = tin.get("edits", []) if tool == "MultiEdit" else [tin] + for e in edits: + needle = e.get("new_string", "") or e.get("old_string", "") or "" + if not needle.strip(): + continue + # every occurrence: a repeated snippet cannot say which function took the edit, + # so each enclosing function's sections are owed (the memo folds the repeats) + at = text.find(needle) + hits = 0 + while at >= 0 and hits < 16: + cites.extend(enclosing_citations(text, at)) + hits += 1 + at = text.find(needle, at + 1) + if not cites: + return 0 + folder = os.path.dirname(os.path.abspath(file_path)) + memo = load_memo(path) + out = [] + seen_here = set() + for doc, anchor in cites: + doc_path = os.path.normpath(os.path.join(folder, doc)).replace("\\", "/") + key = "%s#%s" % (doc_path, anchor) + if key in seen_here: + continue + seen_here.add(key) + section = section_of(read_text(doc_path), anchor) + if not section: + continue + digest = hashlib.sha1(section.encode("utf-8")).hexdigest() + if memo.get(key) == digest: + continue + memo[key] = digest + rel = os.path.relpath(doc_path, payload.get("cwd", "") or os.getcwd()).replace("\\", "/") + out.append("[arch] %s#%s - the section this function answers for:\n%s" % (rel, anchor, section)) + if not out: + return 0 + save_memo(path, memo) + context = ("The function you just edited cites an architecture section. Keep the code true to it, " + "or change the section in the same diff (REVIEW_COMMON.md's [arch] audit duty).\n\n" + + "\n\n".join(out)) + print(json.dumps({"hookSpecificOutput": {"hookEventName": "PostToolUse", + "additionalContext": context}})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/settings.json b/.claude/settings.json index 83b74867ef..fdbf001f76 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,17 +1,52 @@ -{ - "hooks": { - "PreToolUse": [ - { - "matcher": "Monitor|Bash", - "hooks": [ - { - "type": "command", - "command": "jq -c -f \"$CLAUDE_PROJECT_DIR/.claude/hooks/monitor_guard.jq\"", - "timeout": 15, - "statusMessage": "PR-watch guard: pr-babysit is the tool" - } - ] - } - ] - } +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Monitor|Bash", + "hooks": [ + { + "type": "command", + "command": "jq -c -f \"$CLAUDE_PROJECT_DIR/.claude/hooks/monitor_guard.jq\"", + "timeout": 15, + "statusMessage": "PR-watch guard: pr-babysit is the tool" + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Edit|Write|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "python3 \"$CLAUDE_PROJECT_DIR/.claude/hooks/arch_inject.py\"", + "timeout": 10, + "statusMessage": "[arch] section for the edited function" + } + ] + } + ], + "PreCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 \"$CLAUDE_PROJECT_DIR/.claude/hooks/arch_inject.py\" --reset", + "timeout": 5 + } + ] + } + ], + "SessionEnd": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 \"$CLAUDE_PROJECT_DIR/.claude/hooks/arch_inject.py\" --reset", + "timeout": 5 + } + ] + } + ] + } } \ No newline at end of file diff --git a/.github/workflows/extended_checks.yml b/.github/workflows/extended_checks.yml index 2849bc01e2..0c2be971a5 100644 --- a/.github/workflows/extended_checks.yml +++ b/.github/workflows/extended_checks.yml @@ -365,15 +365,17 @@ jobs: # build/doc_verify holds .das extracted verbatim from RST by the doc-verify step above - not authored source $BIN/das-fmt.exe --path ./ --verify --exclude-mask build/ - - name: "Run lint on changed .das files" + - name: "Run lint on changed .das and .md files" if: matrix.target == 'linux' run: | set -eux BASE_REF="${{ github.event.pull_request.base.ref }}" BASE_REF="${BASE_REF:-master}" - mapfile -t CHANGED < <(git diff --name-only --diff-filter=AM "origin/${BASE_REF}...HEAD" -- '*.das') + # a changed .md arms its folder's document rules (LINT025/026/027) with no .das compiled; + # a repo-root .md arms the whole tree - intended, and a few seconds with nothing to compile + mapfile -t CHANGED < <(git diff --name-only --diff-filter=AM "origin/${BASE_REF}...HEAD" -- '*.das' '*.md') if [ ${#CHANGED[@]} -eq 0 ]; then - echo "no .das files changed; skipping lint" + echo "no .das or .md files changed; skipping lint" exit 0 fi echo "linting: ${CHANGED[*]}" diff --git a/ARCHITECTURE_COMMON.md b/ARCHITECTURE_COMMON.md index 34683c7fd4..736ced7da5 100644 --- a/ARCHITECTURE_COMMON.md +++ b/ARCHITECTURE_COMMON.md @@ -28,6 +28,15 @@ and the exception ledger. Nothing else.** **Sections are numbered, and rules cite them by section.** A section number is never reused for different content: append new sections, never renumber. +**A section that code implements carries a `{#anchor}` on its heading, and every anchor is +cited by an `[arch(at="#")]` on a function in the document's own folder tree.** +One anchor per heading; the anchor name is stable across rewording, like the section number. +An anchor no function cites, a citation naming no anchor, and a citation reaching a document +outside the citer's folder tree are all lint findings (LINT026), in every folder. A mechanism +another folder's document states is restated here in prose - a paragraph, not a resolved link - +and the code cites this document. A section no function implements is narrative and carries no +anchor. + **A fact that a rule or a code comment cites is load-bearing: it must stay true.** The same-change duty that keeps it true belongs in the folder's `REVIEW.md`, not here. diff --git a/CLAUDE.md b/CLAUDE.md index ef4ba2cd48..e7fba661f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,12 +44,12 @@ cites.** MCP `arch_of` returns each of a file's citations with its section text; returns a document's anchors with the code citing them, dead anchors and dangling citations included. Citation spelling, path resolution, and failure reasons: `skills/mcp_tools.md`. -**An architecture-doc heading that code cites carries the `{#anchor}` its citation names.** -LINT026 fails a citation naming no such file or anchor, and in a folder whose `.lint_config` -sets `[docs] enforce_arch = true`, an anchor no `[arch]` cites. +**An architecture-doc heading that code cites carries the `{#anchor}` its citation names, and +every anchor is cited.** LINT026 fails a citation naming no such file or anchor, and - in every +folder, no tag arms it - an anchor no `[arch]` cites. An `[arch]` citation replaces the comment +that would have restated the section: the mechanism lives in the document, the code names it. -**A `REVIEW*.md` or `ARCHITECTURE*.md` past 300 lines, in a folder whose `.lint_config` sets -`[docs] rule_docs_only = true` or `[docs] enforce_arch = true`, splits into companions** - +**A `REVIEW*.md` or `ARCHITECTURE*.md` past 300 lines, wherever it sits, splits into companions** - `ARCHITECTURE_.md` for an architecture doc, a `REVIEW_.md` the parent checklist's opening routes to for a checklist. LINT027 is the gate; the `archivist` agent does the architecture-doc splits. diff --git a/LAWS.md b/LAWS.md index 688c6040b5..cc00a79fa6 100644 --- a/LAWS.md +++ b/LAWS.md @@ -27,3 +27,6 @@ compacted, or cited as rules. | 2026-08-27 | skills/internal/make_pr.md | "should we update make_pr skill?" - the comment-drain row becomes the diff-scoped harvest row (the ruled harvest regime, trial passed on dasllama_image.das); rescue-bot references retired from the flow; the format row states no folder strips comments; the tree-frozen-during-preflight rule lands from this session's lib/-hiding incident | | 2026-08-27 | .claude/agents/ (rescue-bot, rescue-sweep-bot -> history/agents/), strip-advisory strings, .claude/hooks/README.md | "yes, lets retire both. they go to /history/agents i guess. this is evolution" - both rescue bots archived; the dormant strip advisory names the harvester as successor | | 2026-08-27 | REVIEW_COMMON.md (the [arch] audit triangle) | "1. if function with [arch(section)] is modified, other functions which link that section are audited. 2. if [arch(foo#section)] is modified, section gets audited to not go stale. 3. if foo#section is modified, all linked functions are audited." + on scope: "repo-wide from day one." - three constitutional rules after the ships-with-tests clause; one pass per anchor discharges all duties, no cascade | +| 2026-08-28 | CLAUDE.md (document system), daslib/lint_config.das, utils/lint (LINT026 reverse + LINT027 arming), modules/dasLLAMA/.lint_config (deleted), utils/lint/README.md | "remove enable_arch link option entirely. its now mandatory." + "the idea is that arch lnks replace comments, where necessary" - the [arch] reverse pass runs in every folder with no tag; the LINT027 line gate arms on any folder holding a rule document; a citation replaces the comment that would restate the section (the harvester deletes the leftovers) | +| 2026-08-28 | REVIEW_COMMON.md (harvest duty), ARCHITECTURE_COMMON.md (anchor form), .claude/agents/harvester.md (FACT names its citer), skills/internal/make_pr.md (row 0a0) | "i would like to ensure that harvested sections of architecture.md are linked to appropriate functions. im thinking a section in REVIEW_COMMON.md should be able to cover it" - a filed FACT lands as statement + anchor + citation from the source function in the same change; the harvester ledger names the citer or the fact is not filed | +| 2026-08-28 | ARCHITECTURE_COMMON.md (anchor form), doc/source/reference/language/lint.rst (LINT026), skills/mcp_tools.md, utils/common/arch_citations.das, .github/workflows/extended_checks.yml (changed-set glob) | "should cite own subtree. if need citation outside - make citation in architecture.md in own subtree, in text form. not like automatic cross-link we resolve, but just a paragraph" + lint updated .md on CI, "skipping should be very cheap" - an [arch] citation must target a document whose folder is the citer folder or an ancestor; CI passes changed .md to the lint (a .md positional arms its folder with no .das compiled) | diff --git a/REVIEW_COMMON.md b/REVIEW_COMMON.md index 12b49d08bd..f4c5f305ff 100644 --- a/REVIEW_COMMON.md +++ b/REVIEW_COMMON.md @@ -40,6 +40,11 @@ describes this function; verify it does. audit pass over an anchor's section text and citer set discharges every audit duty the diff triggers on that anchor - the duties never cascade. +**A diff that moves a fact out of a function's comment into an `ARCHITECTURE*.md` lands three +things in the same change: the statement, a `{#anchor}` on the heading of the section it joins, +and an `[arch(at=...)]` citation on that function - and the comment goes.** The citation is +what keeps the section true: a section nothing cites is never re-checked when the code changes. + **A rule that a test, a lint, or the folder's `REVIEW.das` enforces is deleted.** Automation replaces the rule; the checklist keeps at most "weakening that check is a defect." A rule that COULD be automated is a lint or `REVIEW.das` candidate - say so in the review round. diff --git a/ci/smoke_test_bundle.sh b/ci/smoke_test_bundle.sh index b9d1873114..931b5939ed 100644 --- a/ci/smoke_test_bundle.sh +++ b/ci/smoke_test_bundle.sh @@ -212,8 +212,11 @@ run_check "mcp.das (empty stdin)" bash -c \ run_check "dastest.exe runs a shipped suite" bash -c \ "set -o pipefail; '$BUNDLE/bin/dastest${DASEXE_SUFFIX}' --test utils/common/tests --isolated-mode | tee '$LOG.suite' \ && grep -Eq '^[1-9][0-9]* tests, [1-9][0-9]* passed, 0 failed, 0 errors' '$LOG.suite'" +# LINT026 is off here by construction: an [arch] citation names an ARCHITECTURE*.md, and +# those documents never ship (see the leak check below), so in the bundle every citation +# in daslib is a dangling one. The source tree's lint lanes verify the citations. run_check "lint.exe lints daslib, no skips" bash -c \ - "set -o pipefail; '$BUNDLE/bin/lint${DASEXE_SUFFIX}' daslib | tee '$LOG.lint' \ + "set -o pipefail; '$BUNDLE/bin/lint${DASEXE_SUFFIX}' daslib --disable LINT026 | tee '$LOG.lint' \ && grep -Eq '^[0-9]+ files, 0 issue\(s\), 0 error\(s\)\$' '$LOG.lint'" # Shipped skills must not send the reader to a path the bundle does not contain. diff --git a/daslib/ARCHITECTURE_EMIT.md b/daslib/ARCHITECTURE_EMIT.md index b4b49df7a5..ee073e9330 100644 --- a/daslib/ARCHITECTURE_EMIT.md +++ b/daslib/ARCHITECTURE_EMIT.md @@ -151,9 +151,13 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across 64-bit INT (`arith_width_ok` allows width 64 only for floats); `cpu_only_lattice_width` keys both emitters' fail-closed diagnostic. -## 29. shader_lingua_franca +## 29. shader_lingua_franca {#shader-lingua-franca} - **Every symbol is either an exact CPU mirror of its GPU semantics or a `[sideeffects]` dummy every rail lowers by name** - the dummies return zero on the host, so a CPU replay reproduces GPU semantics only for the real-bodied set. Unsigned overloads never fold into signed twins (glslang picks the unsigned opcode). +- **A width-variant of a lowered-by-name symbol is one more overload here, never an emitter + arm.** `unpack8` carries `int16 -> byte2` and `uint16 -> ubyte2` beside the 32-bit pair; every + overload is the same `reinterpret` on the host and the same single `OpBitcast` on the SPIR-V + rail, so the emitter matches the name and reads the width off the operand type. diff --git a/daslib/REVIEW.md b/daslib/REVIEW.md index f03a67a46c..2ede18641c 100644 --- a/daslib/REVIEW.md +++ b/daslib/REVIEW.md @@ -42,8 +42,8 @@ sides.** Nothing catches it when one side later moves alone. it changes the other side and updates the pair's architecture-doc entry in the same diff.** -**A comment-sweep diff adds an architecture-doc entry only where no name, shape, or test -can carry the fact.** +**A diff that adds an architecture-doc entry adds it only where no name, shape, or test can +carry the fact.** **Weakening `tests/lint/test_nolint_suppression.das` is a defect** - it pins that a string literal, a URL, and a mid-comment `nolint:` do not suppress while a first-token directive diff --git a/daslib/lint_config.das b/daslib/lint_config.das index f5d24cba31..abd51d032d 100644 --- a/daslib/lint_config.das +++ b/daslib/lint_config.das @@ -36,10 +36,8 @@ module lint_config shared private //! ``rule_docs_only = true`` marks a folder that carries rule documents only; //! ``rule_docs_only_at`` reads that one file, and the LINT025 pass in //! ``utils/lint/main.das`` reports any other ``.md`` sitting beside the code. -//! ``enforce_arch = true`` marks a folder whose ``.md`` anchors are all owed an -//! ``[arch]`` citation; ``enforce_arch_at`` reads it, and the LINT026 pass in -//! the same runner reports every anchor no code cites. Either tag also arms that -//! runner's LINT027 line gate over the folder's rule documents. +//! The ``[arch]`` reverse pass (LINT026: an anchor no code cites) and the LINT027 +//! line gate over rule documents need no tag - they run in every folder. require daslib/fio require daslib/json @@ -309,13 +307,6 @@ def public rule_docs_only_at(config_path : string) : bool { return docs_flag_at(config_path, "rule_docs_only") } -//! True when the ``.lint_config`` at `config_path` sets ``[docs] enforce_arch = true`` — every -//! ``{#anchor}`` in the folder's ``.md`` owes an ``[arch]`` citation from the ``.das`` beside it, -//! which LINT026 reports in reverse. A folder property like ``rule_docs_only_at``. -def public enforce_arch_at(config_path : string) : bool { - return docs_flag_at(config_path, "enforce_arch") -} - def private config_dir_of(file : string) : string { let dir = to_generic_path(dir_name(to_generic_path(file))) return empty(dir) ? "." : dir diff --git a/daslib/shader_lingua_franca.das b/daslib/shader_lingua_franca.das index 3fab26a4b4..c027c5f061 100644 --- a/daslib/shader_lingua_franca.das +++ b/daslib/shader_lingua_franca.das @@ -277,6 +277,16 @@ def public unpack8(x : uint) : ubyte4 { return unsafe(reinterpret(x)) } +[arch(at="ARCHITECTURE_EMIT.md#shader-lingua-franca")] +def public unpack8(x : int16) : byte2 { + return unsafe(reinterpret(x)) +} + +[arch(at="ARCHITECTURE_EMIT.md#shader-lingua-franca")] +def public unpack8(x : uint16) : ubyte2 { + return unsafe(reinterpret(x)) +} + def public pack32(v : byte4) : int { return unsafe(reinterpret(v)) } diff --git a/doc/reflections/das2rst.das b/doc/reflections/das2rst.das index a2f73e917f..9f90b0089a 100644 --- a/doc/reflections/das2rst.das +++ b/doc/reflections/das2rst.das @@ -1364,7 +1364,7 @@ def document_module_lint(_root : string) { def document_module_lint_config(_root : string) { var mod = find_module("lint_config") var groups <- array( - group_by_regex("Configuration", mod, %regex~(load_lint_config|load_lint_config_from_path|load_env_disabled|seed_default_disabled|build_lint_macro_disabled|lint_config_forces_on|rule_docs_only_at|enforce_arch_at)$%%), + group_by_regex("Configuration", mod, %regex~(load_lint_config|load_lint_config_from_path|load_env_disabled|seed_default_disabled|build_lint_macro_disabled|lint_config_forces_on|rule_docs_only_at)$%%), group_by_regex("Path excludes", mod, %regex~(load_path_excludes_from_path|matches_path_excludes|is_lint_path_excluded)$%%), group_by_regex("Path-based rule defaults", mod, %regex~(is_daslib_source|is_shipped_library_source|is_core_library_source)$%%), group_by_regex("Lint-surface predicates", mod, %regex~(is_user_authored_body|is_lint_fixture_name|lint_file_skip_reason)$%%), diff --git a/doc/source/reference/language/lint.rst b/doc/source/reference/language/lint.rst index 3107df50a1..23c8676139 100644 --- a/doc/source/reference/language/lint.rst +++ b/doc/source/reference/language/lint.rst @@ -799,23 +799,24 @@ The path resolves **relative to the citing file's folder**, the document must exist, and the anchor must appear in it exactly once as a heading suffix — ``## Residency ramp {#residency}``, at any heading level. An anchor declared twice is reported like a missing one: the citation reads as precise while -pointing at whichever section the reader reaches first. - -The rule also runs in reverse, for folders that ask for it: - -.. code-block:: toml - - # .lint_config beside the code the document describes - [docs] - enforce_arch = true - -In an armed folder every ``{#anchor}`` in every ``.md`` beneath it must be -cited by at least one ``[arch]`` in the ``.das`` beneath it — an anchor is a -promise that some code answers for the section, so an uncited one is either -code that forgot to say so or a section that was never anyone's contract. -Strip the anchor to demote such a section to narrative. Like ``[docs] -rule_docs_only``, ``enforce_arch`` is a folder property with no cascade: each -directory answers for itself. +pointing at whichever section the reader reaches first. The document must +also sit in the citing file's **own folder tree** — its folder is the file's +folder or an ancestor of it. A mechanism that another folder's document states +is restated in prose, in a document of the citer's own tree, and that is what +the code cites; a resolved cross-tree link would be invisible to the folder's +own review walk and to ``arch_sites``. + +The rule also runs in reverse, everywhere: every ``{#anchor}`` in every ``.md`` +under the run's roots must be cited by at least one ``[arch]`` in a ``.das`` +under those roots — an anchor is a promise that some code answers for the +section, so an uncited one is either code that forgot to say so or a section +that was never anyone's contract. Strip the anchor to demote such a section to +narrative. The roots are read as one set, so a citer in ``daslib/`` satisfies +an anchor in a module's document. No ``.lint_config`` key arms or disarms the +reverse direction. The rule is a source-tree rule: the installed SDK ships no +architecture documents, so the bundle's own lint run passes ``--disable +LINT026`` and the citations in shipped ``daslib/`` files are verified only +in the tree they came from. Both directions read source text rather than the AST, so a citation is checked even in a file the linting environment cannot compile — and so a structure's @@ -826,17 +827,10 @@ string literal is not. LINT027 — rule document exceeds 300 lines ========================================== -The third folder rule: in a folder whose ``.lint_config`` carries ``[docs] -rule_docs_only = true`` or ``[docs] enforce_arch = true``, every -``REVIEW*.md`` and ``ARCHITECTURE*.md`` sitting there is capped at 300 lines. -Past that a checklist stops being read end to end, and a rule nobody reaches -is not a rule. - -.. code-block:: toml - - # either tag arms the line gate - [docs] - rule_docs_only = true +The third folder rule: in every folder that holds one, each ``REVIEW*.md`` +and ``ARCHITECTURE*.md`` is capped at 300 lines. Past that a checklist stops +being read end to end, and a rule nobody reaches is not a rule. No +``.lint_config`` key arms the gate. The fix is a split, never a trim — every criterion survives, moved into a companion the reader can hold: ``ARCHITECTURE_.md`` for an diff --git a/history/README.md b/history/README.md index d4571ff785..fa022c71f9 100644 --- a/history/README.md +++ b/history/README.md @@ -27,7 +27,8 @@ Process and criteria: `skills/internal/doc_archiving.md`. - `agents/` - retired agent charters, superseded by their successors (the harvester absorbed both rescue bots) - `dasVulkan/` - the original boost-layer design plan, carried over when the module moved - in-tree; the living docs are `modules/dasVulkan/CLAUDE.md` and its `ROADMAP.md` + in-tree from the standalone borisbat/dasVulkan repo, which is archived with full history; + the living docs are `modules/dasVulkan/ARCHITECTURE.md`, `CLAUDE.md` and `ROADMAP.md` - `linq_fold/` - the linq_fold arc's plans, audits and the archived masterplan; the living reference is `daslib/ARCHITECTURE_LINQ.md` sec. 37, results stay at `benchmarks/sql/results.md` - `examples/` - plans, findings, and research notes behind shipped examples @@ -148,3 +149,4 @@ listed in the log below - search it first when hunting for a doc. - 2026-08-27 `modules/dasLLAMA/ARCHITECTURE.md` (retired-note passages) -> `history/dasLLAMA/architecture_retired_notes.md` - the deleted whisper-decoder attention kernel and the three "superseded/corrected" narrations of the archived design docs, archived when ARCHITECTURE.md was groomed and split into its seven companions - 2026-08-27 `.claude/agents/rescue-bot.md` -> `history/agents/rescue-bot.md` - fresh-scaffolding rescue at the PR gate, superseded by the harvester (make_pr row 0a0) - 2026-08-27 `.claude/agents/rescue-sweep-bot.md` -> `history/agents/rescue-sweep-bot.md` - legacy comment-sweep rescue, superseded by harvest-on-first-touch +- 2026-08-28 `modules/dasVulkan/CLAUDE.md` (the standalone-repo provenance sentence) -> the `dasVulkan/` bullet of this file - archived when CLAUDE.md split into `ARCHITECTURE.md` (the present-tense half) plus agent instructions diff --git a/modules/dasLLAMA/.lint_config b/modules/dasLLAMA/.lint_config deleted file mode 100644 index 766cbe0035..0000000000 --- a/modules/dasLLAMA/.lint_config +++ /dev/null @@ -1,3 +0,0 @@ -# Every {#anchor} in this module's .md files is owed an [arch] citation from module code (LINT026). -[docs] -enforce_arch = true diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index 1a11070b08..67fa005779 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -40,6 +40,9 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). - `ARCHITECTURE_GPU.md` - sec.2.2b: the tensor-GEMM and fused-attention shapes that measured out. - `ARCHITECTURE_GPU_PREFILL.md` - sec.2.2c-2.2i: the Metal prefill driver's GEMM form ladder, dev-W knee map, attention slab, MoE bucket rail, and chunked submission. +- `ARCHITECTURE_GPU_VULKAN.md` - sec.2.2j-2.2p: the Vulkan resident driver - the prefill window + chain, the cm2 decode lanes and tile pick, the class-pipeline build seat, the residency plan, + the GPU-slot marks swap, and the Q8 requant byte store. - `ARCHITECTURE_RUNTIME.md` - sec.2.2, 2.3, 2.3a, 2.4, 2.6-2.9, 2.11, 2.12: kernel shape, caches, lint policy, knobs, coverage, the GPU ramp. - `ARCHITECTURE_MEASUREMENT.md` - sec.2.5, 2.10: the benchmark rig, the tune gate, and the diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU.md b/modules/dasLLAMA/ARCHITECTURE_GPU.md index 7201eb514a..669dc1f3d2 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU.md @@ -2,7 +2,7 @@ Companion to `ARCHITECTURE.md`; section numbers are that document's. -### 1.5 GPU backends +### 1.5 GPU backends {#gpu-backends} A GPU backend is a FAMILY of role files - matching things in matching files across backends, so that a question answered for one backend has an obvious address in the other. The roles: @@ -88,6 +88,11 @@ that a question answered for one backend has an obvious address in the other. Th override registries. `"vulkan"` is the tier string it registers under, not a dependency, which is why it compiles on every box. It requires common back for `Model`/`Session`, so like the Metal drivers it is required from the transformer umbrella, never from common. +- **A dry bake runs the whole resident arm with no device.** `vulkan_bake_role` puts the tier in + bake mode, and each `rdec_*` device seam answers for itself so the arm walk reaches the end: a + seam that only records a layout (`vk_rdec_set_emb`) answers true, a seam that would allocate + device memory (`vk_rdec_upload_emb_f32`) answers false without touching a device. The split is + what keeps a baked `.dlim` layout equal to the one a real arm produces. - **`dasllama_kernel_access.das`** - the shared body-walk read/write classifier both GPU lenses run on, plus the dispatch-lens micro-grammar (the grid/tg/params spec tokenizers and the shared AST-emission core: `is_digit_tok`, `role_ok`, `derived_role`, `mk_uint_cast`, `mk_call1`, @@ -151,6 +156,17 @@ entry here:** footprint gate: Metal's own pipeline compile fails loudly with the footprint in the error. - **Vulkan has no shapes module yet** - `resident_upload` declines ad hoc by feature name; the gap is `followup_vulkan.md` item 1, not a precedent to copy. +- **The device-side token-embedding gather is Vulkan-only.** The engine asks one probe before + it embeds (`register_embed_gpu_gate`, `dasllama_common.das`); on true it stashes the token + ids, skips the CPU embed loop, and the resident driver gathers the rows on device through + the ids-form prefill. That prefill installs SEPARATELY from the resident driver bundle + (`install_rdec_prefill_ids`), so a tier without it never arms the gate, and a late decline + backfills the stash on the CPU in `forward_prefill_body`. The arms mirror `embed_row`'s + ladder: a tied q8 table gathers from the resident cls plane, a raw f32 table uploads whole + to a device plane under a size cap the residency plan counts first, and the kq ladder keeps + the CPU embed. What leaves the window wall is the CPU embed loop and the x upload - the ids + ride a 4-byte-per-row upload instead. Metal has no twin: its whole-forward driver embeds + host-side. Vulkan is the deliberately-designed model of this shape; Metal converges as it is touched. @@ -188,3 +204,5 @@ The positive laws these races established - half operands, stage-only-to-transfo consecutive staging runs, relaxed_precision always - are `REVIEW_GPU.md` rules and the `modules/dasMetal/REVIEW.das` descriptor gate; this section keeps only the refuted shapes and why they lose. + +Sections 2.2j-2.2p, the Vulkan resident driver, are `ARCHITECTURE_GPU_VULKAN.md`. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md new file mode 100644 index 0000000000..7f6e585ab9 --- /dev/null +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md @@ -0,0 +1,159 @@ +# dasLLAMA Architecture - the Vulkan resident driver + +Companion to `ARCHITECTURE_GPU.md`; section numbers are `ARCHITECTURE.md`'s. This document +carries sections 2.2j-2.2p, the mechanisms of the Vulkan resident driver: the prefill window +chain, how a cm2 tile decodes its quant bytes and how one is picked, the class-pipeline build +seat both shader instruments hang on, the residency plan, the marks swap that lets one GPU slot +serve many models, and the Q8 requant byte store. The GPU backend role table these sections +build on stays in `ARCHITECTURE_GPU.md` sec.1.5. + +### 2.2j The Vulkan resident prefill window chain {#vk-prefill-window-chain} + +Companion to `ARCHITECTURE_GPU.md` sec.1.5; the Metal prefill driver's own ladder is +`ARCHITECTURE_GPU_PREFILL.md`. + +**A prompt longer than `PF_WINDOW` rows runs as SEQUENTIAL windows over the same activation +buffers.** Every window's rope and attention address the KV mirror at ABSOLUTE positions, so +window w attends everything the earlier windows stored; only the last window runs the final +requant and the classifier. + +**The k and v GEMMs merge into ONE dispatch when the layer's q, k and v weight planes are all +q8 and the k and v planes sit adjacent in the arena.** The bump allocator places them +back-to-back unless a slab boundary intervenes, so the merged form asks only those two +questions and dispatches at `d = 2 * kvd`: one dispatch instead of two plus a copy, which +doubles the otherwise starved k/v grid and deletes the v copy. Consumers read the merged output +through a `kstride` field - the projection-row stride, `2 * kvd` merged against the split +path's `kvd` - on `RopeKvBArgs` and on `QkRmsArgs`, and through `RopeKvBArgs.voff`, the v +rows' base inside that buffer - `kvd` merged against the split path's `npos * kvd`. It is a +record-cost win, not a GPU one. + +**A window chain submits in chunks so recording overlaps execution.** The chain splits on a +1, 2, 4, 8-layer ramp, so the GPU starts on an early small chunk while the CPU is still +recording, and the doubling stops at 8 layers. Chunks go out through `submit_nofence`; the +window's last submit is the `submit_wait` that signals the one fence the caller waits on. +Ordering between chunks is the hazard rail's: a barrier recorded in chunk N+1 covers the +writes of chunk N because submission order on one queue spans submits, and the terminal +fence covers every earlier submit the same way. The command-buffer ring holds +`3 + ceil(depth/8)` buffers, so chunk N records into a free buffer while chunk N-1 executes; +the per-role GPU profile pins the single submit, so a chunk gap never bills to a role. + +A `VkHaz` is private to one RECORDING SESSION, not to one command buffer: the chunked chain +carries the same `h` across every buffer in the window, so a barrier it records in chunk N+1 +still knows what chunk N wrote. That is also why the batch/hybrid region bits (`VHB_*`) reuse +the same rail under their own namespace - two recorders never share a pending set. + +**The attention chain's K/V host readbacks are recorded at the END of the chain, after the +`wo` GEMM, not beside the preps that produce them.** Driver 610.74 on the RTX 5060 Ti drops +the in-command compute-to-transfer barrier about one run in twelve: a copy recorded right +after the producing dispatch, behind a spec-valid global memory barrier, reads a partial +prefix of the output while a second identical copy at the end of the same command buffer +reads it whole. The intervening attention, requant and `wo` work is what closes that window; +the copies carry a `//!` naming this section, and the placement is a driver-defect mitigation, +not a chain-shape preference. + +**A layer's qkv feed comes out of the previous layer's FUSED add+rms twin when the fuse knob is +on and the feed is not the Q8_K quant form.** The producer is layer l-1's addr_next site, the +consumer is layer l's b+0 slot, and both key on one predicate (`pf_qkv_feed_fused`): where it +holds, addr_next encodes `cls_ar_f16_b` (an f16 feed) or `cls_ar_rq_b` (a Q8_0 feed) straight +out of the row stash and b+0 only stamps; where it does not, the split `cls_ar` writes the +residual row and b+0 converts or requantizes it. The fused twins never write the `xb` plane, so +the last layer always takes the split arm - the final requant reads `xb`. The addr_ffn site +fuses the same way for the gate/up feed. Bit-identity with the split pair is a suite gate. + +**The cm2 flash-attention tile lands its output f16 when the `wo` feed is f16.** The tile +template carries an `OUT16` stamp: the f16 instance converts the O accumulator in-kernel and +writes the `wo` feed plane directly, so the per-layer attn-to-f16 convert never encodes; the +f32 instance serves the quant route. The two device converts agree bit for bit; the CPU's +`float16()` rounds ties differently, so the twin's gate compares device against device. + +### 2.2k The cm2 decode callbacks read their quant bytes as 16-bit lanes {#cm2-decode-16bit-lanes} + +A cm2 tile's decode callback runs inside the driver's block load, and the vendor driver's shader +compiler pattern-matches only one spelling into that path: a 16-bit load (`int16[N]` block +members) followed by `unpack8(w)[i & 1u]` - a byte2 lane select - with sub-fields pulled out by +shift and mask. A 32-bit word with a variable shift runs slower; an `unpack8` of a 32-bit word +indexed by a runtime value (a byte4 dynamic select) drops the whole kernel off the block-load +path, to about a third of the rate. Every cm2 decode - q8, Q4_K, Q6_K - is spelled the 16-bit +way, which is why the block structs are `int16` arrays over the same bytes. + +### 2.2l The cm2 tile pick and the coopmat default ladder {#cm2-tile-pick-and-default} + +**The l/m tile pick is a wave-efficiency comparison.** For a GEMM of width `d` over `cnt` rows +the l tile (256-row columns) and the m tile (128-row columns) each need some number of +workgroups; each grid runs in whole waves over the device's SM count, and the pick compares +occupied slots over allocated slots, cross-multiplied. The m tile wins only on a strict win; a +tie goes to l, whose bigger tile carries twice the arithmetic intensity. Two rules sit ahead of +the comparison: a window of 128 rows or fewer takes m (the l column would run half empty), and +a device that reports no SM count takes l and never splits k. The pick is PURE in +`(d, cnt, sm_count)`, so the class the pipeline binds and the tile rule the meta fill writes +can never disagree. There is no third tile: the narrow-n end is GEMV's. + +**The f16 feed admits exactly three weight formats - q8, Q4_K and Q6_K** - the same set the cm2 +decode callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE generated class. The +prefill driver reaches them through one dispatcher per stage (`cm2_cls_ensure`, `cm2_cls_set`, +`cm2_cls_enc`), all three keyed on the same `(fmt, ml)` pair, so the pipeline a role ensures, +the set it binds and the kernel it encodes can never be three different classes. The decode +GEMV keeps its quant chains: the feed format pick is decoupled from the weight format. + +**The served GEMM mode resolves once, at init, through one ladder.** cm2 where the device has +NV_cooperative_matrix2, else mm where it has KHR_cooperative_matrix, else sdot4; +`DASLLAMA_COOPMAT` overrides the ladder by name, and a cm2 request or force on a device without +the extension lands on mm. The same resolver stamps the mode into the `.dlim` flavor +configuration, so the recorded mode and the running mode cannot drift. + +### 2.2m Class-pipeline creation is the Vulkan tier's one shader A/B seat {#vk-class-pipeline-build} + +`vkd_class_pipe` is the single place a class kernel's SPIR-V becomes a pipeline, so both shader +instruments hang there and nothing else has to know about them. + +**The dump runs before the override.** `DASLLAMA_VK_SPV_DUMP=` writes the EMITTED words as +`/.spv`; `DASLLAMA_VK_SPV_OVERRIDE=` then replaces them with that directory's +file. The order is what makes the pair a round trip: dump a kernel, edit or spirv-opt the file, +serve it back. A dump taken after the override would capture the served words, not the emitted +ones. + +**Full subgroups are a whole-run arm, never a per-pipeline one.** `DASLLAMA_VK_FULLSG` plus a +device that reports the feature sets `g_gpu.full_sg_on` once at device init, and every class +pipeline is then built with `REQUIRE_FULL_SUBGROUPS`. A run never mixes pinned and plain +pipelines, so an A/B compares two whole runs. Plain is the default: pinned measured slower on +the mm_a gate shape. + +### 2.2n The residency plan sizes a whole model before a byte uploads {#resident-plan} + +The resident driver is all-or-nothing, so the plan IS the decision, and it is computed from +`Model` metadata alone. It sizes four numbers against the tier's weight budget: the dense weight +planes, the KV mirror at `seq_cap`, the driver's own device scratch, and the headroom the auto +arm leaves unfilled (zero when the user pins VRAM). KV is reserved BEFORE weights and never +grows: on a discrete card the two compete directly, and evicting weights to grow KV would mean +re-uploading gigabytes. A decline carries a reason, and where the numbers allow one it carries +the remedy that works - a shorter context, because the weights are fixed and the KV is not. + +An OPTIONAL plane rides only the room left under the budget at THIS context - what remains of +`budget_bytes - headroom_bytes` after weights, KV and scratch; the reserved headroom itself +stays unfilled. It never shrinks any of the three, and it reports zero bytes when it does not +fit - so the same model plans the plane in at a short context and out at a long one. The raw f32 embed +table is the one optional plane today. + +### 2.2o One GPU slot, many models: the marks swap {#gpu-slot-marks} + +A multi-model host runs one device tier under several loaded models, and the tier's per-model +state is offset-keyed - two models' marks installed together route one model's dispatches at +the other's planes. `GpuModelMarks` is that state WHOLE: the loader-contract marks plus every +resident-driver per-model global (the activation, the mirror count, the mirror cap, the mirror +codec, and the device-embed arm). The save moves the installed state out and leaves the globals +reading as no-model; the restore is its exact inverse. The whole-model drop clears the same set +and deselects the `"vulkan"` overrides, so a dropped model's prefill and decode take the plain +CPU path and a later re-arm passes `resident_upload`'s no-active-override gate. The three carry +the same set, which is why a model's device state never survives into the next. + +### 2.2p The Q8 requant writers store one quant per byte {#q8-requant-byte-store} + +Every requant writer on the class rail - the prefill and decode-tail kernels that write Q8_0 or +Q8_K quants - declares its output plane `array` and stores one quant per element, over +SPIR-V's 8-bit storage path; the fused decode step `DnStepFused` keeps its packed-word head +requant, the one writer outside this rule. Packing four quants into a `uint` +instead costs a shift-and-or chain per word, and in a Q8_K writer - where four co-active lanes +each hold one byte of the word - two subgroup shuffles per element on top. The stored bytes are +the same under either form: the amax fold, the scale and the rounding decide them, and all +three sit above the store. The path needs the device's 8/16-bit storage feature set, which the +family's device creator enables. diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index 7a6ff9fbca..0d0461a726 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -2,7 +2,7 @@ Companion to `ARCHITECTURE.md`; section numbers are that document's. -### 2.5 There is ONE benchmark rig, and the records are the baseline +### 2.5 There is ONE benchmark rig, and the records are the baseline {#one-benchmark-rig} `benchmarks/lcpp_bench.das` is the only thing that measures performance. It is a *mirror* of the upstream `llama-bench` - the same test shapes, rep counts and timing boundaries, applied to @@ -28,6 +28,14 @@ harness would produce numbers that cannot be compared to any of this. an untuned invocation re-execs into a full retune rather than measuring - so re-mint the box manifest and check its winners against the stored rows' `tune` stamps before trusting a delta. +**The Vulkan GEMM probe attributes prefill GEMM cost on three axes.** +`harness/vk_gemm_probe.das` times one shape at a time: the serving GEMM against its alternates +on the dense role shapes (gate/up, down, q/wo, k/v - the mm_a kernel against the cm2 l and m +tiles, the sdot4 kq tile against the k4 and k6 cm2 tiles); one decode callback against +spellings of itself with the rest of the tile held fixed (the `cm2x` and `k6x` bisect arms); +and our tile against the upstream coopmat2 GEMM blob, served in place of a probe class's body +through `DASLLAMA_VK_SPV_OVERRIDE` (the `ref` arm). A new arm joins one of the three. + **A measured number proves its kernel provenance through `tune_gate()` (`performance/profile_common.das`), one arm per world it can run in.** Three worlds, because `tune_status()` populates in exactly one of them: a standalone exe checks the sidecar the @@ -37,7 +45,10 @@ worse, measures on fallback kernels - which is why every measuring entry point c before its first timed rep. Two rig shapes fall outside "measuring entry point" by the property itself, ledgered here: a kernel A/B lab dispatches its variants through its own arms (never the `[tune]` selection), and `lcpp_bench.das`'s `--tok` cell dispatches no kernels at -all - neither can measure a fallback silently. +all - neither can measure a fallback silently. A kernel A/B lab is also outside the +in-process reference check: `harness/vk_gemm_probe.das` dispatches the shipped, suite-gated +kernels on timing fixtures, compares no arm's output, and marks every row `timing-only`; its +rows never enter a record store, and a decision it seeds is confirmed by the e2e board rows. **The retune re-exec bites scaffolding, and the pin for it is checked in.** Any bare `daslang` run that requires the engine - a probe, a one-off script, a REPL experiment - re-execs into a diff --git a/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md b/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md index ad05fb39fb..eb10eb6e45 100644 --- a/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md +++ b/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md @@ -75,7 +75,7 @@ that runs before the window is staged must ask the capability half only, or it g forever and its feature silently never runs. Split such predicates rather than reordering the caller; an optimistic capability answer is safe when the late path has a fallback, and here it does. -### 2.7 A quantized activation carries its scale lattice (Vulkan) +### 2.7 A quantized activation carries its scale lattice (Vulkan) {#activation-scale-lattice} Two activation quant forms ride the vulkan rail, and they differ in the SCALE LATTICE, not the int8 payload: the Q8_0 form scales per 32 values, the superblock form per 256 (with per-32 @@ -97,7 +97,10 @@ Three consequences the code is shaped around: shape must ride the SAME gate, or profiles desync from what actually dispatched. - **A GEMV group sharing one activation buffer must be lattice-homogeneous.** q/k/v share one quantized x; gate/up share another. Resident arming classifies each member's consumer form - and DECLINES a mixed group rather than serving one member wrong scales. + and DECLINES a mixed group rather than serving one member wrong scales. The prefill f16 feed + answers the same question one step further: one activation buffer serves every GEMM of a + group, so the f16 (cm2 decode-in-load) form engages only when EVERY member of the group is + cm2-servable - one member on the quant route pins its whole group to the quant feed. ### 2.8 Every program root declares its stack budget and its prefill intent @@ -121,7 +124,7 @@ ended on. The guard panics, and a panic takes every live stream down, so an unde a serving outage waiting on its first long prompt. Both halves of root discipline are enforced by `tests/test_program_roots.das`. -### 2.9 Environment knobs +### 2.9 Environment knobs {#env-knobs} A knob is an `[EnvConfig]` field in `dasllama_env.das`, read as `g_env_*.`; the field is also what generates its `ENVIRONMENT.md` row, so a knob declared anywhere else is invisible to diff --git a/modules/dasLLAMA/ENVIRONMENT.md b/modules/dasLLAMA/ENVIRONMENT.md index 00e3945ad6..6a72cf9198 100644 --- a/modules/dasLLAMA/ENVIRONMENT.md +++ b/modules/dasLLAMA/ENVIRONMENT.md @@ -109,20 +109,25 @@ Vulkan GPU backend. Present only where the dasVulkan package is installed. | Variable | Type | Default | Effect | |---|---|---|---| -| `DASLLAMA_COOPMAT` | text | auto | Cooperative-matrix mode; the flash-attention twin needs it even when the GEMM runs sdot4. | +| `DASLLAMA_COOPMAT` | text | auto | Cooperative-matrix mode (auto = cm2 where the device has NV_cooperative_matrix2, else mm, else sdot4); the flash-attention twin needs it even when the GEMM runs sdot4. | | `DASLLAMA_MM_SMALL` | text | 32 | Small-batch tier: 32 = sdot4 (default, beats both coopmat tiles below the crossover), 64 = coopmat M, 128 = always-L. | | `DASLLAMA_MM_SMALLD` | number | 64 | Small-d cutoff routing narrow roles (k/v) to the small tier; widening measured worse, so this is an instrument. | -| `DASLLAMA_VK_FUSE` | flag | on | Fused decode tail (add+rms+requant, qk-norm+rope); 0 pins the split dispatches for a same-build A/B. | +| `DASLLAMA_VK_FUSE` | flag | on | Fused add+rms+requant: the decode tail (plus qk-norm+rope) AND the prefill batch ar+rq pairs; 0 pins the split dispatches for a same-build A/B. | | `DASLLAMA_VK_XFERQ` | flag | on | Stream expert uploads on the dedicated transfer queue, overlapped via a timeline semaphore; 0 keeps the single-queue rail. | | `DASLLAMA_VK_IMPORT` | flag | on | Stream mirrors import the mapped .dlim (VK_EXT_external_memory_host) instead of pinned copies; =0 restores the copy path. | | `DASLLAMA_TRIM` | flag | off | Serve from P3-trimmed vulkan images (big CPU weight families dropped; folded into the flavor identity). | | `DASLLAMA_VK_MEMPRIO` | flag | on | Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours. | | `DASLLAMA_VK_FA` | flag | on | Vulkan flash attention: the decode fa kernel pick AND the cm2 prefill fa tile; 0 falls back to the chunked/scalar paths. | +| `DASLLAMA_VK_KV_MERGE` | flag | on | Merged k|v prefill GEMM - one dispatch over the adjacent k+v arena planes; 0 pins the split k + v dispatches for a same-build A/B. | +| `DASLLAMA_VK_OVERLAP` | flag | on | Prefill record/execute overlap: the window chain submits in ramped chunks (1,2,4,8 layers) so the GPU starts while the CPU still records; 0 pins the single fenced submit (the per-role GPU profile pins it too, so a chunk gap never bills to a role). | +| `DASLLAMA_VK_GPU_EMBED` | flag | on | Device-side token-embedding gather for the resident prefill (a tied q8 table gathers from the resident cls plane; a raw f32 table uploads whole under a 512 MB cap); 0 keeps the CPU embed loop. | +| `DASLLAMA_VK_FULLSG` | flag | off | Pin REQUIRE_FULL_SUBGROUPS on every class pipeline (instrument; measured slower than plain pipelines on the mm_a gate shape, so those are the default). | | `DASLLAMA_VK_REBAR` | flag | on | Use a ReBAR device-local host-visible heap when one larger than 1GB is present. | | `DASLLAMA_VK_KV32` | number | 0 | Arm the resident driver with f32 KV mirrors instead of the f16 default (A/B instrument; only sessions of the armed codec are served). | | `DASLLAMA_CM2_TILE` | number | 0 | cm2 prefill tile pick: 0 = occupancy heuristic, 128 = force the m tile, 256 = force the l tile (A/B instrument). | | `DASLLAMA_CM2_SPLITK` | number | 0 | cm2 split-k: 0 = occupancy heuristic, 1 = off, N = force N k-chunks (A/B instrument; shrinks if N strands an empty tail). | | `DASLLAMA_VK_SPV_OVERRIDE` | path | unset | Directory of .spv files served instead of the emitted words at pipeline creation (offline spirv-opt / hand-patched A/B instrument). | +| `DASLLAMA_VK_SPV_DUMP` | path | unset | Directory to write each kernel's emitted words as .spv at pipeline creation (the override instrument's capture half). | | `DASLLAMA_VK_HAZARD_PARANOID` | flag | off | Barrier at every dispatch (correctness bisect). | | `DASLLAMA_VK_HAZARD_TRACE` | flag | off | Log every detected hazard and the barrier it produced. | diff --git a/modules/dasLLAMA/README.md b/modules/dasLLAMA/README.md index 37ad6a535e..ec4076d652 100644 --- a/modules/dasLLAMA/README.md +++ b/modules/dasLLAMA/README.md @@ -99,6 +99,8 @@ modules/dasLLAMA/ ARCHITECTURE.md # what-goes-where ledger — which module owns which concern ARCHITECTURE_ENGINE.md # companion: the engine, format, load, CPU-tier, support and serving charters ARCHITECTURE_GPU.md # companion: the GPU backend role table, the backend asymmetries, the refuted kernel shapes + ARCHITECTURE_GPU_PREFILL.md # companion: the Metal prefill driver's GEMM ladder + ARCHITECTURE_GPU_VULKAN.md # companion: the Vulkan resident driver - window chain, cm2 tiles, residency plan, slot marks ARCHITECTURE_MEDIA.md # companion: the encoder-tower, audio, ASR and vision charters ARCHITECTURE_IMAGE.md # companion: the prepared-image (.dlim) rail ARCHITECTURE_RUNTIME.md # companion: kernel shape, caches, knobs, coverage, the GPU ramp diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 898e0f11c3..2692025ce4 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -9,7 +9,7 @@ ledger, everything else to the followup ledgers). `tests/REVIEW.md`.** **A timing rig - a script whose output is a measured wall or rate - wherever the diff puts -it, answers to `benchmarks/REVIEW.md`.** +it, answers to this folder's `benchmarks/REVIEW.md`.** **A diff that writes a measured number down - into `PERF_LEDGER.md`, a checked-in doc, a code comment, or a PR body - or adds a servable capability applies `REVIEW_MEASUREMENT.md`.** @@ -23,7 +23,7 @@ fixture. A test or tool merely opening a stocked model file by name does not rou sidecar exchange is the code that downloads tune winners to a box and submits that box's winners back. -**Every `dasllama/` change applies `tests/REVIEW.md`.** +**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** **A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** @@ -119,22 +119,24 @@ the name in the same change). **A change to code or data of `encode`/`bpe_encode` or anything they reach in `dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das` / `dasllama/dasllama_pretok.das` -ships before/after `--tok` rows (`benchmarks/lcpp_bench.das`) for the affected backend** - the +ships before/after `--tok` rows (this folder's `benchmarks/lcpp_bench.das`) for the affected backend** - the instrument is the scaling ratio across the size ladder, and superlinear is a defect. **A change to code or data in `dasllama/dasllama_tokenizer.das`, `dasllama/dasllama_spm.das`, `dasllama/dasllama_bpe.das`, or `dasllama/dasllama_pretok.das`, or to the special-token or -template strings any of them look up, records a `tests/test_tokenizer.das` run with its cases +template strings any of them look up, records a run of this folder's `tests/test_tokenizer.das` with its cases EXECUTED, not skipped.** **A diff that adds an override, or gives one a new effect, without the announce is a defect.** An override is an environment knob, an exported runtime setter, or an on-disk state file that moves a gate, policy, or threshold off its default and thereby changes what the run writes, reads, mints, or computes - a timing knob included when it moves computed numerics. A knob -that changes only WHEN work happens is not one, and a CLI flag is never one. The announce is -a line the run prints where the override changes the outcome, naming it by the spelling a -user would set - the environment variable name, the sidecar or file key, or the setter's -function name. Per-site repeats are fine; a set-but-inert override stays silent. +that changes only WHEN work happens is not one, and a CLI flag is never one. A default-ON knob +announces on the default path, naming the spelling that turns it off; a default-OFF knob +announces when it is set. The announce is a line the run prints where the override changes the +outcome, naming it by the spelling a user would set - the environment variable name, the +sidecar or file key, or the setter's function name. Per-site repeats are fine; a set-but-inert +override stays silent. **A change to user-facing API updates every place it is shown: a tutorial source, `.rst` page, docstring, help string, `README.md`, or checked-in document still showing the old call, flag, or @@ -150,9 +152,10 @@ path under `modules/dasLLAMA/` - is a defect:** a module added to its allowed se match dropped or narrowed, or an error text that no longer names the facade to require instead. The allowed set is the table in the lint. -**A `// nolint:STYLE037` or `// nolint:STYLE038` on a function a follow-up ledger entry calls -reducible is a defect - land the ledgered split instead.** The warning is what keeps the ledger -entry visible. +**A `// nolint:STYLE037` or `// nolint:STYLE038` on a function a follow-up ledger entry says +can be shortened or split is a defect - land the ledgered split instead.** The warning is what +keeps the ledger entry visible. A ledger entry asking for a dedup across bodies - one template +for several twins - does not fire this rule: the one body left still carries its length. **`options _dasllama_internal` belongs only in a file whose job is to reach engine internals: an engine file under `dasllama/`, a test, harness, benchmark, or rig this module @@ -176,7 +179,8 @@ is one that check does not flag. When the check licenses no names, the line says file or a `.das` comment; state what the code does and why its shape wins. Provenance is not attribution: a path naming where checked-in data is regenerated FROM, an env-knob row in `ENVIRONMENT.md` whose value locates the reference binary, and a command line or flag -list in `METHODOLOGY.md`, `PROFILE.md`, or `BRINGUP.md`, all name the binary outright; +list in `METHODOLOGY.md`, `PROFILE.md`, or `BRINGUP.md`, and a follow-up ledger's board row +naming the build it compares against, all name the binary outright; every other `.md` line and `.das` comment writes "the reference exe" or "upstream". Legal attribution lives in `THIRD_PARTY_NOTICES.md` and the `LICENSE.*` files, so prose never carries it. @@ -238,8 +242,8 @@ files. A doc comment naming the family a helper was built for is fine. file** - a single-caller helper sanctioned as tower-worthy is ledgered on `ARCHITECTURE_MEDIA.md` sec.1.7's tower charter line, not argued in review. -**A harness that prints output for another tool to compare exits non-zero when its run ends -without those comparison lines - wrong flags, failed load.** +**A harness whose run can end with zero result rows exits non-zero when it does - wrong +flags, failed load, a device that declines.** **Tool wire text (the text of a model's tool/function call, built or parsed) is produced only in `dasllama/dasllama_tools.das`.** diff --git a/modules/dasLLAMA/REVIEW_GPU.md b/modules/dasLLAMA/REVIEW_GPU.md index 75b5c4eed4..71b1a11379 100644 --- a/modules/dasLLAMA/REVIEW_GPU.md +++ b/modules/dasLLAMA/REVIEW_GPU.md @@ -1,7 +1,7 @@ # dasLLAMA GPU Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_GPU_PREFILL.md`. +docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_GPU_PREFILL.md`, `ARCHITECTURE_GPU_VULKAN.md`. **Routed from `REVIEW.md`: a diff touching a GPU kernel, driver, dispatch class, or the K/V mirrors applies this list together with `REVIEW.md`.** @@ -85,10 +85,12 @@ ladder. The small-work regression hides behind the big-work win. **A diff that changes a tile, grid, threadgroup, or uniform constant shows the value at that constant's authoritative site, in the same change.** An in-body tile constant is confirmed -literal in the generated `*_msl` global or the SPIR-V dump. A grid or threadgroup constant is -read off the class's `[metal_dispatch]` / `[vk_dispatch]` `grid=`/`tg=` spec, whose `"n/c"` -form is a CEIL-divide; the spec alone decides. A uniform's value is read at the single writer -that fills its buffer. +literal in the generated `*_msl` global or the SPIR-V dump (`DASLLAMA_VK_SPV_DUMP=` +writes every class kernel's words). A grid constant is read off the +class's `[metal_dispatch]` / `[vk_dispatch]` `grid=` spec, whose `"n/c"` form is a +CEIL-divide; a threadgroup constant off Metal's `tg=` spec or Vulkan's +`[spirv_kernel(local_size_x=)]`; the spec alone decides. A uniform's value is read at the +single writer that fills its buffer. **A kernel twin that binds a different kargs (kernel-argument struct) type than its sibling twin, or shifts a shared field to a different binding number, is a defect - even where one @@ -115,9 +117,10 @@ an `upload_region` upload never written after arming - is a defect unless it car defect; a per-encode field either omits `@role` or names the access its body performs.** `weight` drops the hazard staging. -**A new kernel class carries `[metal_dispatch]` / `[vk_dispatch]` with every annotation the -generated builder reads - per-field `@binding` / `@role` / `@off` / `@default`, `@workgroup` -state with its `tgmem=` dispatch key.** +**A new kernel class carries `[metal_dispatch]` / `[vk_dispatch]` with every annotation that +backend's generated builder reads - per-field `@binding` / `@role` / `@off` / `@default`, and +`@workgroup` state with its `tgmem=` dispatch key.** A field carrying none of them is dropped +from the bind list with no error. **A kernel field carries `@span` only when every caller binds whole output rows.** A caller binding a column tile of a wider row passes the tile width as the kernel's n while its rows @@ -182,15 +185,13 @@ is created only by a `[vk_dispatch]`-generated `ensure_*` and torn down by bind site cannot shrink a buffer that was sized wrong. **A change to code that a served GPU decode or prefill path executes ships GPU-vs-CPU parity -on one q8 and one kq (K-quant) model with the armed mirror codec.** That code is a driver -(`dasllama/dasllama_metal_decode.das`, `dasllama/dasllama_metal_prefill.das`, -`dasllama/dasllama_vulkan_decode.das`, `dasllama/dasllama_vulkan_prefill.das`), a kernel -class one of them dispatches, that class's builder, the servability gates -(`dasllama/dasllama_metal_shapes.das`), the weight-region cache and residency paths -(`dasllama/dasllama_metal_common.das`), or the residency rail's serving paths -(`dasllama/dasllama_gpu_resident.das`); never the bake paths, never a comment. The parity run -is `harness/parity.das`, or the in-suite instruments `tests/test_metal_decode_parity.das` / -`tests/test_metal_prefill_parity.das` through `tests/run.das`. +on one q8 and one kq (K-quant) model with the armed mirror codec.** That code is anything a +served GPU decode or prefill call executes - a driver, a kernel class it dispatches, that +class's builder, a servability gate, a weight-region or residency path, the tier forwarders +and engine seams the call routes through; never the bake paths, never a comment. The parity +run is `harness/parity.das` on either backend, or - on Metal only - the in-suite instruments +`tests/test_metal_decode_parity.das` / `tests/test_metal_prefill_parity.das` through +`tests/run.das`. **Parity evidence counts only when its backend was armed: the Metal arm ran with `--ngl`; the Vulkan arm ran with `DASLLAMA_GPU=1` - never `--ngl` - and its log shows `resident driver @@ -242,8 +243,8 @@ same-codec session rows and mirror rows.** A cross-codec copy corrupts the host' cache. **Never cache a descriptor set across dispatches in state that `vk_drop_model_state` does not -clear** - put it in a `*_ready` latch, or in a field inside `g_gpu` or the weight arena in -`dasllama/dasllama_vulkan_common.das`. +clear** - put it in a `*_ready` latch, or in a holder that function already clears in +`dasllama/dasllama_vulkan_common.das`: `g_rd`, `g_gpu`, the weight arena. **A diff that changes anything a hand-binding arm must mirror to dispatch a kernel - binding numbers, kargs layout, threadgroup memory, grid or threadgroup geometry - fixes or deletes, @@ -258,3 +259,22 @@ the lab exists only for that decision, its driver and remaining arm go too.** An timing script whose output SELECTS between implementations of the same compute, wherever it lives (`benchmarks/`, `harness/`); a decided arm that outlives its decision degrades into an unmaintained duplicate of the kernel it seeded. + +**Never read a `[spirv_decode]` callback's quant bytes by indexing `unpack8` of a 32-bit word +with a runtime value - read them as 16-bit lanes instead: an `int16[N]` block member selected +with `unpack8(w)[i & 1u]`, sub-fields pulled out by shift and mask.** The vendor driver's shader +compiler pattern-matches only the 16-bit spelling into its block-load path, and a runtime byte +select drops the whole kernel off it. + +**A diff that changes when the resident prefill that takes token ids rather than embeddings +(`vk_rdec_prefill_ids` and the resident prefill override that routes to it) accepts a call +changes the engine's GPU-embed probe - the gate registered through `register_embed_gpu_gate`, +`vulkan_embed_gpu_gate` in `dasllama/dasllama_gpu_resident.das` - in the same change.** The +engine skips the CPU embed on a true probe, so a probe that is true where that path declines +hands the next consumer an unfilled residual stream. + +**A diff that adds a module-level variable to `dasllama/dasllama_gpu_resident.das` whose value +depends on the installed model also adds it to `moe_gpu_model_marks_save_`, +`moe_gpu_model_marks_restore_` and `moe_gpu_drop_model_`, in the same change.** A global +missing from one of the three survives a model swap and routes the next model's dispatches at +the old model's planes. diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index 6c36663ea0..884b4e06aa 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -2,7 +2,8 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `../ARCHITECTURE_MEASUREMENT.md` (the benchmark rig is sec.2.5). Planned work: -`../PERF_LEDGER.md` for a performance followup, `../followup_general.md` for everything else. +`../followup_vulkan.md` for anything about the Vulkan backend, `../PERF_LEDGER.md` for any +other performance followup, `../followup_general.md` for everything else. **A diff that adds or changes an instrument whose timed body runs a forward pass through a pipeline the model runtime selected also calls `tune_gate()` @@ -43,6 +44,7 @@ reader takes the sweep's arms for an adoption decision it never made. `../performance/gen_bench_records.das`, or a `lcpp_bench.das` cell with its own `../PROFILE.md` section, instead.** A served turn is a whole prefill-plus-decode run. A second instrument's numbers cannot be compared to any row the board already carries. + **An out-of-process observer never measures what the benchmark process can measure about itself - that measurement goes inside the process instead.** An out-of-process observer is a script that measures a benchmark process from outside. diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 416169af2e..49ea2a577f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1502,6 +1502,7 @@ def ffn_moe_prefill(t : Model; var s : Session; l : int64; npos : int64) { } } +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] def forward_prefill_embd(t : Model; var s : Session; embd : array; npos, start_pos : int64; non_causal : bool = false; src_off : int64 = 0l; skip_logits : bool = false; span_lo : int64 = -1l; span_hi : int64 = -1l; ds_wide : bool = false) { @@ -1535,8 +1536,6 @@ def forward_prefill_embd(t : Model; var s : Session; embd : array; npos, let dim = t.config.dim let ts_embed = prof_ticks() if (ds_wide) { - // arm the wide BORROW instead of splitting: a wide-capable override uploads the whole - // quantum and extracts slices on the GPU; the CPU fallback splits via ds_split_quantum unsafe { s.wide_src = addr < float const? >(embd[0]) } @@ -1567,6 +1566,7 @@ def forward_prefill_embd(t : Model; var s : Session; embd : array; npos, s.attn_uniform_lo = 0l s.attn_uniform_end = 0l } + s.embed_gpu_pending = false // this entry filled x_b itself - no stash for the body to backfill forward_prefill_body(t, s, npos, start_pos, skip_logits) s.attn_uniform_lo = 0l s.attn_uniform_end = 0l diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 9a52c6ce11..a387b3299c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -280,6 +280,16 @@ def register_ple_gpu_gate(fn : PleGpuGateFn) { g_ple_gpu_gate_set = true } +typedef EmbedGpuGateFn = function<(t : Model; s : Session; npos : int64; start_pos : int64) : bool> +var private g_embed_gpu_gate : EmbedGpuGateFn +var private g_embed_gpu_gate_set = false + +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def register_embed_gpu_gate(fn : EmbedGpuGateFn) { + g_embed_gpu_gate = fn + g_embed_gpu_gate_set = true +} + // ===== gemma-4 PLE pre-step hooks (registered by dasllama_ple at [init]) ===== // PLE needs Model/Session, so it requires this module back and cannot be required from here — the // forward path reaches its per-pass pre-step through these hooks, and the umbrella breaks the cycle. @@ -1803,6 +1813,8 @@ struct Session { @scratch ple_gate : array // per-(position) gate vector for the layer block (npos x ple) @scratch ple_gpu_tokens : array // stashed token rows for a driver-owned pre-step (see register_ple_gpu_gate) ple_gpu_pending : bool // forward_prefill skipped the CPU pre-step — the engaged driver builds it + @scratch embed_gpu_tokens : array + embed_gpu_pending : bool // qwen35 deltanet recurrent state (persists across tokens; zeroed at position 0) + scratch. // Empty unless config.recr_mask != 0. State is forward-only — dn_pos guards against rewind. dn_pos : int64 // the next position the deltanet stack expects @@ -4714,7 +4726,7 @@ def attn_out_suffix(t : Model; var s : Session; l : int64; npos : int64) { // The GPU arm shared by the std and gated prefill blocks: the whole attention block (GEMMs, // qk-norm, rope, flash-style attention, wo) as one device chain per window; norm, activation // image, KV-store, and residual stay CPU. Numerics: GPU f32 tile accumulation — drift-class vs CPU flash. -[no_alloc, no_env, no_io] +[no_alloc, no_env, no_io, arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] def forward_prefill(t : Model; var s : Session; tokens : array; npos, start_pos : int64; skip_logits : bool = false) { let c = t.config if (npos <= 0l) { @@ -4734,11 +4746,20 @@ def forward_prefill(t : Model; var s : Session; tokens : array; npos, sta // token embeddings -> residual stream x_b [npos x dim] let dim = c.dim let ts_embed = prof_ticks() - for (p in range64(npos)) { - embed_row(t, tokens[p], unsafe(addr(s.x_b[p * dim]))) - } - if (c.embed_scale != 1.0) { // Gemma2: scale the embedding by sqrt(dim) - scale_inplace(unsafe(addr(s.x_b[0])), c.embed_scale, npos * dim) + if (g_embed_gpu_gate_set && invoke(g_embed_gpu_gate, t, s, npos, start_pos)) { + s.embed_gpu_tokens |> resize(npos) + for (p in range64(npos)) { + s.embed_gpu_tokens[p] = tokens[p] + } + s.embed_gpu_pending = true + } else { + s.embed_gpu_pending = false + for (p in range64(npos)) { + embed_row(t, tokens[p], unsafe(addr(s.x_b[p * dim]))) + } + if (c.embed_scale != 1.0) { // Gemma2: scale the embedding by sqrt(dim) + scale_inplace(unsafe(addr(s.x_b[0])), c.embed_scale, npos * dim) + } } prof_add("embed", ts_embed) if (has_ple(c)) { // gemma-4 E-series: build each position's per-(layer) side input before the stack @@ -4957,16 +4978,30 @@ def private prefill_rope_tables(t : Model; var s : Session; npos, start_pos : in prof_add("rope_build", ts_rope_build) // one-time cos/sin table build (separate bucket) } +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def private embed_backfill_cpu(t : Model; var s : Session; npos : int64) { + let ts_embed = prof_ticks() + let dim = t.config.dim + for (p in range64(npos)) { + embed_row(t, s.embed_gpu_tokens[p], unsafe(addr(s.x_b[p * dim]))) + } + if (t.config.embed_scale != 1.0) { + scale_inplace(unsafe(addr(s.x_b[0])), t.config.embed_scale, npos * dim) + } + s.embed_gpu_pending = false + prof_add("embed", ts_embed) +} + // the shared prefill core: rope table + layer stack + final norm/classifier, over a pre-filled // s.x_b residual stream (token or embedding sourced) +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] def forward_prefill_body(t : Model; var s : Session; npos, start_pos : int64; skip_logits : bool = false) { let c = t.config let dim = c.dim prefill_rope_tables(t, s, npos, start_pos) // an active whole-prefill override may claim the entire stack (false = decline, fall back - // to the CPU loop); a claimer must honor s.attn_uniform_end (metal: AttnArgs.uend). An - // mrope quantum serves only on a per-row-table roper (metal); others decline it loudly. + // to the CPU loop); a claimer must honor s.attn_uniform_end (metal: AttnArgs.uend) var stack_done = false g_prefill_logits_done = false if (!empty(g_prefill_override_name) && !empty(s.mrope_pos) && !active_prefill_serves_mrope()) { @@ -4977,6 +5012,9 @@ def forward_prefill_body(t : Model; var s : Session; npos, start_pos : int64; sk stack_done = invoke(g_prefill_override, t, s, npos, start_pos) } if (!stack_done) { + if (s.embed_gpu_pending) { + embed_backfill_cpu(t, s, npos) + } if (s.ds_active && s.wide_src != null) { // the wide borrow was for an override that declined let ts_split = prof_ticks() ds_split_quantum(t, s, npos) diff --git a/modules/dasLLAMA/dasllama/dasllama_env.das b/modules/dasLLAMA/dasllama/dasllama_env.das index e6ed7dab00..dbb336b859 100644 --- a/modules/dasLLAMA/dasllama/dasllama_env.das +++ b/modules/dasLLAMA/dasllama/dasllama_env.das @@ -280,10 +280,10 @@ struct public MetalEnv { // ===== Vulkan backend ===== -[EnvConfig(env_prefix = "DASLLAMA")] +[EnvConfig(env_prefix = "DASLLAMA"), arch(at="../ARCHITECTURE_RUNTIME.md#env-knobs")] struct public VulkanEnv { @clarg_default_doc = "auto" - @clarg_doc = "Cooperative-matrix mode; the flash-attention twin needs it even when the GEMM runs sdot4." + @clarg_doc = "Cooperative-matrix mode (auto = cm2 where the device has NV_cooperative_matrix2, else mm, else sdot4); the flash-attention twin needs it even when the GEMM runs sdot4." coopmat : string = "" @clarg_default_doc = "32" @@ -293,7 +293,7 @@ struct public VulkanEnv { @clarg_doc = "Small-d cutoff routing narrow roles (k/v) to the small tier; widening measured worse, so this is an instrument." mm_smalld : int64 = 64l - @clarg_doc = "Fused decode tail (add+rms+requant, qk-norm+rope); 0 pins the split dispatches for a same-build A/B." + @clarg_doc = "Fused add+rms+requant: the decode tail (plus qk-norm+rope) AND the prefill batch ar+rq pairs; 0 pins the split dispatches for a same-build A/B." vk_fuse : bool = true @clarg_doc = "Stream expert uploads on the dedicated transfer queue, overlapped via a timeline semaphore; 0 keeps the single-queue rail." @@ -311,6 +311,18 @@ struct public VulkanEnv { @clarg_doc = "Vulkan flash attention: the decode fa kernel pick AND the cm2 prefill fa tile; 0 falls back to the chunked/scalar paths." vk_fa : bool = true + @clarg_doc = "Merged k|v prefill GEMM - one dispatch over the adjacent k+v arena planes; 0 pins the split k + v dispatches for a same-build A/B." + vk_kv_merge : bool = true + + @clarg_doc = "Prefill record/execute overlap: the window chain submits in ramped chunks (1,2,4,8 layers) so the GPU starts while the CPU still records; 0 pins the single fenced submit (the per-role GPU profile pins it too, so a chunk gap never bills to a role)." + vk_overlap : bool = true + + @clarg_doc = "Device-side token-embedding gather for the resident prefill (a tied q8 table gathers from the resident cls plane; a raw f32 table uploads whole under a 512 MB cap); 0 keeps the CPU embed loop." + vk_gpu_embed : bool = true + + @clarg_doc = "Pin REQUIRE_FULL_SUBGROUPS on every class pipeline (instrument; measured slower than plain pipelines on the mm_a gate shape, so those are the default)." + vk_fullsg : bool = false + @clarg_doc = "Use a ReBAR device-local host-visible heap when one larger than 1GB is present." vk_rebar : bool = true @@ -327,6 +339,10 @@ struct public VulkanEnv { @clarg_doc = "Directory of .spv files served instead of the emitted words at pipeline creation (offline spirv-opt / hand-patched A/B instrument)." vk_spv_override : string = "" + @clarg_path + @clarg_doc = "Directory to write each kernel's emitted words as .spv at pipeline creation (the override instrument's capture half)." + vk_spv_dump : string = "" + @clarg_doc = "Barrier at every dispatch (correctness bisect)." vk_hazard_paranoid : bool = false diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index a8899fb9d6..4feb33ad48 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -351,6 +351,7 @@ struct ResidentPlan { scratch_bytes : int64 //!< the driver's own device scratch (prefill window + decode smalls) headroom_bytes : int64 //!< desktop slack the auto arm leaves unfilled (0 when VRAM is pinned) budget_bytes : int64 + emb_f32_bytes : int64 //!< the raw f32 embd table the gather would upload - counted only when it rides the headroom left after weights + KV (0 = the CPU embed serves) fits : bool reason : string //!< "" when it fits; otherwise why, with a remedy where one exists } @@ -363,6 +364,7 @@ let private RDEC_MISC_BYTES = 67_108_864l // decode smalls + batch act // the driver's device allocations past weights + KV — the prefill window buffers dominate. // An estimate, deliberately a touch high; the flat term carries the decode/batch smalls. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#resident-plan")] def private rdec_scratch_bytes(t : Model) : int64 { let c = t.config let qd = layer_qd(c, 0l) @@ -372,8 +374,7 @@ def private rdec_scratch_bytes(t : Model) : int64 { + 8l * hid + 4l * layer_head_size(c, 0l)) + c.vocab_size * 4l + RDEC_MISC_BYTES } -//! Size `t` for the resident driver at `seq_cap` context. Declines (fits = false) carry a reason: -//! the commonest support question on this tier is "why did it decline", so the answer is the API. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#resident-plan")] def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan { let c = t.config var p = ResidentPlan(budget_bytes = moe_gpu_weight_budget_()) @@ -385,8 +386,6 @@ def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan p.reason = "MoE - the per-op offload tier serves routed experts, not the resident driver" return p } - // KV is reserved BEFORE weights and never grows: on a discrete card it competes directly with - // them, and evicting weights to grow it would mean re-uploading gigabytes. var krow = 0l var vrow = 0l for (l in range64(c.n_layers)) { @@ -424,6 +423,10 @@ def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan p.weight_bytes += moe_gpu_plane_bytes(c.dim, c.vocab_size, int(cls_fmt)) let total = p.weight_bytes + p.kv_bytes + p.scratch_bytes let usable = p.budget_bytes - p.headroom_bytes + let emb_want = g_env_vulkan.vk_gpu_embed && rdec_prefill_ids_installed() && !t.cls_q8 && !cls_kq(t) && !t.planes_trimmed + let emb_bytes = emb_want ? rdec_emb_f32_bytes(c.vocab_size, c.dim) : 0l + p.emb_f32_bytes = total + emb_bytes <= usable ? emb_bytes : 0l + p.weight_bytes += p.emb_f32_bytes p.fits = total <= usable if (!p.fits) { // a shorter context is the remedy that actually works — weights are fixed, KV is not @@ -439,6 +442,7 @@ def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan // ===== the resident whole-stack decode driver (dense models that fully fit) ===== var private g_rdec_active = false // this loaded model runs the resident decode driver +var private g_rdec_emb_gpu = false var private g_rdec_cnt = 0l // cached positions in the device KV mirror (built as we decode) var private g_rdec_cap = 0l // mirror context the driver armed at (<= model ctx when VRAM-capped) var private g_rdec_gen = 0l // mirror ownership generation (bumped per resident prefill) @@ -512,8 +516,10 @@ def private resident_layer_fmts(t : Model; l : int64) : tuple 0l) { return false @@ -555,6 +561,11 @@ def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype) : bool { // no } // tally arena blocks per format var need : table + let emb_q8_want = (g_env_vulkan.vk_gpu_embed && rdec_prefill_ids_installed() + && c.shared_weights && t.cls_q8 && cls_fmt == KqFmt.q8) + let emb_f32_want = (g_env_vulkan.vk_gpu_embed && rdec_prefill_ids_installed() + && !t.cls_q8 && !cls_kq(t) && !t.planes_trimmed + && resident_plan(t, seq_cap, rkdt, rkdt).emb_f32_bytes > 0l) // the plan counted it at this context var planes : array> planes |> reserve(c.n_layers * 7l + 1l) for (l in range64(c.n_layers)) { @@ -630,6 +641,24 @@ def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype) : bool { // no return false } rdec_set_cls(bcls, int(cls_fmt)) + var emb_q8_placed = false + var emb_f32_placed = false + if (emb_q8_want) { + emb_q8_placed = rdec_set_emb(bcls) + if (emb_q8_placed) { + to_log(LOG_INFO, "dasLLAMA: resident driver - embed gather on device (tied cls plane; DASLLAMA_VK_GPU_EMBED=0 keeps the CPU embed)\n") + } else { + to_log(LOG_WARNING, "dasLLAMA: resident driver - the embed-gather rail declined, the CPU embed serves\n") + } + } elif (emb_f32_want) { + emb_f32_placed = rdec_upload_emb_f32(t.fblob, t.tok_emb_off, c.vocab_size, dim) + if (emb_f32_placed) { + to_log(LOG_INFO, "dasLLAMA: resident driver - embed gather on device (f32 plane; DASLLAMA_VK_GPU_EMBED=0 keeps the CPU embed)\n") + } else { + to_log(LOG_WARNING, "dasLLAMA: resident driver - the f32 embed table stays on the CPU (over the cap, the budget, or the device range)\n") + } + } + g_rdec_emb_gpu = emb_q8_placed || emb_f32_placed // upload the norm rows: [rms_att[l], rms_ffn[l]] x L, then rms_final; qk_norm appends the // per-head [rms_q[l], rms_k[l]] x L rows (hs floats each) past the fixed layout var norms : array @@ -722,7 +751,7 @@ def private vulkan_resident_decode(t : Model; var s : Session; token, pos : int6 } // the "vulkan" whole-prefill override — host KV hydrates back on demand, so the CPU rails always take over from a valid authority -[hot_path] // per prefill call +[hot_path, arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] // per prefill call def private vulkan_resident_prefill(t : Model; var s : Session; npos, start_pos : int64) : bool { // armed + embedder-allowed + flat cache in the ARMED mirror codec + causal (this path encodes the causal mask) if (!g_rdec_active || !g_resident_prefill_allowed @@ -749,7 +778,12 @@ def private vulkan_resident_prefill(t : Model; var s : Session; npos, start_pos for (p in range64(npos)) { rdec_rope_row(t, p + s.rope_pos_delta, hs, half, cb, p * hs) } - rdec_prefill(s.x_b, cb, npos, s.logits) + if (s.embed_gpu_pending) { + rdec_prefill_ids(s.embed_gpu_tokens, c.embed_scale, cb, npos, s.logits) + s.embed_gpu_pending = false + } else { + rdec_prefill(s.x_b, cb, npos, s.logits) + } g_rdec_cnt = npos // the mirror now holds [0, npos) — the decode's continuity witness g_rdec_gen ++ s.rdec_gen = g_rdec_gen @@ -811,13 +845,20 @@ def private vulkan_resident_batch_decode(t : Model; var ws : BatchWorkspace; var return true } -[init] +[hot_path, unused_argument(t), arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] // per prefill quantum - reached only through the engine's registered probe +def private vulkan_embed_gpu_gate(t : Model; s : Session; npos, start_pos : int64) : bool { + return (active_prefill_override() == "vulkan" && g_rdec_active && g_resident_prefill_allowed && g_rdec_emb_gpu + && start_pos == 0l && npos <= g_rdec_cap + && s.kv_dtype_k == g_rdec_kvdt && s.kv_dtype_v == g_rdec_kvdt && s.kv_pool == null + && s.attn_uniform_end == 0l && empty(s.mrope_pos) && !s.ds_active) +} + +[init, arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] def resident_driver_register() { + register_embed_gpu_gate(@@vulkan_embed_gpu_gate) register_decode_override("vulkan", @@vulkan_resident_decode) register_batch_decode_override("vulkan", @@vulkan_resident_batch_decode) register_prefill_override("vulkan", @@vulkan_resident_prefill) - // the resident chain encodes the causal mask device-side — a fused mid-turn span cannot - // serve on it, so eval_embd_span_ keeps the three-eval splice while vulkan is active register_prefill_override_split_span("vulkan") register_gpu_upload_resident(@@moe_gpu_upload_resident) set_moe_gpu_slice_window_fn(@@vk_slice_window_provider) @@ -834,6 +875,7 @@ struct GpuModelMarks { rdec_cnt : int64 rdec_cap : int64 rdec_kvdt : KVDtype + rdec_emb_gpu : bool } //! A fresh no-model GpuModelMarks — the facade's ctor-shaped re-export delegates here @@ -842,28 +884,32 @@ def gpu_model_marks_init_() : GpuModelMarks { return <- GpuModelMarks() } -//! Move the INSTALLED per-model tier state out into `st`; the globals read as no-model after. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#gpu-slot-marks")] def moe_gpu_model_marks_save_(var st : GpuModelMarks) { moe_gpu_marks_save(st.marks) st.rdec_active = g_rdec_active st.rdec_cnt = g_rdec_cnt st.rdec_cap = g_rdec_cap st.rdec_kvdt = g_rdec_kvdt + st.rdec_emb_gpu = g_rdec_emb_gpu g_rdec_active = false g_rdec_cnt = 0l g_rdec_cap = 0l + g_rdec_emb_gpu = false } -//! Install `st` as the per-model tier state (the save's inverse; `st` reads as no-model after). +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#gpu-slot-marks")] def moe_gpu_model_marks_restore_(var st : GpuModelMarks) { moe_gpu_marks_restore(st.marks) g_rdec_active = st.rdec_active g_rdec_cnt = st.rdec_cnt g_rdec_cap = st.rdec_cap g_rdec_kvdt = st.rdec_kvdt + g_rdec_emb_gpu = st.rdec_emb_gpu st.rdec_active = false st.rdec_cnt = 0l st.rdec_cap = 0l + st.rdec_emb_gpu = false } //! Is the resident whole-stack decode driver active for the INSTALLED model? (The arm-outcome @@ -873,6 +919,7 @@ def moe_gpu_resident_active() : bool => g_rdec_active //! Capture the INSTALLED model's tier state into `st` (see ``moe_gpu_model_marks_save_``) and //! classify what it had: "gpu:resident" | "gpu:rails" | "cpu". The load-time half of a //! multi-model host's slot bookkeeping. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#gpu-slot-marks")] def gpu_slot_capture_(var st : GpuModelMarks) : string { let was_resident = g_rdec_active moe_gpu_model_marks_save_(st) @@ -882,6 +929,7 @@ def gpu_slot_capture_(var st : GpuModelMarks) : string { //! Re-arm the INSTALLED model onto the GPU tier (marks already restored): want, arm, resident //! upload — through the bake-slice path when `t` is a mapped vulkan-flavor image (a live //! re-gather on a planes-trimmed image panics). Returns "gpu:resident" | "gpu:rails" | "cpu". +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#gpu-slot-marks")] def gpu_slot_rearm_(want : GpuTierWant; t : Model) : string { set_gpu_tier_want_(want) moe_gpu_tier_arm_() @@ -910,6 +958,7 @@ def moe_gpu_hydrate_session_(t : Model; var s : Session) { //! Drop the INSTALLED model's whole GPU state (every model-owned device object + the routing //! marks); the model keeps serving on the CPU rails, and re-arm = the same three calls a load //! runs. Caller guarantees quiescence: no step in flight, sessions hydrated first. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#gpu-slot-marks")] def moe_gpu_drop_model_() { if (!moe_gpu_tier_installed()) { return @@ -918,6 +967,7 @@ def moe_gpu_drop_model_() { g_rdec_active = false g_rdec_cnt = 0l g_rdec_cap = 0l + g_rdec_emb_gpu = false // resident_upload selected the "vulkan" overrides; deselect so (a) a later re-arm's // resident_upload passes its no-active-override gate (the warm re-arm decline), and (b) a // dropped model's prefill/decode take the plain CPU path without consulting dead state diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das b/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das index 570e5eb0ff..9934120052 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das @@ -191,6 +191,51 @@ def public install_moe_gpu_resident(reserve : RdecReserveFn; place : RdecPlaceFn g_rdec_installed = true } +typedef RdecPrefillIdsFn = function<(tokens : array; emb_scale : float; cos_batch : array; npos : int64; var logits : array) : void> +typedef RdecSetEmbFn = function<(emb_block : int64) : bool> +typedef RdecUploadEmbF32Fn = function<(fblob : array; off : int64; vocab : int64; dim : int64) : bool> + +//! The raw f32 embd table rides the residency plan's headroom only up to this size; past it the CPU embed serves. +let public RDEC_EMB_F32_CAP = 512l * 1024l * 1024l + +//! The device bytes the f32 embed-gather arm would upload for a (vocab x dim) table - 0 past its cap. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#resident-plan")] +def public rdec_emb_f32_bytes(vocab, dim : int64) : int64 { + let bytes = vocab * dim * 4l + return bytes > RDEC_EMB_F32_CAP ? 0l : bytes +} + +[unused_argument(tokens, emb_scale, cos_batch, npos, logits)] +def private rdec_unset_prefill_ids(tokens : array; emb_scale : float; cos_batch : array; npos : int64; var logits : array) { + panic("dasLLAMA: ids-form resident prefill hit without an installed GPU driver") +} +[unused_argument(emb_block)] +def private rdec_unset_set_emb(emb_block : int64) : bool => false +[unused_argument(fblob, off, vocab, dim)] +def private rdec_unset_upload_emb_f32(fblob : array; off, vocab, dim : int64) : bool => false + +var g_rdec_prefill_ids = @@rdec_unset_prefill_ids +var g_rdec_set_emb = @@rdec_unset_set_emb +var g_rdec_upload_emb_f32 = @@rdec_unset_upload_emb_f32 +var g_rdec_prefill_ids_installed = false + +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def public install_rdec_prefill_ids(prefill_ids : RdecPrefillIdsFn; set_emb : RdecSetEmbFn; upload_emb_f32 : RdecUploadEmbF32Fn) { + g_rdec_prefill_ids = prefill_ids + g_rdec_set_emb = set_emb + g_rdec_upload_emb_f32 = upload_emb_f32 + g_rdec_prefill_ids_installed = true +} + +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def public rdec_prefill_ids_installed() : bool => g_rdec_prefill_ids_installed +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def public rdec_prefill_ids(tokens : array; emb_scale : float; cos_batch : array; npos : int64; var logits : array) { invoke(g_rdec_prefill_ids, tokens, emb_scale, cos_batch, npos, logits) } +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def public rdec_set_emb(emb_block : int64) : bool => invoke(g_rdec_set_emb, emb_block) +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def public rdec_upload_emb_f32(fblob : array; off, vocab, dim : int64) : bool => invoke(g_rdec_upload_emb_f32, fblob, off, vocab, dim) + def public moe_gpu_resident_installed() : bool => g_rdec_installed def public rdec_reserve(fmt : int; cap_blocks : int64) : bool => invoke(g_rdec_reserve, fmt, cap_blocks) def public rdec_place(wq : array; ws : array; n, rows : int64; fmt : int) : int64 => invoke(g_rdec_place, wq, ws, n, rows, fmt) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das index 93d0d69c34..2f8fb709ed 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das @@ -284,7 +284,7 @@ def private vk_bake_tag : string { return dlim_vulkan_tag(v) } -[init] +[init, arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] def dasllama_math_vulkan_register() { register_kernel_backend(KernelBackend(name = "vulkan", mm = @@vulkan_no_gemv, batch = @@vulkan_no_batch, group3 = @@vulkan_no_group3, repack = @@vulkan_repack_noop, @@ -299,6 +299,7 @@ def dasllama_math_vulkan_register() { set_moe_gpu_device_report(@@vk_device_name) set_moe_gpu_budget_hooks(@@vk_weight_budget, @@vk_plane_bytes) set_moe_gpu_binding_cap(@@vk_binding_cap) + install_rdec_prefill_ids(@@vk_rdec_prefill_ids, @@vk_rdec_set_emb, @@vk_rdec_upload_emb_f32) install_moe_gpu_resident(@@vk_arena_reserve, @@vk_arena_place, @@vk_rdec_prepare, @@vk_rdec_upload_norms, @@vk_rdec_set_layer, @@vk_rdec_set_cls, @@vk_rdec_token, @@vk_rdec_prefill, @@vk_rdec_sync_kv, @@vk_rdec_read_kv, @@vk_rdec_read_kv_bulk) diff --git a/modules/dasLLAMA/dasllama/dasllama_version.das b/modules/dasLLAMA/dasllama/dasllama_version.das index 08a7bc459e..19e44dad70 100644 --- a/modules/dasLLAMA/dasllama/dasllama_version.das +++ b/modules/dasLLAMA/dasllama/dasllama_version.das @@ -9,4 +9,4 @@ require dasllama/dasllama_lint public //! dasLLAMA's release counter — decoupled from the daslang version and LLVM_JIT_CODEGEN_VERSION. //! ANY kernel work bumps it (REVIEW.md): equal versions mean an equal kernel roster, and the //! sidecar exchange keys validity on (version, box). Bench records and sidecar provenance carry it. -let DASLLAMA_VERSION = 11 // v11: the M5 pp deep-dense arc - tall in-kernel-dequant kq stamps (TH128), MoE q5_1 tensor twins, dev-W tiles to 32 +let DASLLAMA_VERSION = 12 // v12: the vulkan cm2 default arc - byte-store requant writers, fused ar twins, f16-out fa stamps, K4/K6 cm2 tiles, embed gather diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index e8d90e2074..1bb1be0e82 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -39,7 +39,7 @@ struct ArArgs { ascale : float } -// the shared Q8_0 quantize pieces (pure — the amax fold is subgroup-wide, the pack is scalar) +// the shared Q8_0 quantize pieces (pure — the amax fold is subgroup-wide) def private q8_amax3(m0 : float) : float { var m = m0 m = max(m, subgroupShuffleXor(m, 1u)) @@ -48,17 +48,12 @@ def private q8_amax3(m0 : float) : float { return m } -def private q8_pack4(v0, v1, v2, v3 : float; id : float) : uint { - let q0 = int(round(v0 * id)) - let q1 = int(round(v1 * id)) - let q2 = int(round(v2 * id)) - let q3 = int(round(v3 * id)) - return ((uint(q0) & 0xFFu) | ((uint(q1) & 0xFFu) << 8u) - | ((uint(q2) & 0xFFu) << 16u) | ((uint(q3) & 0xFFu) << 24u)) +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] +def private q8_quant1(v, id : float) : int8 { + return int8(int(round(v * id))) } -// ... and the Q8_K twins: whole-subgroup 5-shuffle amax fold, butterfly byte pack (lanes -// 4j..4j+3 fold one round's bytes into word j — the lane%4==0 caller stores it) +// ... and the Q8_K twin: whole-subgroup 5-shuffle amax fold def private q8k_amax5(m0 : float) : float { var m = m0 m = max(m, subgroupShuffleXor(m, 1u)) @@ -69,13 +64,6 @@ def private q8k_amax5(m0 : float) : float { return m } -def private q8k_pack_butterfly(v, id : float) : uint { - let q = int(round(v * id)) - let b8 = uint(q) & 0xFFu - let p = b8 | (subgroupShuffleXor(b8, 1u) << 8u) - return p | (subgroupShuffleXor(p, 2u) << 16u) -} - // the workgroup-rms machinery the row kernels share (one wg reduces one row) class RmsWgBase { @workgroup part : float[64] // one partial per subgroup (subgroupSize >= 4 floor) @@ -150,10 +138,10 @@ class ClsArAddRms : ArBase { // xb never reaches memory. Verbatim reduce/amax/rounding => bit-identical to the split pair. [vk_dispatch(name = "cls_ar_rq", grid = "wgs", params = "wgs : int64")] class ArAddRmsRq : ArBase { - @ssbo @binding = 3 outq : array // the quantized words + @ssbo @binding = 3 outq : array // the quantized bytes @ssbo @binding = 4 outs : array // per-32-block scales - [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_spv")] + [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] def run { let tid = gl_LocalInvocationID.x let ss = accum_row(0u) @@ -174,12 +162,69 @@ class ArAddRmsRq : ArBase { if (lane8 == 0u) { outs[b] = d } - outq[b * 8u + lane8] = q8_pack4(v0, v1, v2, v3, id) + outq[base] = q8_quant1(v0, id) + outq[base + 1u] = q8_quant1(v1, id) + outq[base + 2u] = q8_quant1(v2, id) + outq[base + 3u] = q8_quant1(v3, id) + b += 32u + } + } +} + +[vk_dispatch(name = "cls_ar_rq_b", grid = "nrows", params = "nrows : int64")] +class ClsArAddRmsRqB : ArBase { + @ssbo @binding = 3 outq : array // the quantized bytes, row-major blocks + @ssbo @binding = 4 outs : array // per-32-block scales + + [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_b_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] + def run { + let tid = gl_LocalInvocationID.x + let rbase = gl_WorkGroupID.x * pa.dim + let ss = accum_row(rbase) + let inv = rms_inv(ss) + let nblk = pa.dim / 32u + let qb0 = gl_WorkGroupID.x * nblk + let lane8 = tid % 8u + var b = tid / 8u + while (b < nblk) { + let base = b * 32u + lane8 * 4u + let v0 = wn[pa.woff + base] * (row[base] * inv) + let v1 = wn[pa.woff + base + 1u] * (row[base + 1u] * inv) + let v2 = wn[pa.woff + base + 2u] * (row[base + 2u] * inv) + let v3 = wn[pa.woff + base + 3u] * (row[base + 3u] * inv) + let m = q8_amax3(max(max(abs(v0), abs(v1)), max(abs(v2), abs(v3)))) + let d = m / 127.0 + let id = d != 0.0 ? 1.0 / d : 0.0 + if (lane8 == 0u) { + outs[qb0 + b] = d + } + let qb = qb0 * 32u + base + outq[qb] = q8_quant1(v0, id) + outq[qb + 1u] = q8_quant1(v1, id) + outq[qb + 2u] = q8_quant1(v2, id) + outq[qb + 3u] = q8_quant1(v3, id) b += 32u } } } +[vk_dispatch(name = "cls_ar_f16_b", grid = "nrows", params = "nrows : int64")] +class ClsArAddRmsF16B : ArBase { + @ssbo @binding = 3 outh : array // the normed rows, f16 + + [spirv_kernel(local_size_x = 256, name = "cls_ar_f16_b_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] + def run { + let rbase = gl_WorkGroupID.x * pa.dim + let ss = accum_row(rbase) + let inv = rms_inv(ss) + var k = gl_LocalInvocationID.x + while (k < pa.dim) { + outh[rbase + k] = float16(wn[pa.woff + k] * (row[k] * inv)) + k += 256u + } + } +} + struct RqArgs { inbase : uint // element base of the source rows nblk : uint // block / superblock count @@ -189,11 +234,11 @@ struct RqArgs { [vk_dispatch(name = "cls_dn_rq", kernel = "run", family = "rq_cls", grid = "wgs", params = "wgs : int64")] class DnRequant { @ssbo @binding = 0 src : array - @ssbo @binding = 1 outq : array + @ssbo @binding = 1 outq : array @ssbo @binding = 2 outs : array @push_constant pa : RqArgs - [spirv_kernel(local_size_x = 256, name = "cls_dn_rq_spv")] + [spirv_kernel(local_size_x = 256, name = "cls_dn_rq_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] def run { let gid = gl_GlobalInvocationID.x let sg = gid / 32u @@ -212,7 +257,11 @@ class DnRequant { if (lane8 == 0u) { outs[b] = d } - outq[b * 8u + lane8] = q8_pack4(v0, v1, v2, v3, id) + let qb = b * 32u + lane8 * 4u + outq[qb] = q8_quant1(v0, id) + outq[qb + 1u] = q8_quant1(v1, id) + outq[qb + 2u] = q8_quant1(v2, id) + outq[qb + 3u] = q8_quant1(v3, id) } } } @@ -221,7 +270,7 @@ class DnRequant { [vk_dispatch(name = "cls_q8k_rq", kernel = "run", family = "rq_cls", grid = "wgs", params = "wgs : int64")] class Q8kRequant { @ssbo @binding = 0 src : array - @ssbo @binding = 1 outq : array + @ssbo @binding = 1 outq : array @ssbo @binding = 2 outs : array @push_constant pa : RqArgs @@ -233,14 +282,7 @@ class Q8kRequant { return d != 0.0 ? 1.0 / d : 0.0 } - def pack_word(b, r, lane : uint; v, id : float) { - let w = q8k_pack_butterfly(v, id) - if (lane % 4u == 0u) { - outq[b * 64u + r * 8u + lane / 4u] = w - } - } - - [spirv_kernel(local_size_x = 256, name = "cls_q8k_rq_spv")] + [spirv_kernel(local_size_x = 256, name = "cls_q8k_rq_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] def run { let gid = gl_GlobalInvocationID.x let b = gid / 32u @@ -256,7 +298,7 @@ class Q8kRequant { } let id = blk_scale(b, lane, m) for [unroll] (r in range(8)) { - pack_word(b, uint(r), lane, vals[r], id) + outq[b * 256u + uint(r) * 32u + lane] = q8_quant1(vals[r], id) } } } @@ -281,26 +323,30 @@ def private act_mul_sel(v, u : float; gelu : uint) : float { class ActRqMembers { @ssbo @binding = 0 gate : array // gate rows (act's v side) @ssbo @binding = 1 up : array // up rows (act's u side) - @ssbo @binding = 2 outq : array // quantized words out + @ssbo @binding = 2 outq : array // quantized bytes out @ssbo @binding = 3 outs : array // per-block scales out @push_constant pa : ActArgs def act_mul(v, u : float) : float => act_mul_sel(v, u, pa.gelu) } -// fused act + Q8_0 requant: subgroup per 4 blocks, word per lane, 8-lane shuffle amax (xor 1/2/4) +// fused act + Q8_0 requant: subgroup per 4 blocks, 4 quants per lane, 8-lane shuffle amax (xor 1/2/4) [vk_dispatch(name = "q8_actrq_cls", kernel = "run", family = "actrq_cls", grid = "wgs", params = "wgs : int64")] class Q8ActRq : ActRqMembers { - // 8-lane block quantize: amax fold over the block's 8 co-active lanes, d to outs[b], - // returns this lane's packed word - def blk_pack(b, lane8 : uint; v0, v1, v2, v3 : float) : uint { + // 8-lane block quantize: amax fold over the block's 8 co-active lanes, d to outs[b] + [arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] + def blk_store(b, lane8 : uint; v0, v1, v2, v3 : float) { let m = q8_amax3(max(max(abs(v0), abs(v1)), max(abs(v2), abs(v3)))) let d = m / 127.0 let id = d != 0.0 ? 1.0 / d : 0.0 if (lane8 == 0u) { outs[b] = d } - return q8_pack4(v0, v1, v2, v3, id) + let qb = b * 32u + lane8 * 4u + outq[qb] = q8_quant1(v0, id) + outq[qb + 1u] = q8_quant1(v1, id) + outq[qb + 2u] = q8_quant1(v2, id) + outq[qb + 3u] = q8_quant1(v3, id) } [spirv_kernel(local_size_x = 256, name = "q8_actrq_cls_spv")] @@ -315,13 +361,12 @@ class Q8ActRq : ActRqMembers { let v1 = act_mul(gate[eb + 1u], up[eb + 1u]) let v2 = act_mul(gate[eb + 2u], up[eb + 2u]) let v3 = act_mul(gate[eb + 3u], up[eb + 3u]) - outq[b * 8u + lane % 8u] = blk_pack(b, lane % 8u, v0, v1, v2, v3) + blk_store(b, lane % 8u, v0, v1, v2, v3) } } } -// the Q8_K twin: one subgroup per 256-weight superblock, whole-subgroup 5-shuffle amax, -// butterfly byte pack +// the Q8_K twin: one subgroup per 256-weight superblock, whole-subgroup 5-shuffle amax [vk_dispatch(name = "q8k_actrq_cls", kernel = "run", family = "actrq_cls", grid = "wgs", params = "wgs : int64")] class Q8kActRq : ActRqMembers { def blk_scale(b, lane : uint; m0 : float) : float { @@ -332,14 +377,7 @@ class Q8kActRq : ActRqMembers { return d != 0.0 ? 1.0 / d : 0.0 } - def pack_word(b, r, lane : uint; v, id : float) { - let w = q8k_pack_butterfly(v, id) - if (lane % 4u == 0u) { - outq[b * 64u + r * 8u + lane / 4u] = w - } - } - - [spirv_kernel(local_size_x = 256, name = "q8k_actrq_cls_spv")] + [spirv_kernel(local_size_x = 256, name = "q8k_actrq_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] def run { let gid = gl_GlobalInvocationID.x let b = gid / 32u @@ -356,7 +394,7 @@ class Q8kActRq : ActRqMembers { } let id = blk_scale(b, lane, m) for [unroll] (r in range(8)) { - pack_word(b, uint(r), lane, vals[r], id) + outq[base + uint(r) * 32u + lane] = q8_quant1(vals[r], id) } } } @@ -403,6 +441,60 @@ class F16Cvt { } } +struct EmbArgs { + npos : uint + dim : uint + wblk0 : uint // the embd plane's slab-local block base + embed_scale : float +} + +[vk_dispatch(name = "emb_gather_cls", grid = "wgs", params = "wgs : int64")] +class EmbGather { + @ssbo @binding = 0 @role = "weight" wq : array // weight quant words + @ssbo @binding = 1 @role = "weight" wsh : array // per-block weight scales + @ssbo @binding = 2 ids : array // one token id per row + @ssbo @binding = 3 xout : array + @push_constant pa : EmbArgs + + [spirv_kernel(local_size_x = 256, name = "emb_gather_cls_spv"), arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] + def run { + let e = gl_GlobalInvocationID.x * 4u + if (e < pa.npos * pa.dim) { + let row = e / pa.dim + let col = e % pa.dim + let ib = pa.wblk0 + ids[row] * (pa.dim / 32u) + col / 32u + let lw = (col % 32u) / 4u + let v = (float4(int4(unpack8(int(wq[ib * 8u + lw])))) * float(wsh[ib])) * pa.embed_scale // dequant then scale - the CPU's order + xout[e] = v.x + xout[e + 1u] = v.y + xout[e + 2u] = v.z + xout[e + 3u] = v.w + } + } +} + +[vk_dispatch(name = "emb_gather_f32_cls", grid = "wgs", params = "wgs : int64")] +class EmbGatherF32 { + @ssbo @binding = 0 @role = "weight" wf : array // the f32 embd table, vocab x dim rows + @ssbo @binding = 2 ids : array // one token id per row (binding 1 unused: ids/xout sit where the q8 twin binds them) + @ssbo @binding = 3 xout : array + @push_constant pa : EmbArgs + + [spirv_kernel(local_size_x = 256, name = "emb_gather_f32_cls_spv"), arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] + def run { + let e = gl_GlobalInvocationID.x * 4u + if (e < pa.npos * pa.dim) { + let row = e / pa.dim + let col = e % pa.dim + let src = ids[row] * pa.dim + col + xout[e] = wf[src] * pa.embed_scale + xout[e + 1u] = wf[src + 1u] * pa.embed_scale + xout[e + 2u] = wf[src + 2u] * pa.embed_scale + xout[e + 3u] = wf[src + 3u] * pa.embed_scale + } + } +} + // ===== the MoE combine ===== struct CombineArgs { @@ -1429,15 +1521,16 @@ struct FaCm2Args { // hs=64 (tinyllama-class). The h128 twin below differs only in the head-size-shaped tiles — // duplication tolerated until the kernel-reification arc. -[vk_dispatch(name = "fa_cm2_h64_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] -class FaCm2H64 { +[ |> template_struct_instance] +class template FaCm2H64T { @ssbo @binding = 0 qpanel : array // the prepped/roped window q rows (window-local) @ssbo @binding = 1 kh : array // f16 K shadow at absolute positions @ssbo @binding = 2 vh : array // f16 V shadow at absolute positions - @ssbo @binding = 3 outp : array // attention out plane (rows x qd) + @ssbo @binding = 3 outp : array // attention out plane (rows x qd) @push_constant pa : FaCm2Args + @template_constant OUT16 : bool = false - [spirv_kernel(local_size_x = 128, name = "fa_cm2_h64_cls_spv")] + [spirv_kernel(local_size_x = 128), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] def run { let qtiles = (pa.rows + 63u) / 64u let h = gl_WorkGroupID.x / qtiles @@ -1505,23 +1598,41 @@ class FaCm2H64 { coopmatReduce(ld, lrow, COOPMAT_REDUCE_ROW, @@facm2_smear) coopmatPerElement(ld, ld, @@facm2_recip) coopmatMulElem(o, ld, o) // O /= L (padding rows discard at store) - coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 64u, 64u) + static_if (OUT16) { + var of : coopmatWgAcc_f16_64x64 + coopmatConvert(of, o) + coopmatStoreTensor(of, outp, 0u, tlo, q0, 64u, h * 64u, 64u) + } else { + coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 64u, 64u) + } } } +[vk_dispatch(name = "fa_cm2_h64_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] +class FaCm2H64 : FaCm2H64T { + typedef OT = float +} + +[vk_dispatch(name = "fa_cm2_h64_f16_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] +class FaCm2H64F16 : FaCm2H64T { + typedef OT = float16 + override OUT16 = true +} + // hs=128 (llama-3.2-3B-class): same algorithm at Bc=32 (the head-size cut — K^T/V/softmax // state at 32 columns, KV loop steps 32). Fast standalone (24.7 TFLOP/s, right beside the h64) and // WINS end-to-end when the shadows fit (3B 4287 vs 3955 no-fa) — serving is gated only on the // resident plan learning the f16 shadow cost (see the pf_facm2 gate) -[vk_dispatch(name = "fa_cm2_h128_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] -class FaCm2H128 { +[ |> template_struct_instance] +class template FaCm2H128T { @ssbo @binding = 0 qpanel : array @ssbo @binding = 1 kh : array @ssbo @binding = 2 vh : array - @ssbo @binding = 3 outp : array + @ssbo @binding = 3 outp : array @push_constant pa : FaCm2Args + @template_constant OUT16 : bool = false - [spirv_kernel(local_size_x = 128, name = "fa_cm2_h128_cls_spv")] + [spirv_kernel(local_size_x = 128), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] def run { let qtiles = (pa.rows + 63u) / 64u let h = gl_WorkGroupID.x / qtiles @@ -1589,10 +1700,27 @@ class FaCm2H128 { coopmatReduce(ld, lrow, COOPMAT_REDUCE_ROW, @@facm2_smear) coopmatPerElement(ld, ld, @@facm2_recip) coopmatMulElem(o, ld, o) - coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 128u, 128u) + static_if (OUT16) { + var of : coopmatWgAcc_f16_64x128 + coopmatConvert(of, o) + coopmatStoreTensor(of, outp, 0u, tlo, q0, 64u, h * 128u, 128u) + } else { + coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 128u, 128u) + } } } +[vk_dispatch(name = "fa_cm2_h128_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] +class FaCm2H128 : FaCm2H128T { + typedef OT = float +} + +[vk_dispatch(name = "fa_cm2_h128_f16_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] +class FaCm2H128F16 : FaCm2H128T { + typedef OT = float16 + override OUT16 = true +} + struct DaAttnBArgs { npos : uint // window query rows nh : uint @@ -3236,7 +3364,7 @@ class KqQ40CmF16 : MoeCmBase { // ===== the cm2 (NV_cooperative_matrix2) prefill tiles: native fmt-0 planes, decode-in-load ===== struct VkQ8Blk { - qs : uint[8] // one fmt-0 q8 block: 32 packed int8 quants; the scale lives in wsh + qs : int16[16] // one fmt-0 q8 block: 32 packed int8 quants as 16-bit lanes; the scale lives in wsh } // the cm2 quant "l" geometry on the NATIVE fmt-0 two-plane layout: 256 threads, one @@ -3245,20 +3373,19 @@ struct VkQ8Blk { // exists. Binding 4 unused (sparse) [vk_dispatch(name = "q8_batch_cm2l_cls", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (8 words each) + @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) @ssbo @binding = 3 xf16 : array // f16 activation plane @ssbo @binding = 5 y : array @workgroup wg_blk0 : uint // the region's block base, staged for the decode method - [spirv_decode] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let w = blk.qs[int(cib.y >> 2u)] - let q = (int(w) << ((3 - int(cib.y & 3u)) * 8)) >> 24 - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(q)) + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) } - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2l_cls_spv")] + [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled let reg = region_rec() let wblk0 = reg.x @@ -3423,24 +3550,23 @@ class Q8Cm2LBatch : MoeCmBase { } } -// the m-tile sibling (cm2 quant "m": BN=128, same A / BK / thread count) — picked by -// cm2_tile_cols when the l grid underfills the SMs; body mirrors Q8Cm2LBatch at 128-token width +// the m-tile sibling (cm2 quant "m": BN=128, same A / BK / thread count); the body mirrors +// Q8Cm2LBatch at 128-token width [vk_dispatch(name = "q8_batch_cm2m_cls", grid = "wgs", params = "wgs : int64")] class Q8Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (8 words each) + @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) @ssbo @binding = 3 xf16 : array // f16 activation plane @ssbo @binding = 5 y : array @workgroup wg_blk0 : uint // the region's block base, staged for the decode method - [spirv_decode] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let w = blk.qs[int(cib.y >> 2u)] - let q = (int(w) << ((3 - int(cib.y & 3u)) * 8)) >> 24 - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(q)) + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) } - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2m_cls_spv")] + [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled let reg = region_rec() let wblk0 = reg.x @@ -3592,62 +3718,760 @@ class Q8Cm2MBatch : MoeCmBase { } } -// ===== the split-k reduce (sums the cm2 partial planes into y) ===== +// ===== the cm2 K-quant tiles: Q4_K decode-in-load over the repacked superblock planes ===== -struct SkRedArgs { - nelem : uint // one partial plane's float count (cnt x d; always a 4-multiple, d is 32-aligned) - k : uint // planes +struct VkK4Blk { + qs : int16[64] // one Q4_K superblock's 128 nibble bytes as 16-bit lanes; its scale row (d|dmin + 8 sc + 8 mn bytes) lives in wsu } -// 4 floats per thread, like the elementwise family -[vk_dispatch(name = "splitk_reduce_cls", grid = "wgs", params = "wgs : int64")] -class SplitKReduce { - @ssbo @binding = 0 part : array - @ssbo @binding = 1 y : array - @push_constant pa : SkRedArgs +// the l-tile Q4_K twin of Q8Cm2LBatch: same geometry, superblock (1, 256) blocks, nibble + per-32-group scale/min decode +[vk_dispatch(name = "kq_batch_k4_cm2l_cls", grid = "wgs", params = "wgs : int64")] +class K4Cm2LBatch : MoeCmBase { + @ssbo @binding = 0 @role = "weight" wq : array // Q4_K quant plane (128-byte superblocks) + @ssbo @binding = 1 @role = "weight" wsu : array // scale plane: 5 words per superblock + @ssbo @binding = 3 xf16 : array // f16 activation plane + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint // the region's SUPERBLOCK base, staged for the decode + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dm = unpackHalf2x16(wsu[srow]) + let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) + } - [spirv_kernel(local_size_x = 256, name = "splitk_reduce_cls_spv")] - def run { - let e = gl_GlobalInvocationID.x * 4u - if (e < pa.nelem) { - var s0 = 0.0 - var s1 = 0.0 - var s2 = 0.0 - var s3 = 0.0 - var p = 0u - while (p < pa.k) { - let b = p * pa.nelem + e - s0 += part[b] - s1 += part[b + 1u] - s2 += part[b + 2u] - s3 += part[b + 3u] - p++ + [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] + def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled + let reg = region_rec() + let wblk0 = reg.x + let row0 = reg.y + let cnt = reg.z + let ttiles = (cnt + 255u) / 256u + var tix = reg.w + var ks = 0u + var k1 = pa.n + var ybase = 0u + if (pa.ksplit != 0u) { + let ptiles = ((pa.d + 127u) / 128u) * ttiles + ks = tix / ptiles + tix -= ks * ptiles + k1 = min(pa.n, (ks + 1u) * pa.ksplit) + ybase = ks * (row0 + cnt) * pa.d + } + let k0 = ks * pa.ksplit + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() // wg_blk0 visible before the first decode load + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + // FAST PATH — see Q8Cm2LBatch (kq n is always a 256-multiple, so only tile bounds gate) + if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 256u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 8u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + return } - y[e] = s0 - y[e + 1u] = s1 - y[e + 2u] = s2 - y[e + 3u] = s3 + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) + return + } + // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) + var tla : tensorLayout2DPad + tensorLayoutCreate(tla) + tensorLayoutSetBlockSize(tla, 1u, 256u) + tensorLayoutSetDimension(tla, pa.d, pa.n) + tensorLayoutSetStride(tla, pa.n / 256u, 1u) + var tlb : tensorLayout2DPad + tensorLayoutCreate(tlb) + tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) + tensorLayoutSetStride(tlb, pa.n, 1u) + var tlo : tensorLayout2DPad + tensorLayoutCreate(tlo) + tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) + tensorLayoutSetStride(tlo, pa.d, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) } } -// ===== rope + KV store + qk-norm ===== - -struct QkRmsArgs { - hs : uint - nh : uint - nkvh : uint - qstride : uint - kstride : uint - kbase : uint - qwoff : uint - kwoff : uint - eps : float -} +// the m-tile Q4_K sibling (BN=128) — the k4 decode on the m geometry +[vk_dispatch(name = "kq_batch_k4_cm2m_cls", grid = "wgs", params = "wgs : int64")] +class K4Cm2MBatch : MoeCmBase { + @ssbo @binding = 0 @role = "weight" wq : array + @ssbo @binding = 1 @role = "weight" wsu : array + @ssbo @binding = 3 xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dm = unpackHalf2x16(wsu[srow]) + let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) + } -// per-head RMSNorm on q/k between the projections and rope (qk_norm models), one wg per -// head-row, in place -[vk_dispatch(name = "qk_rms_cls", grid = "wgs", params = "wgs : int64")] + [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] + def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled + let reg = region_rec() + let wblk0 = reg.x + let row0 = reg.y + let cnt = reg.z + let ttiles = (cnt + 127u) / 128u + var tix = reg.w + var ks = 0u + var k1 = pa.n + var ybase = 0u + if (pa.ksplit != 0u) { + let ptiles = ((pa.d + 127u) / 128u) * ttiles + ks = tix / ptiles + tix -= ks * ptiles + k1 = min(pa.n, (ks + 1u) * pa.ksplit) + ybase = ks * (row0 + cnt) * pa.d + } + let k0 = ks * pa.ksplit + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x128 + var acc : coopmatWgAcc_f16_128x128 + let t0 = row0 + xt * 128u + let m0 = wt * 128u + if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 256u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 8u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) + return + } + var tla : tensorLayout2DPad + tensorLayoutCreate(tla) + tensorLayoutSetBlockSize(tla, 1u, 256u) + tensorLayoutSetDimension(tla, pa.d, pa.n) + tensorLayoutSetStride(tla, pa.n / 256u, 1u) + var tlb : tensorLayout2DPad + tensorLayoutCreate(tlb) + tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) + tensorLayoutSetStride(tlb, pa.n, 1u) + var tlo : tensorLayout2DPad + tensorLayoutCreate(tlo) + tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) + tensorLayoutSetStride(tlo, pa.d, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) + } +} + +struct VkK6Blk { + ql : int16[64] // one Q6_K superblock's 128 low-nibble bytes as 16-bit lanes + qh : int16[32] // ... and its 64 high-2-bit bytes; scales (16 int8 + f16 d) live in wsu +} + +// the l-tile Q6_K twin: 6-bit compose (nibble | qh 2 bits) - 32, per-16 signed sub-scale +[vk_dispatch(name = "kq_batch_k6_cm2l_cls", grid = "wgs", params = "wgs : int64")] +class K6Cm2LBatch : MoeCmBase { + @ssbo @binding = 0 @role = "weight" wq : array // Q6_K quant plane (192-byte superblocks) + @ssbo @binding = 1 @role = "weight" wsu : array // scale plane: 5 words per superblock + @ssbo @binding = 3 xf16 : array // f16 activation plane + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint // the region's SUPERBLOCK base, staged for the decode + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let lb = bu * 16u + j + let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu + let qb = (bu >> 2u) * 32u + hh * 16u + j + let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu + let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let sidx = e >> 4u + let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 + let d = unpackHalf2x16(wsu[srow + 4u]).x + return float16(d * float(sc) * float(q6)) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] + def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled + let reg = region_rec() + let wblk0 = reg.x + let row0 = reg.y + let cnt = reg.z + let ttiles = (cnt + 255u) / 256u + var tix = reg.w + var ks = 0u + var k1 = pa.n + var ybase = 0u + if (pa.ksplit != 0u) { + let ptiles = ((pa.d + 127u) / 128u) * ttiles + ks = tix / ptiles + tix -= ks * ptiles + k1 = min(pa.n, (ks + 1u) * pa.ksplit) + ybase = ks * (row0 + cnt) * pa.d + } + let k0 = ks * pa.ksplit + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() // wg_blk0 visible before the first decode load + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 256u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 8u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) + return + } + // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) + var tla : tensorLayout2DPad + tensorLayoutCreate(tla) + tensorLayoutSetBlockSize(tla, 1u, 256u) + tensorLayoutSetDimension(tla, pa.d, pa.n) + tensorLayoutSetStride(tla, pa.n / 256u, 1u) + var tlb : tensorLayout2DPad + tensorLayoutCreate(tlb) + tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) + tensorLayoutSetStride(tlb, pa.n, 1u) + var tlo : tensorLayout2DPad + tensorLayoutCreate(tlo) + tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) + tensorLayoutSetStride(tlo, pa.d, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) + } +} + +// the m-tile Q6_K sibling (BN=128) +[vk_dispatch(name = "kq_batch_k6_cm2m_cls", grid = "wgs", params = "wgs : int64")] +class K6Cm2MBatch : MoeCmBase { + @ssbo @binding = 0 @role = "weight" wq : array + @ssbo @binding = 1 @role = "weight" wsu : array + @ssbo @binding = 3 xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let lb = bu * 16u + j + let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu + let qb = (bu >> 2u) * 32u + hh * 16u + j + let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu + let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let sidx = e >> 4u + let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 + let d = unpackHalf2x16(wsu[srow + 4u]).x + return float16(d * float(sc) * float(q6)) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] + def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled + let reg = region_rec() + let wblk0 = reg.x + let row0 = reg.y + let cnt = reg.z + let ttiles = (cnt + 127u) / 128u + var tix = reg.w + var ks = 0u + var k1 = pa.n + var ybase = 0u + if (pa.ksplit != 0u) { + let ptiles = ((pa.d + 127u) / 128u) * ttiles + ks = tix / ptiles + tix -= ks * ptiles + k1 = min(pa.n, (ks + 1u) * pa.ksplit) + ybase = ks * (row0 + cnt) * pa.d + } + let k0 = ks * pa.ksplit + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x128 + var acc : coopmatWgAcc_f16_128x128 + let t0 = row0 + xt * 128u + let m0 = wt * 128u + if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 256u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 8u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) + return + } + var tla : tensorLayout2DPad + tensorLayoutCreate(tla) + tensorLayoutSetBlockSize(tla, 1u, 256u) + tensorLayoutSetDimension(tla, pa.d, pa.n) + tensorLayoutSetStride(tla, pa.n / 256u, 1u) + var tlb : tensorLayout2DPad + tensorLayoutCreate(tlb) + tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) + tensorLayoutSetStride(tlb, pa.n, 1u) + var tlo : tensorLayout2DPad + tensorLayoutCreate(tlo) + tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) + tensorLayoutSetStride(tlo, pa.d, 1u) + if (pa.ksplit == 0u) { + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) + return + } + var k = k0 + for (_i in range(int((k1 - k0) / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < k1) { + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x128 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) + } +} + +// ===== the split-k reduce (sums the cm2 partial planes into y) ===== + +struct SkRedArgs { + nelem : uint // one partial plane's float count (cnt x d; always a 4-multiple, d is 32-aligned) + k : uint // planes +} + +// 4 floats per thread, like the elementwise family +[vk_dispatch(name = "splitk_reduce_cls", grid = "wgs", params = "wgs : int64")] +class SplitKReduce { + @ssbo @binding = 0 part : array + @ssbo @binding = 1 y : array + @push_constant pa : SkRedArgs + + [spirv_kernel(local_size_x = 256, name = "splitk_reduce_cls_spv")] + def run { + let e = gl_GlobalInvocationID.x * 4u + if (e < pa.nelem) { + var s0 = 0.0 + var s1 = 0.0 + var s2 = 0.0 + var s3 = 0.0 + var p = 0u + while (p < pa.k) { + let b = p * pa.nelem + e + s0 += part[b] + s1 += part[b + 1u] + s2 += part[b + 2u] + s3 += part[b + 3u] + p++ + } + y[e] = s0 + y[e + 1u] = s1 + y[e + 2u] = s2 + y[e + 3u] = s3 + } + } +} + +// ===== rope + KV store + qk-norm ===== + +struct QkRmsArgs { + hs : uint + nh : uint + nkvh : uint + qstride : uint + kstride : uint + kbase : uint + qwoff : uint + kwoff : uint + eps : float +} + +// per-head RMSNorm on q/k between the projections and rope (qk_norm models), one wg per +// head-row, in place +[vk_dispatch(name = "qk_rms_cls", grid = "wgs", params = "wgs : int64")] class QkRms : RmsWgBase { @ssbo @binding = 0 qrows : array // q rows, normed in place @ssbo @binding = 1 krows : array // k rows, normed in place @@ -3770,7 +4594,8 @@ struct RopeKvBArgs { layerbase : uint // this layer's K/V mirror base npos : uint pos0 : uint // the window's first absolute position - voff : uint // v rows' base inside the kv projection buffer + voff : uint + kstride : uint } // batched rope + KV-store (prefill, npos positions): q roped in place, k/v from the projection @@ -3785,7 +4610,7 @@ class template RopeKvStoreBT { @push_constant pa : RopeKvBArgs @template_constant CLAMP16 : bool = true - [spirv_kernel(local_size_x = 256)] + [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] def run { let gid = gl_GlobalInvocationID.x if (gid < pa.npos * pa.ppp) { @@ -3802,11 +4627,11 @@ class template RopeKvStoreBT { let e1 = pa.neox != 0u ? j + pa.half : j * 2u + 1u let apos = pa.pos0 + p if (is_k) { - let kb = p * pa.kvd + h * pa.hs + let kb = p * pa.kstride + h * pa.hs let a0 = kvrows[kb + e0] let a1 = kvrows[kb + e1] let mo = pa.layerbase + apos * pa.kvd + h * pa.hs - let vb = pa.voff + p * pa.kvd + h * pa.hs + let vb = pa.voff + p * pa.kstride + h * pa.hs static_if (CLAMP16) { kmir[mo + e0] = float16(clamp(a0 * fcr - a1 * fci, -65504.0, 65504.0)) kmir[mo + e1] = float16(clamp(a0 * fci + a1 * fcr, -65504.0, 65504.0)) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 8d37e3a49f..4966564d58 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -126,14 +126,15 @@ def cm2_splitk_env : int64 { } -// the cm2 tile pick over our two tiles (no s tile — GEMV owns the -// narrow-n end): l unless the m grid fills the SMs better. Pure in (d, cnt, sm_count), so the -// pipe pick and the meta fill can never disagree +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def cm2_tile_cols(d, cnt : int64) : int64 { let forced = cm2_tile_env() if (forced == 128l || forced == 256l) { return forced } + if (cnt <= 128l) { + return 128l + } let cores = int64(g_gpu.sm_count) if (cores == 0l) { return 256l @@ -141,11 +142,9 @@ def cm2_tile_cols(d, cnt : int64) : int64 { let wtiles = (d + 127l) / 128l let tiles_l = wtiles * ((cnt + 255l) / 256l) let tiles_m = wtiles * ((cnt + 127l) / 128l) - // prefer l when either grid overfills the SMs, or l fits with a 3-way split where m would - // not fit with a 2-way one - let prefer_large = (tiles_m > cores || tiles_l > cores - || (tiles_l <= cores / 3l && tiles_m > cores / 2l)) - return cnt > 128l && prefer_large ? 256l : 128l + let waves_l = (tiles_l + cores - 1l) / cores + let waves_m = (tiles_m + cores - 1l) / cores + return tiles_m * waves_l > tiles_l * waves_m ? 128l : 256l } // the split-k heuristic: long K on an under-half-full grid splits the reduction across @@ -194,9 +193,10 @@ let COMB_WI_BYTES = 2_097_152l // combine w / inv plane cap (npos*k grid ent let CLS_META_BYTES = 64l // classifier GEMV params (one region, filled once) var g_coopmat_mode_force = -1 -//! Test hook: pin the GEMM mode (0 sdot4 / 1 f16 / 2 int8 / 3 mm) ahead of the mm default and +//! Test hook: pin the GEMM mode (0 sdot4 / 1 f16 / 2 int8 / 3 mm / 4 cm2) ahead of the resolved default and //! DASLLAMA_COOPMAT — must run before the tier's lazy device init. The CPU-reference tier tests //! pin 0 so their oracles stay reference-kernel-deterministic under any router default. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def vk_force_coopmat_mode(m : int) { g_coopmat_mode_force = m } @@ -415,7 +415,8 @@ struct GpuState { has_coopmat : bool // device supports VK_KHR_cooperative_matrix (the f16/int8 tensor tiles) has_coopmat2 : bool // device supports NV_cooperative_matrix2 (tensor-addressed wg tiles + decode fns) has_coopmat2_fa : bool // ... plus the cm2 flash-attention trio (reductions, conversions, per-element ops) - coopmat_mode : int // Q8_0 prefill GEMM: 0 = sdot4, 1 = f16 coopmat, 2 = int8 coopmat, 3 = mul_mm L-tile (default), 4 = cm2 decode-in-load + full_sg_on : bool // DASLLAMA_VK_FULLSG asked and the device has the feature: every class pipeline pins REQUIRE_FULL_SUBGROUPS + coopmat_mode : int // Q8_0 prefill GEMM: 0 = sdot4, 1 = f16 coopmat, 2 = int8 coopmat, 3 = mul_mm L-tile, 4 = cm2 decode-in-load (the default where the device has it, else 3) sm_count : int // shaderSMCount (VK_NV_shader_sm_builtins; 0 = unknown -> l-tile always, no split-k) weight_budget : int64 // resident-weight cap — queried heap budget minus reserve, or VRAM_BUDGET dry : bool // OFFLINE BAKE: accept/refuse arithmetic runs, every device call is gated off @@ -723,11 +724,9 @@ def nonowning_buf(h : uint64) : Buffer { return b } -// The coopmat-mode ladder, shared by vk_moe_init and the DlimConfiguration source — one -// resolver, so the config's recorded mode can never drift from the mode the device runs. +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def resolve_coopmat_mode(has_cm, has_cm2 : bool) : int { - // fastest MEASURED default (mm; cm2 opt-in until it beats mm); DASLLAMA_COOPMAT overrides - var cmode = has_cm ? 3 : 0 + var cmode = has_cm2 ? 4 : (has_cm ? 3 : 0) if (has_cm) { let mode = g_env_vulkan.coopmat if (mode == "sdot4") { @@ -858,7 +857,7 @@ def private query_sm_count(phys : VkPhysicalDevice) : int { return int(sp.shaderSMCount) } -[cold_path] // one-time device bring-up; steady-state cost is the two guard checks +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-class-pipeline-build")] def vk_moe_init : bool { if (g_init_failed) { // checked FIRST: a partially-built g_gpu must never report ready return false @@ -896,7 +895,7 @@ def vk_moe_init : bool { g_gpu.phys = select_physical_device(g_gpu.instance) if (!storage_8_16_supported(g_gpu.phys) || !subgroup_compute_ops_supported(g_gpu.phys) || !integer_dot_product_supported(g_gpu.phys)) { - to_log(LOG_ERROR, "dasLLAMA vulkan tier: device lacks 8/16-bit SSBO storage, compute subgroup ops, or shaderIntegerDotProduct - tier disabled\n") + to_log(LOG_ERROR, "dasLLAMA vulkan tier: device lacks the 8/16-bit SSBO storage set (incl. shaderInt16), compute subgroup ops, or shaderIntegerDotProduct - tier disabled\n") return false } g_gpu.fam = select_graphics_queue_family(g_gpu.phys) @@ -915,12 +914,12 @@ def vk_moe_init : bool { if (g_gpu.has_coopmat2 && g_gpu.coopmat_mode == 4) { // #77's creator: coopmat2 + BDA + vulkanMemoryModel + the query-then-enable extras g_gpu.device <- create_device_storage_8_16_int_dot_coopmat2(g_gpu.phys, g_gpu.fam, xfam) - to_log(LOG_INFO, "dasLLAMA vulkan tier: Q8_0 prefill on NV_coopmat2 tensor tiles (decode-in-load, {g_gpu.sm_count} SMs)\n") + to_log(LOG_INFO, "dasLLAMA vulkan tier: Q8_0 prefill on NV_coopmat2 tensor tiles (decode-in-load, {g_gpu.sm_count} SMs; DASLLAMA_COOPMAT=mm restores the mul_mm tiles)\n") } elif (g_gpu.has_coopmat) { g_gpu.device <- create_device_storage_8_16_int_dot_coopmat(g_gpu.phys, g_gpu.fam, xfam) if (g_gpu.coopmat_mode != 0) { let cm_name = g_gpu.coopmat_mode == 1 ? "f16" : (g_gpu.coopmat_mode == 2 ? "int8" : "mul_mm L-tile") - to_log(LOG_INFO, "dasLLAMA vulkan tier: Q8_0 prefill on {cm_name} cooperative-matrix tiles\n") + to_log(LOG_INFO, "dasLLAMA vulkan tier: Q8_0 prefill on {cm_name} cooperative-matrix tiles (DASLLAMA_COOPMAT picks the family)\n") } } else { g_gpu.device <- create_device_storage_8_16_int_dot(g_gpu.phys, g_gpu.fam) @@ -1020,7 +1019,8 @@ def vk_moe_init : bool { g_gpu.staging = make_host_buf(STAGING_BYTES, false) g_gpu.host_mem |> erase(g_gpu.staging.buf) // process-lifetime — exempt from the model-drop sweep g_init_failed = false - to_log(LOG_INFO, "dasLLAMA vulkan tier: device ready (subgroup {int(sg)}, {g_gpu.rows_per_wg} rows/wg)\n") + g_gpu.full_sg_on = g_env_vulkan.vk_fullsg && compute_full_subgroups_supported(g_gpu.phys) + to_log(LOG_INFO, "dasLLAMA vulkan tier: device ready (subgroup {int(sg)}, {g_gpu.rows_per_wg} rows/wg{g_gpu.full_sg_on ? ", full subgroups (DASLLAMA_VK_FULLSG)" : ""})\n") if (dasllama_noisy()) { // storage-range is the limit that actually binds the arena; allocs/descriptors do not var lp : VkPhysicalDeviceProperties @@ -1913,8 +1913,15 @@ struct RDec { pf_logits_host : HostBuf pf_meta : array // one host meta per (layer, role) + prologue/final/cls pf_sets : array - @scratch pf_cls_gemm : table // GEMM-role class sets, keyed idx*8 + router variant (lazy insert, steady-state hit) + @scratch pf_cls_gemm : table // GEMM-role class sets, keyed idx*16 + variant (lazy insert, steady-state hit) pf_cmd : VkCommandBuffer + pf_cmd_ring : array + emb_blk : int64 = -1l + emb_f32 : uint64 + emb_f32_bytes : int64 + pf_ids : uint64 // [MAX_NPOS] token ids for the device-side embed gather + pf_emb_set : VkDescriptorSet + pf_arq_sets : array // fused ar+rq|f16 per layer: [l*2] = the ffn site, [l*2+1] = addr_next } let PF_WINDOW = 512l // prefill activation-buffer rows; longer prompts run as sequential windows @@ -2056,8 +2063,7 @@ let VHZ_FFO = 0x4000u // ffn down out let VHZ_LOG = 0x10000u // logits let VHZ_COS = 0x20000u // rope cos/sin rows -// batch/hybrid-tier region bits — a separate namespace over the same rail (each recorder's VkHaz -// is private to its cmd buffer, so the two families never meet in one pending set) +// batch/hybrid-tier region bits — a separate namespace over the same rail let VHB_ACT = 0x1u // batch_xq/batch_xs activation staging let VHB_META = 0x2u // stack batch metas + the chain's params (ffn/dn/at meta) let VHB_Y1 = 0x4u // batch_y1 (gate / qkv / q|gate / down / wo out) @@ -2264,14 +2270,34 @@ def private spv_override_load(kernel : string; var spv : array) { } } -[cold_path] // pipeline creation — once per kernel +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-class-pipeline-build")] +def private spv_dump_save(kernel : string; spv : array) { + let dir = g_env_vulkan.vk_spv_dump + if (empty(dir)) { + return + } + let path = path_join(dir, "{kernel}.spv") + fopen(path, "wb") $(f) { + if (f != null) { + f |> fwrite(spv) + to_log(LOG_INFO, "dasLLAMA vulkan tier: DASLLAMA_VK_SPV_DUMP wrote {path}\n") + } else { + to_log(LOG_WARNING, "dasLLAMA vulkan tier: DASLLAMA_VK_SPV_DUMP could not write {path}\n") + } + } +} + +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-class-pipeline-build")] def vkd_class_pipe(var c : VkdClass; ord : int; var spv : array; rmask, wmask : uint; kernel : string) { if (length(c.pipes) <= ord) { c.pipes |> resize(ord + 1) // empty slots hold null handles until their kernel ensures } + spv_dump_save(kernel, spv) // the EMITTED words, before an override replaces them spv_override_load(kernel, spv) var shader <- create_shader_module(g_gpu.device, spv) - c.pipes[ord] <- create_compute_pipeline(g_gpu.device, c.pipe_layout, shader) + c.pipes[ord] <- (g_gpu.full_sg_on + ? create_compute_pipeline_full_subgroups(g_gpu.device, c.pipe_layout, shader) + : create_compute_pipeline(g_gpu.device, c.pipe_layout, shader)) let pk = intptr(boost_value_to_vk(c.pipes[ord])) g_pipe_access[pk] = uint2(rmask, wmask) g_pipe_names[pk] = kernel @@ -2477,10 +2503,10 @@ def alloc_cmd : VkCommandBuffer { return raw } -def submit_nowait(var raw : VkCommandBuffer) { +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] +def private submit_core(var raw : VkCommandBuffer; fence : VkFence) { var submit = VkSubmitInfo() submit.commandBufferCount = 1u - let fence = g_gpu.fence // a pending transfer-queue handoff attaches here — later in-order submits inherit the ordering var tss = VkTimelineSemaphoreSubmitInfo() var wval = g_gpu.pend_wait @@ -2503,6 +2529,16 @@ def submit_nowait(var raw : VkCommandBuffer) { } } +def submit_nowait(var raw : VkCommandBuffer) { + submit_core(raw, g_gpu.fence) +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] +def submit_nofence(var raw : VkCommandBuffer) { + var nofence : VkFence + submit_core(raw, nofence) +} + def wait_fence { var fence = g_gpu.fence unsafe { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index 9dec59c7c6..b4056861a9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -11,6 +11,7 @@ require dasllama/dasllama_vulkan_common // g_gpu/g_rd state, plumbing, hazard require dasllama/dasllama_vulkan_classes // the class-kernel rail (generated set_/enc_ + gemv_cls_*) require dasllama/dasllama_vulkan_decode // vk_moe_dn_step — the dn chain's npos==1 arm routes through the decode step require dasllama/dasllama_kqformat // KqFmt — the arena fmt slots are its ordinals +require dasllama/dasllama_env // g_env_vulkan — the vk_kv_merge / vk_overlap / vk_fuse A/B hatches require dasllama/dasllama_gpu_tier // moe_gpu_stream_need — the stream-slot carve's source of truth require vulkan require vulkan/vulkan_boost @@ -35,30 +36,97 @@ require math let private PF_ROLES = 16l // quant_xb q k v rope attn quant_at wo addr_ffn quant_xb2 gate up actrq down addr_next qkn -// mode 4 feeds the dense q8 prefill GEMMs f16 rows through the cm2 tiles (the decode GEMV -// keeps its q8 chain — the format pick is decoupled) -def private pf_f16_feed(f : int) : bool => f == int(KqFmt.q8) && g_gpu.coopmat_mode == 4 +var private @scratch g_pf_ids_stage : array + +[arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] +def private pf_f16_feed(f : int) : bool { + return (f == int(KqFmt.q8) || f == int(KqFmt.k4) || f == int(KqFmt.k6)) && g_gpu.coopmat_mode == 4 +} + +[arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] +def private pf_qkv6(l : int64) : bool => (pf_f16_feed(g_rd.layers[l].fq) + && pf_f16_feed(g_rd.layers[l].fk) && pf_f16_feed(g_rd.layers[l].fv)) +[arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] +def private pf_gu6(l : int64) : bool => pf_f16_feed(g_rd.layers[l].f1) && pf_f16_feed(g_rd.layers[l].f3) + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] +def private pf_qkv_feed_fused(l : int64) : bool => g_env_vulkan.vk_fuse && (pf_qkv6(l) || !kq_sb(g_rd.layers[l].fq)) + +let private PF_CHUNK_MAX = 8l // the overlap ramp doubles 1,2,4 then holds here; the cmd ring is sized from it + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +def private cm2_cls_ensure(fmt : int; ml : bool) : bool { + verify(fmt == int(KqFmt.q8) || fmt == int(KqFmt.k4) || fmt == int(KqFmt.k6), "vk prefill: a cm2 tile for a format the f16 feed never admits") + if (fmt == int(KqFmt.k4)) { + return ml ? ensure_kq_batch_k4_cm2l_cls() : ensure_kq_batch_k4_cm2m_cls() + } + if (fmt == int(KqFmt.k6)) { + return ml ? ensure_kq_batch_k6_cm2l_cls() : ensure_kq_batch_k6_cm2m_cls() + } + return ml ? ensure_q8_batch_cm2l_cls() : ensure_q8_batch_cm2m_cls() +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +def private cm2_cls_set(fmt : int; ml : bool; bufs : uint64 const[5]; sizes : int64 const[5]; gbits : uint const[5]) : VkDescriptorSet { + if (fmt == int(KqFmt.k4)) { + return ml ? set_kq_batch_k4_cm2l_cls(bufs, sizes, gbits) : set_kq_batch_k4_cm2m_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.k6)) { + return ml ? set_kq_batch_k6_cm2l_cls(bufs, sizes, gbits) : set_kq_batch_k6_cm2m_cls(bufs, sizes, gbits) + } + return ml ? set_q8_batch_cm2l_cls(bufs, sizes, gbits) : set_q8_batch_cm2m_cls(bufs, sizes, gbits) +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +def private cm2_cls_enc(fmt : int; ml : bool; raw : VkCommandBuffer; var h : VkHaz; var s : VkDescriptorSet; var pc : BatchArgs; groups : int64) { + if (fmt == int(KqFmt.k4)) { + if (ml) { + enc_kq_batch_k4_cm2l_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k4_cm2m_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.k6)) { + if (ml) { + enc_kq_batch_k6_cm2l_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k6_cm2m_cls(raw, h, s, pc, groups) + } + } elif (ml) { + enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) + } else { + enc_q8_batch_cm2m_cls(raw, h, s, pc, groups) + } +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] +def private pf_kv_merged(l, dim, kvd : int64) : bool { + if (!g_env_vulkan.vk_kv_merge || g_rd.layers[l].fq != int(KqFmt.q8) + || g_rd.layers[l].fk != int(KqFmt.q8) || g_rd.layers[l].fv != int(KqFmt.q8)) { + return false + } + return g_rd.layers[l].bv == g_rd.layers[l].bk + kvd * (dim / 32l) +} // one prefill GEMM role on the class rail: schedule fill + a per-(role, variant) cached set + -// BatchArgs push. fmt picks the class: fmt-0 in mode 4 = the cm2 l/m tiles, kq = the kq tile -// family, else the q8 router (whose variant moves with wlen — the last window is shorter). +// BatchArgs push. kq formats take the kq tile family, else the q8 router (whose variant moves +// with wlen — the last window is shorter). +[arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] def private pf_gemm_enc(raw : VkCommandBuffer; var h : VkHaz; idx : int; fmt : int; wq, ws, ab, sb, yb : uint64; wqn, wsn, an, sn, yn : int64; - abit, ybit : uint; n, d, blk, wlen : int64) { - let cm2l = fmt == int(KqFmt.q8) && g_gpu.coopmat_mode == 4 + abit, ybit : uint; n, d, blk, wlen : int64; f16feed : bool) { // cm2 tile + split picks (pure in d/wlen/n/sm_count) — computed once, fed to fill AND enc var nsplit = 1l var ksplit = 0l var cm2_tc = 0l - if (cm2l) { + if (f16feed) { cm2_tc = batch_tile_edges(fmt, d, wlen, true).tc let sp = cm2_split_k(d, wlen, n, cm2_tc) nsplit = sp.nsplit ksplit = sp.ksplit } - let groups = int64(fill_arena_batch_sched(g_rd.pf_meta[idx], d, blk, wlen, fmt, cm2l, nsplit)) + let groups = int64(fill_arena_batch_sched(g_rd.pf_meta[idx], d, blk, wlen, fmt, f16feed, nsplit)) var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u, ksplit = nsplit > 1l ? uint(ksplit) : 0u) - if (cm2l) { + if (f16feed) { // native fmt-0 decode-in-load; a split pick reroutes the GEMM into the scratch planes let split = nsplit > 1l if (split) { @@ -70,25 +138,13 @@ def private pf_gemm_enc(raw : VkCommandBuffer; var h : VkHaz; idx : int; fmt : i let ml = cm2_tc == 256l let key = idx * 16 + (ml ? (split ? 9 : 8) : (split ? 11 : 10)) if (!key_exists(g_rd.pf_cls_gemm, key)) { - if (ml) { - verify(ensure_q8_batch_cm2l_cls(), "vk prefill: the cm2 l-tile class rail must engage") - g_rd.pf_cls_gemm[key] = set_q8_batch_cm2l_cls( - fixed_array(wq, ws, g_rd.pf_meta[idx].buf, ab, yob), - fixed_array(wqn, wsn, BATCH_META_BYTES, an, yon), - fixed_array(0u, 0u, 0u, abit, yog)) - } else { - verify(ensure_q8_batch_cm2m_cls(), "vk prefill: the cm2 m-tile class rail must engage") - g_rd.pf_cls_gemm[key] = set_q8_batch_cm2m_cls( - fixed_array(wq, ws, g_rd.pf_meta[idx].buf, ab, yob), - fixed_array(wqn, wsn, BATCH_META_BYTES, an, yon), - fixed_array(0u, 0u, 0u, abit, yog)) - } - } - if (ml) { - enc_q8_batch_cm2l_cls(raw, h, g_rd.pf_cls_gemm[key], pc, groups) - } else { - enc_q8_batch_cm2m_cls(raw, h, g_rd.pf_cls_gemm[key], pc, groups) + verify(cm2_cls_ensure(fmt, ml), "vk prefill: the cm2 tile class rail must engage") + g_rd.pf_cls_gemm[key] = cm2_cls_set(fmt, ml, + fixed_array(wq, ws, g_rd.pf_meta[idx].buf, ab, yob), + fixed_array(wqn, wsn, BATCH_META_BYTES, an, yon), + fixed_array(0u, 0u, 0u, abit, yog)) } + cm2_cls_enc(fmt, ml, raw, h, g_rd.pf_cls_gemm[key], pc, groups) if (split) { let rkey = idx * 16 + 12 if (!key_exists(g_rd.pf_cls_gemm, rkey)) { @@ -128,7 +184,7 @@ def private pf_gemm_enc(raw : VkCommandBuffer; var h : VkHaz; idx : int; fmt : i pfq_ts(raw) } -[cold_path] // once per residency — buffers, sets, fa arming +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] // once per residency — buffers, sets, fa arming def private pf_setup { if (g_rd.pf_ready) { return @@ -146,7 +202,7 @@ def private pf_setup { g_rd.pf_xs = make_device_buf(np * (wide / 32l) * 4l) g_rd.pf_q = make_device_buf(np * qd * 4l) g_rd.pf_kv = make_device_buf(np * 2l * kvd * 4l) - g_rd.pf_v = make_device_buf(np * kvd * 4l) // v GEMM output (copied into pf_kv's v half; the batch GEMM can't offset its output independently of its input row) + g_rd.pf_v = make_device_buf(np * kvd * 4l) // split-path v GEMM output (copied into pf_kv's v half); idle when the merged k|v GEMM serves the layer g_rd.pf_attn = make_device_buf(np * qd * 4l) g_rd.pf_aq = make_device_buf(np * qd) g_rd.pf_as = make_device_buf(np * (qd / 32l) * 4l) @@ -157,6 +213,29 @@ def private pf_setup { g_rd.pf_cos = make_device_buf(np * g_rd.head_size * 4l) g_rd.pf_logits_host = make_host_buf(g_rd.vocab * 4l, false, [cached = true]) g_rd.pf_cmd = alloc_cmd() + if (g_env_vulkan.vk_overlap) { + let nring = 3l + (g_rd.n_layers + PF_CHUNK_MAX - 1l) / PF_CHUNK_MAX + g_rd.pf_cmd_ring |> reserve(nring) + for (_i in range64(nring)) { + g_rd.pf_cmd_ring |> push(alloc_cmd()) + } + } + if (g_rd.emb_blk >= 0l) { + g_rd.pf_ids = make_device_buf(np * 4l) + verify(ensure_emb_gather_cls(), "vk prefill: the embed-gather class rail must engage") + let pe = arena_planes(int(KqFmt.q8), g_rd.emb_blk) + g_rd.pf_emb_set = set_emb_gather_cls( + fixed_array(pe.wq, pe.ws, g_rd.pf_ids, g_rd.pf_x), + fixed_array(pe.wqb, pe.wsb, np * 4l, np * dim * 4l), + fixed_array(0u, 0u, 0u, VHZ_X)) + } elif (g_rd.emb_f32 != 0ul) { + g_rd.pf_ids = make_device_buf(np * 4l) + verify(ensure_emb_gather_f32_cls(), "vk prefill: the f32 embed-gather class rail must engage") + g_rd.pf_emb_set = set_emb_gather_f32_cls( + fixed_array(g_rd.emb_f32, g_rd.pf_ids, g_rd.pf_x), + fixed_array(g_rd.emb_f32_bytes, np * 4l, np * dim * 4l), + fixed_array(0u, 0u, VHZ_X)) + } ensure_batch_state() // hq_dev/hs_dev for the FFN act // f16-fed groups (pf_f16_feed) take f16 convert roles; kq consumers the Q8_K quant/act forms var any6 = false @@ -185,6 +264,7 @@ def private pf_setup { } let nsets = int(g_rd.n_layers * PF_ROLES + 2l) // + prologue, final quantize of the last row g_rd.pf_sets |> resize(nsets) + g_rd.pf_arq_sets |> resize(g_rd.n_layers * 2l) g_rd.pf_meta |> resize(nsets) for (mi in range(nsets)) { g_rd.pf_meta[mi] = make_host_buf(BATCH_META_BYTES, true) @@ -196,25 +276,36 @@ def private pf_setup { g_rd.pf_facm2 = (g_gpu.has_coopmat2_fa && vk_fa_on() && kv16 && (g_rd.head_size == 64l || g_rd.head_size == 128l)) let fa128s = !g_rd.pf_facm2 && g_gpu.has_coopmat && g_rd.head_size == 128l && vk_fa_on() - verify(ensure_cls_dn_rq() && ensure_cls_ar() + verify(ensure_cls_dn_rq() && ensure_cls_ar() && ensure_cls_ar_rq_b() && (kv16 ? ensure_rope_kv_b_f16_cls() : ensure_rope_kv_b_cls()) && (fa128s ? (kv16 ? ensure_da_attn_b_h128_f16_cls() : ensure_da_attn_b_h128_cls()) : (kv16 ? ensure_da_attn_b_f16_cls() : ensure_da_attn_b_cls())) && (!g_rd.pf_facm2 || (g_rd.head_size == 64l ? ensure_fa_cm2_h64_cls() : ensure_fa_cm2_h128_cls())) + && (!g_rd.pf_facm2 || !any6 + || (g_rd.head_size == 64l ? ensure_fa_cm2_h64_f16_cls() : ensure_fa_cm2_h128_f16_cls())) && ensure_q8_actrq_cls() && (!g_rd.qk_norm || ensure_qk_rms_cls()) - && (!any6 || (ensure_f16cvt_cls() && ensure_actf16_cls())) + && (!any6 || (ensure_f16cvt_cls() && ensure_actf16_cls() && ensure_cls_ar_f16_b())) && (!anykq || (ensure_cls_q8k_rq() && ensure_q8k_actrq_cls())), "vk prefill: the class rails must engage on a live device") if (g_rd.pf_facm2) { to_log(LOG_INFO, "dasLLAMA vulkan tier: cm2 flash attention armed (hs {g_rd.head_size}, native f16 mirrors)\n") } + var nkvm = 0l + for (l in range64(g_rd.n_layers)) { + if (pf_kv_merged(l, dim, kvd)) { + nkvm++ + } + } + if (nkvm > 0l) { + to_log(LOG_INFO, "dasLLAMA vulkan tier: merged k/v prefill GEMM armed ({nkvm} of {g_rd.n_layers} layers; DASLLAMA_VK_KV_MERGE=0 pins the split pair)\n") + } for (l in range64(g_rd.n_layers)) { var L & = unsafe(g_rd.layers[l]) let b = int(l * PF_ROLES) - let qkv6 = pf_f16_feed(L.fq) + let qkv6 = pf_qkv6(l) let wo6 = pf_f16_feed(L.fo) - let gu6 = pf_f16_feed(L.f1) + let gu6 = pf_gu6(l) let dn6 = pf_f16_feed(L.f2) // 0 quant_xb: pf_xb -> pf_xq/pf_xs (f16-fed: f16 convert -> pf_xf) g_rd.pf_sets[b + 0] = (qkv6 @@ -234,10 +325,13 @@ def private pf_setup { } // 5 attn (class: q, Kmir, Vmir, out) — the cm2 fa tile / h128 coopmat twin when they serve if (g_rd.pf_facm2) { - g_rd.pf_sets[b + 5] = set_fa_cm2_cls( - fixed_array(g_rd.pf_q, g_rd.k_mirror, g_rd.v_mirror, g_rd.pf_attn), - fixed_array(np * qd * 4l, mirbytes, mirbytes, np * qd * 4l), - fixed_array(VHZ_Q, VHZ_MIR, VHZ_MIR, VHZ_ATT)) + g_rd.pf_sets[b + 5] = (wo6 + ? set_fa_cm2_cls(fixed_array(g_rd.pf_q, g_rd.k_mirror, g_rd.v_mirror, g_rd.pf_af), + fixed_array(np * qd * 4l, mirbytes, mirbytes, np * qd * 2l), + fixed_array(VHZ_Q, VHZ_MIR, VHZ_MIR, VHZ_AQ)) + : set_fa_cm2_cls(fixed_array(g_rd.pf_q, g_rd.k_mirror, g_rd.v_mirror, g_rd.pf_attn), + fixed_array(np * qd * 4l, mirbytes, mirbytes, np * qd * 4l), + fixed_array(VHZ_Q, VHZ_MIR, VHZ_MIR, VHZ_ATT))) } else { let abufs5 = fixed_array(g_rd.pf_q, g_rd.k_mirror, g_rd.v_mirror, g_rd.pf_attn) let asizes5 = fixed_array(np * qd * 4l, mirbytes, mirbytes, np * qd * 4l) @@ -276,6 +370,23 @@ def private pf_setup { g_rd.pf_sets[b + 14] = set_cls_ar(fixed_array(g_rd.pf_x, g_rd.pf_ffnout, g_rd.norms_dev, g_rd.pf_xb), fixed_array(np * dim * 4l, np * dim * 4l, normbytes, np * dim * 4l), fixed_array(VHZ_X, VHZ_FFO, 0u, VHZ_XB)) + g_rd.pf_arq_sets[int(l * 2l)] = (gu6 + ? set_cls_ar_f16_b(fixed_array(g_rd.pf_x, g_rd.pf_xb2, g_rd.norms_dev, g_rd.pf_xf), + fixed_array(np * dim * 4l, np * dim * 4l, normbytes, np * dim * 2l), + fixed_array(VHZ_X, VHZ_XB2, 0u, VHZ_XQ)) + : set_cls_ar_rq_b( + fixed_array(g_rd.pf_x, g_rd.pf_xb2, g_rd.norms_dev, g_rd.pf_xq, g_rd.pf_xs), + fixed_array(np * dim * 4l, np * dim * 4l, normbytes, np * dim, np * (dim / 32l) * 4l), + fixed_array(VHZ_X, VHZ_XB2, 0u, VHZ_XQ, VHZ_XQ))) + let next_qkv6 = l + 1l < g_rd.n_layers && pf_qkv6(l + 1l) + g_rd.pf_arq_sets[int(l * 2l + 1l)] = (next_qkv6 + ? set_cls_ar_f16_b(fixed_array(g_rd.pf_x, g_rd.pf_ffnout, g_rd.norms_dev, g_rd.pf_xf), + fixed_array(np * dim * 4l, np * dim * 4l, normbytes, np * dim * 2l), + fixed_array(VHZ_X, VHZ_FFO, 0u, VHZ_XQ)) + : set_cls_ar_rq_b( + fixed_array(g_rd.pf_x, g_rd.pf_ffnout, g_rd.norms_dev, g_rd.pf_xq, g_rd.pf_xs), + fixed_array(np * dim * 4l, np * dim * 4l, normbytes, np * dim, np * (dim / 32l) * 4l), + fixed_array(VHZ_X, VHZ_FFO, 0u, VHZ_XQ, VHZ_XQ))) // 15 qk rmsnorm (qk_norm models; class: q, k rows, norm rows) if (g_rd.qk_norm) { g_rd.pf_sets[b + 15] = set_qk_rms_cls(fixed_array(g_rd.pf_q, g_rd.pf_kv, g_rd.norms_dev), @@ -299,7 +410,57 @@ def private pf_setup { //! (cos_batch, [npos x hs]) -> the last row's logits, filling the KV mirror at positions [0, npos). [hot_path] // the per-window resident prefill — encode + one submit per window def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int64; var logits : array) { - assert(g_rd != null && g_rd.ready, "vk_rdec_prefill before prepare/set_cls") + let noids : array + pf_run(x_batch, noids, false, 1.0, cos_batch, npos, logits) +} + +[hot_path, arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def vk_rdec_prefill_ids(tokens : array; emb_scale : float; cos_batch : array; npos : int64; var logits : array) { + assert(g_rd != null && (g_rd.emb_blk >= 0l || g_rd.emb_f32 != 0ul), "vk_rdec_prefill_ids without a placed embd plane") + let nox : array + pf_run(nox, tokens, true, emb_scale, cos_batch, npos, logits) +} + +//! False when the gather rail cannot build on this device (the caller keeps the CPU embed) +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def vk_rdec_set_emb(emb_block : int64) : bool { + if (g_gpu != null && g_gpu.dry) { + return true + } + assert(g_rd != null, "vk_rdec_set_emb before prepare") + if (!ensure_emb_gather_cls()) { + return false + } + g_rd.emb_blk = emb_block + return true +} + +//! False past RDEC_EMB_F32_CAP, the device's SSBO range, or the weight budget, or when the gather +//! rail cannot build (the caller keeps the CPU embed) +[arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] +def vk_rdec_upload_emb_f32(fblob : array; off, vocab, dim : int64) : bool { + if (g_gpu != null && g_gpu.dry) { + return false + } + assert(g_rd != null, "vk_rdec_upload_emb_f32 before prepare") + let bytes = vocab * dim * 4l + if (bytes > min(RDEC_EMB_F32_CAP, vk_max_storage_range()) || g_gpu.resident_bytes + bytes > g_gpu.weight_budget + || !ensure_emb_gather_f32_cls()) { + return false + } + g_rd.emb_f32 = make_device_buf(bytes) + g_rd.emb_f32_bytes = bytes + g_gpu.resident_bytes += bytes + unsafe { + upload_region_at(g_rd.emb_f32, 0l, addr(fblob[int(off)]), bytes) + } + return true +} + +[hot_path, arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] +def private pf_run(x_batch : array; ids : array; use_ids : bool; emb_scale : float; + cos_batch : array; npos : int64; var logits : array) { + assert(g_rd != null && g_rd.ready, "vk prefill before prepare/set_cls") pf_setup() let dim = g_rd.dim let qd = g_rd.qd @@ -308,15 +469,24 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let nlfin = g_rd.n_layers * 2l * dim let pairs = int(qd / 2l + kvd / 2l) unsafe { - // prompts over the PF_WINDOW activation buffers run as sequential windows: each window's - // rope/attention address the mirror at absolute positions, so window w attends everything - // [0, w0 + row] the earlier windows stored. Only the last window runs fin_rq + cls. var w0 = 0l while (w0 < npos) { let tp0 = ref_time_ticks() let wlen = min(g_rd.pf_np, npos - w0) let last = w0 + wlen >= npos - upload_region_at(g_rd.pf_x, 0l, addr(x_batch[int(w0 * dim)]), wlen * dim * 4l) + if (use_ids) { + g_pf_ids_stage |> resize(wlen) + for (i in range64(wlen)) { + let id = ids[w0 + i] + if (id < 0l || id >= g_rd.vocab) { // fail closed loudly, as the CPU's dequant_q8_row does on the same input + panic("vk prefill: token id {id} outside the vocab ({g_rd.vocab}) at row {w0 + i}") + } + g_pf_ids_stage[int(i)] = uint(id) + } + upload_region_at(g_rd.pf_ids, 0l, addr(g_pf_ids_stage[0]), wlen * 4l) + } else { + upload_region_at(g_rd.pf_x, 0l, addr(x_batch[int(w0 * dim)]), wlen * dim * 4l) + } upload_region_at(g_rd.pf_cos, 0l, addr(cos_batch[int(w0 * g_rd.head_size)]), wlen * g_rd.head_size * 4l) let bp = int(g_rd.n_layers * PF_ROLES) let t_prep = get_time_usec(tp0) @@ -334,32 +504,62 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let fa128 = !g_rd.pf_facm2 && g_gpu.has_coopmat && g_rd.head_size == 128l && vk_fa_on() let attnwg = fa128 ? g_rd.n_heads * ((wlen + 31l) / 32l) : g_rd.n_heads * ((wlen + 7l) / 8l) let hs = g_rd.head_size + if (use_ids) { + // no extra pfq_ts - the gather cost folds into the prologue stamp + var pce = EmbArgs(npos = uint(wlen), dim = uint(dim), + wblk0 = g_rd.emb_blk >= 0l ? uint(arena_local_blk(g_rd.emb_blk)) : 0u, embed_scale = emb_scale) + if (g_rd.emb_blk >= 0l) { + enc_emb_gather_cls(raw, h, g_rd.pf_emb_set, pce, (wlen * dim / 4l + 255l) / 256l) + } else { + enc_emb_gather_f32_cls(raw, h, g_rd.pf_emb_set, pce, (wlen * dim / 4l + 255l) / 256l) + } + } var pc_pro = ArArgs(dim = uint(dim), add_on = 0u, woff = 0u, eps = g_rd.eps, ascale = 1.0) enc_cls_ar(raw, h, g_rd.pf_sets[bp + 0], pc_pro, wlen) // prologue norm pfq_ts(raw) + let overlap = g_env_vulkan.vk_overlap && !vk_prof() + let fuse_arq = g_env_vulkan.vk_fuse + var nextb = 1l + var cstep = 1l + var nchunk = 0 for (l in range64(g_rd.n_layers)) { + if (overlap && l == nextb) { + vk_check(vkEndCommandBuffer(raw), null) + submit_nofence(raw) + assert(nchunk < length(g_rd.pf_cmd_ring), "vk prefill: overlap ring exhausted") + raw = g_rd.pf_cmd_ring[nchunk] + nchunk++ + let rfc : VkCommandBufferResetFlags + vk_check(vkResetCommandBuffer(raw, rfc), null) + let beginc = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, beginc), null) + cstep = min(cstep * 2l, PF_CHUNK_MAX) + nextb += cstep + } let b = int(l * PF_ROLES) - // f16-fed roles (fmt-6, and fmt-0 under mode 4's l-tile): f32->f16 convert feeds - // the decode-in-load GEMM; the fill's tile rule keys off the same group flag - let fq6 = pf_f16_feed(g_rd.layers[l].fq) + // f16-fed roles (mode 4's cm2 tiles): the f32->f16 convert feeds decode-in-load; the fill's tile rule keys off the same flag + let fq6 = pf_qkv6(l) let fo6 = pf_f16_feed(g_rd.layers[l].fo) - let gu6 = pf_f16_feed(g_rd.layers[l].f1) + let gu6 = pf_gu6(l) let dn6 = pf_f16_feed(g_rd.layers[l].f2) - // kq consumers take the Q8_K quant/act forms (the arm declines mixed groups) - let fqk = kq_sb(g_rd.layers[l].fq) - let fok = kq_sb(g_rd.layers[l].fo) - let guk = kq_sb(g_rd.layers[l].f1) - let dnk = kq_sb(g_rd.layers[l].f2) + // kq consumers take the Q8_K quant/act forms — unless the f16 feed serves them (k4 cm2) + let fqk = kq_sb(g_rd.layers[l].fq) && !fq6 + let fok = kq_sb(g_rd.layers[l].fo) && !fo6 + let guk = kq_sb(g_rd.layers[l].f1) && !gu6 + let dnk = kq_sb(g_rd.layers[l].f2) && !dn6 let mirb = l * g_rd.seq_cap * kvd // nolint:LINT021 — the mirror offset multiply must run 64-bit before the push-field narrow - if (fq6) { - var pc0 = ActArgs(nelem = uint(wlen * dim), gelu = 0u, nblk = 0u) - enc_f16cvt_cls(raw, h, g_rd.pf_sets[b + 0], pc0, (wlen * dim / 4l + 255l) / 256l) - } elif (fqk) { - var pc0 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 256l)) - enc_cls_q8k_rq(raw, h, g_rd.pf_sets[b + 0], pc0, (wlen * dim / 8l + 255l) / 256l) - } else { - var pc0 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 32l)) - enc_cls_dn_rq(raw, h, g_rd.pf_sets[b + 0], pc0, (wlen * dim / 4l + 255l) / 256l) + let feed_already_fused = l > 0l && pf_qkv_feed_fused(l) + if (!feed_already_fused) { + if (fq6) { + var pc0 = ActArgs(nelem = uint(wlen * dim), gelu = 0u, nblk = 0u) + enc_f16cvt_cls(raw, h, g_rd.pf_sets[b + 0], pc0, (wlen * dim / 4l + 255l) / 256l) + } elif (fqk) { + var pc0 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 256l)) + enc_cls_q8k_rq(raw, h, g_rd.pf_sets[b + 0], pc0, (wlen * dim / 8l + 255l) / 256l) + } else { + var pc0 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 32l)) + enc_cls_dn_rq(raw, h, g_rd.pf_sets[b + 0], pc0, (wlen * dim / 4l + 255l) / 256l) + } } pfq_ts(raw) let pq = arena_planes(g_rd.layers[l].fq, g_rd.layers[l].bq) @@ -371,16 +571,23 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let xqn = fq6 ? np * dim * 2l : np * dim let xsn = fq6 ? 256l : np * (dim / 32l) * 4l pf_gemm_enc(raw, h, b + 1, g_rd.layers[l].fq, pq.wq, pq.ws, xqb, xsb, g_rd.pf_q, - pq.wqb, pq.wsb, xqn, xsn, np * qd * 4l, VHZ_XQ, VHZ_Q, dim, qd, g_rd.layers[l].bq, wlen) - pf_gemm_enc(raw, h, b + 2, g_rd.layers[l].fk, pk.wq, pk.ws, xqb, xsb, g_rd.pf_kv, - pk.wqb, pk.wsb, xqn, xsn, np * 2l * kvd * 4l, VHZ_XQ, VHZ_KVK, dim, kvd, g_rd.layers[l].bk, wlen) - pf_gemm_enc(raw, h, b + 3, g_rd.layers[l].fv, pv.wq, pv.ws, xqb, xsb, g_rd.pf_v, - pv.wqb, pv.wsb, xqn, xsn, np * kvd * 4l, VHZ_XQ, VHZ_VST, dim, kvd, g_rd.layers[l].bv, wlen) - vhz_dep(raw, h, VHZ_VST, VHZ_KVV, true) - cmd_copy_range(raw, g_rd.pf_v, 0l, g_rd.pf_kv, wlen * kvd * 4l, wlen * kvd * 4l) // v into pf_kv's v half + pq.wqb, pq.wsb, xqn, xsn, np * qd * 4l, VHZ_XQ, VHZ_Q, dim, qd, g_rd.layers[l].bq, wlen, fq6) + let kv_merged = pf_kv_merged(l, dim, kvd) + if (kv_merged) { + pf_gemm_enc(raw, h, b + 2, g_rd.layers[l].fk, pk.wq, pk.ws, xqb, xsb, g_rd.pf_kv, + pk.wqb, pk.wsb, xqn, xsn, np * 2l * kvd * 4l, VHZ_XQ, VHZ_KVK | VHZ_KVV, dim, 2l * kvd, g_rd.layers[l].bk, wlen, fq6) + pfq_ts(raw) // the v stamp: the merged GEMM's cost lands on k's delta + } else { + pf_gemm_enc(raw, h, b + 2, g_rd.layers[l].fk, pk.wq, pk.ws, xqb, xsb, g_rd.pf_kv, + pk.wqb, pk.wsb, xqn, xsn, np * 2l * kvd * 4l, VHZ_XQ, VHZ_KVK, dim, kvd, g_rd.layers[l].bk, wlen, fq6) + pf_gemm_enc(raw, h, b + 3, g_rd.layers[l].fv, pv.wq, pv.ws, xqb, xsb, g_rd.pf_v, + pv.wqb, pv.wsb, xqn, xsn, np * kvd * 4l, VHZ_XQ, VHZ_VST, dim, kvd, g_rd.layers[l].bv, wlen, fq6) + vhz_dep(raw, h, VHZ_VST, VHZ_KVV, true) + cmd_copy_range(raw, g_rd.pf_v, 0l, g_rd.pf_kv, wlen * kvd * 4l, wlen * kvd * 4l) // v into pf_kv's v half + } if (g_rd.qk_norm) { var pcn = QkRmsArgs(hs = uint(hs), nh = uint(g_rd.n_heads), nkvh = uint(kvd / hs), - qstride = uint(qd), kstride = uint(kvd), kbase = 0u, + qstride = uint(qd), kstride = uint(kv_merged ? 2l * kvd : kvd), kbase = 0u, qwoff = uint(nlfin + dim + l * 2l * hs), kwoff = uint(nlfin + dim + (l * 2l + 1l) * hs), eps = g_rd.eps) enc_qk_rms_cls(raw, h, g_rd.pf_sets[b + 15], pcn, wlen * (g_rd.n_heads + kvd / hs)) @@ -388,7 +595,8 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int } var pcr = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(hs / 2l), neox = g_rd.neox ? 1u : 0u, kpair0 = uint(qd / 2l), ppp = uint(pairs), - layerbase = uint(mirb), npos = uint(wlen), pos0 = uint(w0), voff = uint(wlen * kvd)) + layerbase = uint(mirb), npos = uint(wlen), pos0 = uint(w0), + voff = uint(kv_merged ? kvd : wlen * kvd), kstride = uint(kv_merged ? 2l * kvd : kvd)) if (g_rd.kv16) { enc_rope_kv_b_f16_cls(raw, h, g_rd.pf_sets[b + 4], pcr, ropewg) } else { @@ -402,9 +610,17 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int scale = g_rd.scale) let fawgs = g_rd.n_heads * ((wlen + 63l) / 64l) if (hs == 64l) { - enc_fa_cm2_h64_cls(raw, h, g_rd.pf_sets[b + 5], pcf, fawgs) + if (fo6) { + enc_fa_cm2_h64_f16_cls(raw, h, g_rd.pf_sets[b + 5], pcf, fawgs) + } else { + enc_fa_cm2_h64_cls(raw, h, g_rd.pf_sets[b + 5], pcf, fawgs) + } } else { - enc_fa_cm2_h128_cls(raw, h, g_rd.pf_sets[b + 5], pcf, fawgs) + if (fo6) { + enc_fa_cm2_h128_f16_cls(raw, h, g_rd.pf_sets[b + 5], pcf, fawgs) + } else { + enc_fa_cm2_h128_cls(raw, h, g_rd.pf_sets[b + 5], pcf, fawgs) + } } } else { var pca = DaAttnBArgs(npos = uint(wlen), nh = uint(g_rd.n_heads), kvd = uint(kvd), @@ -425,7 +641,9 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int } } pfq_ts(raw) - if (fo6) { + let attn_landed_f16 = fo6 && g_rd.pf_facm2 + if (attn_landed_f16) { + } elif (fo6) { var pc6 = ActArgs(nelem = uint(wlen * qd), gelu = 0u, nblk = 0u) enc_f16cvt_cls(raw, h, g_rd.pf_sets[b + 6], pc6, (wlen * qd / 4l + 255l) / 256l) } elif (fok) { @@ -442,19 +660,28 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let aqn = fo6 ? np * qd * 2l : np * qd let asn = fo6 ? 256l : np * (qd / 32l) * 4l pf_gemm_enc(raw, h, b + 7, g_rd.layers[l].fo, po.wq, po.ws, aqb, asb, g_rd.pf_xb2, - po.wqb, po.wsb, aqn, asn, np * dim * 4l, VHZ_AQ, VHZ_XB2, qd, dim, g_rd.layers[l].bo, wlen) + po.wqb, po.wsb, aqn, asn, np * dim * 4l, VHZ_AQ, VHZ_XB2, qd, dim, g_rd.layers[l].bo, wlen, fo6) var pc8 = ArArgs(dim = uint(dim), add_on = 1u, woff = uint((l * 2l + 1l) * dim), eps = g_rd.eps, ascale = 1.0) - enc_cls_ar(raw, h, g_rd.pf_sets[b + 8], pc8, wlen) - pfq_ts(raw) - if (gu6) { - var pc9 = ActArgs(nelem = uint(wlen * dim), gelu = 0u, nblk = 0u) - enc_f16cvt_cls(raw, h, g_rd.pf_sets[b + 9], pc9, (wlen * dim / 4l + 255l) / 256l) - } elif (guk) { - var pc9 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 256l)) - enc_cls_q8k_rq(raw, h, g_rd.pf_sets[b + 9], pc9, (wlen * dim / 8l + 255l) / 256l) + if (fuse_arq && !guk) { + if (gu6) { + enc_cls_ar_f16_b(raw, h, g_rd.pf_arq_sets[int(l * 2l)], pc8, wlen) + } else { + enc_cls_ar_rq_b(raw, h, g_rd.pf_arq_sets[int(l * 2l)], pc8, wlen) + } + pfq_ts(raw) } else { - var pc9 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 32l)) - enc_cls_dn_rq(raw, h, g_rd.pf_sets[b + 9], pc9, (wlen * dim / 4l + 255l) / 256l) + enc_cls_ar(raw, h, g_rd.pf_sets[b + 8], pc8, wlen) + pfq_ts(raw) + if (gu6) { + var pc9 = ActArgs(nelem = uint(wlen * dim), gelu = 0u, nblk = 0u) + enc_f16cvt_cls(raw, h, g_rd.pf_sets[b + 9], pc9, (wlen * dim / 4l + 255l) / 256l) + } elif (guk) { + var pc9 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 256l)) + enc_cls_q8k_rq(raw, h, g_rd.pf_sets[b + 9], pc9, (wlen * dim / 8l + 255l) / 256l) + } else { + var pc9 = RqArgs(inbase = 0u, nblk = uint(wlen * dim / 32l)) + enc_cls_dn_rq(raw, h, g_rd.pf_sets[b + 9], pc9, (wlen * dim / 4l + 255l) / 256l) + } } pfq_ts(raw) let p1 = arena_planes(g_rd.layers[l].f1, g_rd.layers[l].b1) @@ -464,9 +691,9 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let gqn = gu6 ? np * dim * 2l : np * dim let gsn = gu6 ? 256l : np * (dim / 32l) * 4l pf_gemm_enc(raw, h, b + 10, g_rd.layers[l].f1, p1.wq, p1.ws, gqb, gsb, g_rd.pf_gate, - p1.wqb, p1.wsb, gqn, gsn, np * hid * 4l, VHZ_XQ, VHZ_GATE, dim, hid, g_rd.layers[l].b1, wlen) + p1.wqb, p1.wsb, gqn, gsn, np * hid * 4l, VHZ_XQ, VHZ_GATE, dim, hid, g_rd.layers[l].b1, wlen, gu6) pf_gemm_enc(raw, h, b + 11, g_rd.layers[l].f3, p3.wq, p3.ws, gqb, gsb, g_rd.pf_up, - p3.wqb, p3.wsb, gqn, gsn, np * hid * 4l, VHZ_XQ, VHZ_UP, dim, hid, g_rd.layers[l].b3, wlen) + p3.wqb, p3.wsb, gqn, gsn, np * hid * 4l, VHZ_XQ, VHZ_UP, dim, hid, g_rd.layers[l].b3, wlen, gu6) if (dn6) { var pc12 = ActArgs(nelem = uint(wlen * hid), gelu = 0u, nblk = uint(wlen * hid / 32l)) enc_actf16_cls(raw, h, g_rd.pf_sets[b + 12], pc12, (wlen * hid / 4l + 255l) / 256l) @@ -484,10 +711,19 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let hqn = dn6 ? np * hid * 2l : np * hid let hsn = dn6 ? 256l : np * (hid / 32l) * 4l pf_gemm_enc(raw, h, b + 13, g_rd.layers[l].f2, p2.wq, p2.ws, hqb, hsb, g_rd.pf_ffnout, - p2.wqb, p2.wsb, hqn, hsn, np * dim * 4l, VHZ_HQ, VHZ_FFO, hid, dim, g_rd.layers[l].b2, wlen) + p2.wqb, p2.wsb, hqn, hsn, np * dim * 4l, VHZ_HQ, VHZ_FFO, hid, dim, g_rd.layers[l].b2, wlen, dn6) let nxt = l + 1l < g_rd.n_layers ? (l + 1l) * 2l * dim : nlfin // nolint:LINT021 — composed from int64 layout offsets var pc14 = ArArgs(dim = uint(dim), add_on = 1u, woff = uint(nxt), eps = g_rd.eps, ascale = 1.0) - enc_cls_ar(raw, h, g_rd.pf_sets[b + 14], pc14, wlen) + let next_qkv6 = l + 1l < g_rd.n_layers && pf_qkv6(l + 1l) + if (l + 1l < g_rd.n_layers && pf_qkv_feed_fused(l + 1l)) { + if (next_qkv6) { + enc_cls_ar_f16_b(raw, h, g_rd.pf_arq_sets[int(l * 2l + 1l)], pc14, wlen) + } else { + enc_cls_ar_rq_b(raw, h, g_rd.pf_arq_sets[int(l * 2l + 1l)], pc14, wlen) + } + } else { + enc_cls_ar(raw, h, g_rd.pf_sets[b + 14], pc14, wlen) + } pfq_ts(raw) } if (last) { @@ -519,9 +755,7 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int w0 += wlen } memcpy(addr(logits[0]), g_rd.pf_logits_host.mapped, g_rd.vocab * 4l) - // per-role GPU aggregation (q0 start, q1 meta, q2 prologue, rpl/layer, final rq, cls), only when - // every stamp landed (deep graphs overflow PFQ_CAP). Multi-window prompts: LAST window only. - // Overlapped levels (q|k|v, gate|up) put the whole level's cost on the LAST member's delta. + // per-role GPU aggregation, LAST window, only when every stamp landed; overlapped levels (q|k|v, gate|up) bill the level on the last member's delta let rpl = g_rd.qk_norm ? 16l : 15l if (vk_prof() && g_pfq_n == 5u + uint(g_rd.n_layers * rpl)) { var role : double[16] @@ -1339,8 +1573,9 @@ def private ensure_at_state { } // one attention window: acts/meta/smalls staging copies, q|gate + k + v tiles, the prep pair -// (deinterleave/qk-rms/rope; k lands at absolute positions), v append + k/v host DMA in the -// transfer window, the flash attention pass, o requant, wo tile, y DMA +// (deinterleave/qk-rms/rope; k lands at absolute positions), v append, the flash attention pass, +// o requant, wo tile, y DMA +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] def private record_at_cmd(s_q, s_k, s_v, s_o : int; nwg_q, nwg_k, nwg_v, nwg_o : uint; rows, w0, qd, kv_dim, dim, n_heads, hs, half, n_kv, kv_mul, o_nblk : int64; axq_bytes, axs_bytes, sm_bytes : int64; o_kq, gated, qknorm : bool; @@ -1372,13 +1607,9 @@ def private record_at_cmd(s_q, s_k, s_v, s_o : int; nwg_q, nwg_k, nwg_v, nwg_o : flags = pflags, rms_off = uint(AT_SM_RMSK), cos_off = uint(AT_SM_COS), sin_off = uint(AT_SM_COS + rows * half), obase = uint(w0), qsrc = 0u, eps = eps) enc_at_prep_cls(raw, h, g_gpu.at_prep_k_set, pck, rows) - // transfer window: append raw v at absolute positions, DMA roped k/raw v to host + // append raw v at absolute positions vhz_dep(raw, h, VHB_VRAW, VHB_ATV, true) cmd_copy_range(raw, g_gpu.at_vraw_dev, 0l, g_gpu.at_v_dev, w0 * kv_dim * 4l, rows * kv_dim * 4l) - vhz_dep(raw, h, VHB_ATK, 0u, true) - cmd_copy_range(raw, g_gpu.at_k_dev, w0 * kv_dim * 4l, g_gpu.at_kv_host.buf, 0l, rows * kv_dim * 4l) - vhz_dep(raw, h, VHB_VRAW, 0u, true) - cmd_copy_range(raw, g_gpu.at_vraw_dev, 0l, g_gpu.at_kv_host.buf, AT_WINDOW * AT_MAX_KV * 4l, rows * kv_dim * 4l) var pca = AtAttnArgs(rows = uint(rows), w0 = uint(w0), qd = uint(qd), kvd = uint(kv_dim), hs = uint(hs), kv_mul = uint(kv_mul), flags = gated ? 1u : 0u, dpl = uint(hs / 32l), scale = scale) enc_at_attn_cls(raw, h, g_gpu.at_attn_set, pca, n_heads * ((rows + 7l) / 8l)) @@ -1392,6 +1623,11 @@ def private record_at_cmd(s_q, s_k, s_v, s_o : int; nwg_q, nwg_k, nwg_v, nwg_o : stack_batch_enc(raw, h, s_o, nwg_o, qd, dim) vhz_dep(raw, h, VHB_Y1, 0u, true) cmd_copy_whole(raw, g_gpu.batch_y1_dev, g_gpu.batch_y1.buf, rows * dim * 4l) + //! the K/V host DMAs stay at the END of the chain (ARCHITECTURE_GPU_VULKAN.md sec.2.2j: a driver drops the in-cmd barrier) + vhz_dep(raw, h, VHB_ATK, 0u, true) + cmd_copy_range(raw, g_gpu.at_k_dev, w0 * kv_dim * 4l, g_gpu.at_kv_host.buf, 0l, rows * kv_dim * 4l) + vhz_dep(raw, h, VHB_VRAW, 0u, true) + cmd_copy_range(raw, g_gpu.at_vraw_dev, 0l, g_gpu.at_kv_host.buf, AT_WINDOW * AT_MAX_KV * 4l, rows * kv_dim * 4l) if (vk_prof()) { to_log(LOG_INFO, "vk_attn hz: {h.disp} nodes, {h.barriers} barriers\n") } diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index 0e2240c7e3..404acfd5b6 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -113,6 +113,139 @@ Ordered roughly by user-visible value; re-rank against zen2 measurements before DASLLAMA_COOPMAT=sdot4|f16|int8|mm|cm2 (+ DASLLAMA_CM2_TILE/DASLLAMA_CM2_SPLITK as A/B instruments); reference side: GGML_VK_DISABLE_COOPMAT / _COOPMAT2 / _COOPMAT2_DECODE_VECTOR (verified in its device-init walk). + Baseline re-pin (2026-08-27, zen2 / 5060 Ti, driver 610.74, debug-jit vs the reference exe + b10659; fa + native f16 mirrors now in the default path): llama-3.2-3B Q8 das + 6644.8 +/- 98.4 pp512 / 105.4 +/- 0.2 tg128 vs 7691.0 +/- 42.7 / 110.0 +/- 0.3 = + 86.4% pp / 95.9% tg (was 75.5% / 95.1% at the 8/06 family walkthrough; upstream itself + did not move b9860 -> b10659). GPU_PROF split of the pp window: ~99% GPU-busy (submit + 75.2 ms, gpu 74.7, prep+record ~2), so the gap is per-GEMM kernel rate, not CPU + serialization - FFN GEMMs are 63% of the window and gate/up run ~43 TFLOP/s where + down/q reach ~50-53; attn is 3% post-fa. Decode sits at the bandwidth ceiling (~358 of + 448 GB/s; theirs ~371). Open probes: (e) capture the reference exe's same-shape dispatch under + ngfx and diff the four counters (ours: tensor 27.4 / L2 45.0 / dram 10.4 / l1tex 19.5); + (f) why the widest GEMMs (gate/up, d=8192) run ~20% below down at the same M. + PROBES (e)+(f) ANSWERED (2026-08-27, `harness/vk_gemm_probe.das` - the mm_a serving + kernel isolated at the 3B role shapes, record-once + timed submits; one arg pins a + single shape as the ngfx capture window): (f) the kernel is SHAPE-UNIFORM - + gate/up 48.3 / down 49.3 / q 49.8 TFLOP/s isolated; the in-situ 43-vs-53 split is + timestamp/stall attribution, not shape behavior. k/v starved ISOLATED (26.5 TFLOP/s at + 32 wgs) but NOT in situ: the chain records q,k,v with no barrier between them, the level + overlaps on the device, and the merged-k|v GEMM (shipped behind DASLLAMA_VK_KV_MERGE, one + dispatch over the adjacent planes at d = 2*kvd) measured a WASH on the 3B + (6571 +/- 78 vs 6555 +/- 68 = +0.24%). The merge stays for the record shortening + (28 dispatches + 28 copies + a barrier per window gone - CPU record cost, the overlap + arc's term), not as a GPU win. mm_a at 48 vs the old cm2-l harness's 28.9 also + re-confirms mm as the right default. (e) their + isolated q8_0 GEMM op (its per-op perf harness, m=4096 n=512 k=14336, their only q8/n=512 + stock case) = 51.5 TFLOP/s - the isolated kernel-rate gap is <= ~6%. Counter diff + THE CM2 CLIFF, FOUND AND FIXED (2026-08-27): our cm2 l-tile ran 28-35 TFLOP/s where + upstream's identical geometry ran 62-66 - root cause was ONE BIT in the SPIR-V emitter: + `coopmatClamp`'s hand-emitted per-element loop carried `OpLoopMerge ... None`; unrolled + (glslang spells the same loop `[[unroll]]`), the driver keeps the wg-scope accumulator in + tensor-register form - left rolled, the dynamic per-element index demotes the coopmat to + addressable storage FOR THE WHOLE KERNEL. Fix: `spirv_emit.das` coopmatClamp loop control + None -> Unroll; min-kernel 34.2 -> 57.8 TFLOP/s (+64%). Hunt instruments (all in + `harness/vk_gemm_probe.das` + envs): `cm2x` arg = decode-cost bisect variants; `their` arg + = the reference exe's glslc-built coopmat2 GEMM blob dispatched in OUR harness via + DASLLAMA_VK_SPV_OVERRIDE (spec constants patched by spirv-opt to the l geometry); + DASLLAMA_VK_SPV_DUMP (new env) = the override's capture half. Bisect ledger (gate shape, + drain-free): their stock 66.0 / their+external-scale-plane 60 / +our-decode-arithmetic 55.4 + / a GLSL twin of OUR minimal kernel 58.1 / our minimal pre-fix 34.7 - so decode spelling + costs ~9% (their 16-bit unpack8 form needs Int8 caps we do not emit) and the interleaved + scale ~5%; everything else was the clamp-loop bit. Post-fix shipped-kernel table + (cnt=512, drain-free): cm2l 58.1/60.3 gate/down BEATS mm 53.7/54.9; with the probe's + write-write barriers cm2l 47.8/40.8 vs mm 48.9/52.0 - the smaller cm2 grids (half mm's + wg count) pay wave-quantization drains. E2E 3B (debug-jit, REBAR=0): mm 7058 +/- 28 pp + (unchanged serving default), mode-4 cm2 6676 +/- 32 - the kernel now wins isolated but + the chain packaging (grid sizes, barrier drains, the f16 staging step) still favors mm; + blunt DASLLAMA_CM2_SPLITK=2 across all GEMMs = 4058 (the reduce tax on well-filled + shapes). Items (b)+(c) are therefore LIVE again: the l/m/split heuristics were tuned + against the 2x-slower kernel and must be re-tuned before the mode-4 default flip. + THE DECODE SPELLING, CLOSED (2026-08-27 late): the last ~15% was the decode-callback + ARITHMETIC FORM. Probe ladder on their kernel, our two-plane data (gate, drain-free): + 16-bit load + unpack8 + [i&1] lane = 62.7 (their stock 63.3 - the external scale plane + costs ~1%); 32-bit word + variable shifts = 55.4; 32-bit unpack8 + dynamic 4-lane + select = 20.3 (VectorExtractDynamic on v4char poisons the block-load path outright). + The driver pattern-matches THEIR EXACT 16-bit spelling. Shipped: VkQ8Blk is int16[16] + and decode_q8 is `unpack8(qs[(cib.y & 30) >> 1])[cib.y & 1]` - the das storage-type + surface (int8/16 SSBO members, unpack8/pack32, Int8/Int16 + storage caps) already + existed golden-tested in dasSpirv (test_storage_8_16); the ONLY additions were the + unpack8(int16/uint16) -> byte2/ubyte2 lingua-franca overloads (zero emitter change) and + the core shaderInt16 device feature across the vulkan_boost storage_8_16 creator family + (+ the storage_8_16_supported gate; the caps validated only by luck before). RESULTS: + cm2l drain-free 62.2/64.7 gate/down = par with their blob, +15-18% over mm; E2E 3B + mode-4 pp 7293.9 +/- 47 - NEW BEST, BEATS mm (7058 +/- 28) by +3.3% = 94.8% of their + cm2 build; tg 104.4 (decode decoupled). tinyllama mode-4 18102 vs mm 19754 (-9%): the + small shapes starve the 128x256 l grids (kv = 4 wgs) - item (b)'s re-tune is what the + (c) default flip waits on, per-model or per-shape. Newly visible after the fix: the + wg_blk0 Workgroup-storage read in decode costs ~9% (lit 51.6 vs full 47.3 at cnt=512 + with barriers) - a push-constant block base for single-region dense dispatches is the + next kernel-side lever. OWED from the storage-type plan: small-int SSBO STORE coverage + (no fixture writes int8/int16 today) - unblocks deleting the hand-rolled word-packing + in every requant writer kernel. + (b)+(c) CLOSED (2026-08-27, commit 57437c8f7): cm2_tile_cols rewritten to a + wave-efficiency comparison (cross-multiplied occupied/allocated wave slots, m only on a + strict win, ties to l) - probe-fit on all 8 role-shape points; tinyllama mode-4 + 18102 -> 20159 on this alone. With that, mode 4 beats mm back-to-back on BOTH serving + models and `resolve_coopmat_mode` now DEFAULTS to cm2 on coopmat2 hardware. Default-path + board vs b10659: 3B 7406.8 +/- 310 pp / 105.7 tg = 96.3%/96.1%; tinyllama + 20188.1 +/- 185 / 294.4 = 99.6% pp (inside their row noise), tg ahead. Mode-4 headroom + still unported: the ar+rq fusion (the fq6 gate skips it), the kvm merge (mode-4 + excluded), the wg_blk0 push-constant base (~9% of the decode callback). + ar fusion PORTED (2026-08-27, commit 4b690e77e): cls_ar_f16_b - the fused add+rms twin's + f16 form, bit-identical to the split cls_ar + f16cvt pair (gated). vk_fuse A/B: 3B pp + 7463 -> 7584 (+1.6%), tg +2.6%; tinyllama pp 19978 -> 20374 (+2.0%), tg +4.0% - both + models' new bests, tinyllama pp now ~100.5% of their row. + wg_blk0 lever DEAD (same day): the cm2x probe grew a `push` variant (base off pa.ksplit) + - push is the SLOWEST spelling (gate/up 49.6 vs full 52.5 vs lit 51.4 TF/s; down 41.7 / + 43.9 / 42.8), and lit no longer beats the shipped form either. The old lit-51.6-vs-47.3 + delta predates the 16-bit decode respelling; with the cheap decode the shared wg_blk0 + read is free. Item closed as measured-no. + kvm merge PORTED to mode 4 (same day, commit 2a4431fb2): the exclusion was pure caution - + pf_gemm_enc is parametric in (d, blk). vk_kv_merge A/B on cm2: 3B pp 7419 -> 7633 (+2.9%), + tinyllama +0.5%. fa f16-out stamp (commit 5267a63b1): FaCm2H64/H128 templated + (OUT16/typedef OT), the O accumulator converts in-kernel and lands the wo feed - the + per-layer b+6 attn->f16 convert never encodes; bit-exact vs the split pair's own device + f16cvt (CPU float16() differs on rounding ties - device converts agree with each other). + A/B: 3B 7669 -> 7737/7708 (+0.7-0.9%), tinyllama 20796 -> 20986 (+0.9%). + END-OF-DAY BOARD vs b10659: 3B pp 7737.2 +/- 67 = 100.6% - AHEAD of the reference exe for the + first time; tinyllama pp 20986 +/- 357 = ~103.5%, tg ahead. 3B tg 105.1 = ~95.5% (decode + chain untouched today). + Small-int STORE ledger CLOSED (2026-08-28, commit a59d095d9): the 8/16-bit store half got + its coverage - a golden fixture (narrowing converts + 8/16-bit access-chain stores, + spirv-val clean) and a live-device exact-bytes cell (test_storage_8_16_store_gpu) - and on + that foundation every Q8 requant writer stores quants as bytes: q8_pack4 and the q8k + butterfly (2 subgroup shuffles per element) deleted, outq members array. Bit-exact + by the gates; perf-neutral where the writers run hot (mm-mode 3B pair 7077 vs 7061, tg + equal). Remaining tail: item (a) K-quant generalization, (d) decode_vector driver-blocked. + Item (a) OPENED with Q4_K (2026-08-28, commit f72694fbe): K4Cm2LBatch/K4Cm2MBatch - the + Q8 tile geometry with a Q4_K decode callback (nibble + per-32-group scale/min off the + repacked planes, (1, 256) layout blocks). Oracle-gated 0-off; probe: 35.8-38.2 TF/s vs + the kq tile's 12.0-12.7 on every Qwen3-4B role shape (~70% of Q8-cm2's rate - the + nibble+scale extraction). Wiring: pf_f16_feed admits k4, the feed flags are GROUP-wide + ANDs (a k6 sibling pins its group to the kq route - Q4_K_M mixes k4+k6 in one group). + Qwen3-4B Q4_K_M mode-3/4 pair: pp 1626 -> 2654 (+63%), tg equal, parity token-exact. + Q6_K tiles LANDED PINNED (same day, commit d89b74681): oracle 0-off on both tiles, but + the rate collapsed to 9.3-13.4 TF/s vs the kq tile's 11.9 - unpinned e2e regressed. + Q6_K CLIFF FOUND AND FIXED (same day, commit 4603a7373): the k6x bisect (nil 59.6 / + flat 39.7 / ql 47.8 / pair 13.4) proved the two-plane 6-bit compose costs only ~33% - + the killer was ONE byte4 DYNAMIC select in the sub-scale extract (unpack8(word)[i&3]), + the same death shape the Q8 chase found; byte2 [i&1] selects are fine. Respelled as + shift + arithmetic-shift sign extension: 12.8 -> 32.9 TF/s. RULE for every future + decode: NEVER index unpack8 of a 32-bit word dynamically - shift+mask, or byte2 [i&1]. + k6 UNPINNED: Qwen3-4B Q4_K_M pp 1626 (mode 3) -> 2669 (k4) -> 3188 (k4+k6) = +96%. + NEXT: k5/q40 stamps (mechanical now the trap is named), then (d) driver-blocked. + (ngfx GPU Trace, our gate loop vs their GEMM loop; counters now read UNELEVATED): + ours tensor 44.6 / L2 54.2 / l1tex 44.9 / dram 15.3, theirs tensor 56.1 / L2 23.8 / + l1tex 27.6 / dram 29.7 - their cm2 keeps the MMA pipe ~26% busier and streams weights + DRAM->MMA with little cache traffic, while our staged L-tile pays L2/L1 bandwidth as + overhead (caveat: their 58.7 MB working set cannot sit in L2, ours ~25 MB can, so the + dram/L2 halves partly reflect working-set size; the tensor-busy delta is the honest + headline). Their HMMA-per-FLOP is ~18% higher than ours (0.140 vs 0.112 per cycle at + only 1.066x the FLOP rate) - unexplained, parked. Decomposition of the 14% pp window + gap: <= ~6% per-GEMM rate + our non-GEMM dispatch chain (~4.4 ms elementwise + ~2 ms + per-dispatch drain across 452 nodes / 367 barriers per window) - so the levers are + epilogue fusion / barrier reduction and the k/v grid, before any cm2 chase. 12. **Arena slabs - the 4 GiB storage-range ceiling (LANDED in-arc 2026-08-06; was the PR gate - the MAIN FACTOR for MoltenVK/M1 enablement, where maxStorageBufferRange is far @@ -176,6 +309,19 @@ Ordered roughly by user-visible value; re-rank against zen2 measurements before the pageable-aware device-local signal WDDM wants - has zero references in the tree. Small addition: enable when present, and consider demoting cold stacks' priority instead of only boosting everything. + MEASURED INCIDENT (2026-08-27, zen2): the ReBAR weight arena (mapped + host-visible|device-local heap, "uploads write direct to VRAM") LOST WDDM residency + mid-session - every weight-reading role fell to PCIe speed (decode 254 ms/token = 13.4 + GB/s exactly; 3B tg 105 -> 3.9, pp 6645 -> 395) while attn/rope/elementwise stayed at + rate and the reference exe in the same minutes stayed healthy (its weights are UNMAPPED + device-local; it also ships priority 1.0 - #17624 - and no pageable extension, no + heartbeat). Priority 1.0 did not hold the mapped heap; `DASLLAMA_VK_REBAR=0` (staged + uploads, unmapped device-local) restored 6584 immediately, same session. The morning + half of the session served the mapped heap at full speed, so the hazard ARMS with some + driver/desktop state (ngfx profiling sessions and the Parsec virtual display both ran + that day). Design consequence to rule on: long-lived weight planes out of the mapped + heap by default (ReBAR kept for transient staging), with this item's runtime priority + as the second layer and a Metal-style residency heartbeat in reserve. 18. **`rsqrt` vs `1.0/sqrt` - the RMS-norm parity spelling (ledgered 2026-08-07, found by the cross-backend similarity audit).** The three rails spell the same inverse norm two @@ -241,3 +387,62 @@ module) is independent and can land any time - it is pure structure. `x`/`y`/`ndim`/`ddim` binding block, differing only in the weight-plane views and the decode. Give those two families a base the way `MetalMoeMulMmBase` already does, so a binding or epilogue fix lands once per family instead of once per variant. + +23. **Command-chain overlap + record-once for the re-recording tiers (ruled 2026-08-10, + post-#3681; parked behind the jit-infra work then - this entry is the durable copy of + that ruling).** The resident DENSE decode ladder already records once per + set_layer/set_cls epoch (`rd_record_token`; `--rerecord-ab` prices the re-encode + delta - it is why decode wins tg). What still re-records: PREFILL (the full window + chain, one submit per window) and the chunked/MoE `g_gpu` tier + (`ffn_gemv_prep`/qkv per token). Upstream re-records everything every graph evaluation + but overlaps CPU recording with GPU execution via incremental submits every + ~200 GFLOP. The plan: (a) prefill overlap - split the window chain into a few + submits, fence at the end, pipeline across windows (also hides `embed_row`); + (b) MoE-tier record-once - routing already rides the `fill_stack_sched_rows` + meta-buffer CONTENT; the two leaks are the stack binding (`find_stack` per token -> + bind the slab union / sched carries the stack id) and the streamed-miss arm (stays a + dynamic prelude, the slow path); CPU top-k is the natural chain split; MoE prefill + grids vary per window -> overlap only there. + ACCEPTANCE (Boris, 2026-08-10): the O0-vs-O3 pp512 delta IS the CPU-on-critical-path + share (measured then on tinyllama: 18200.64 O3 vs 14785.32 O0 = -18.8%, ~6.5 ms + CPU/window; tg free at O0 - record-once decode has no per-token CPU); overlap + succeeds when the two rows CONVERGE. Two bench rows, no profiler, drift-cancelling. + SIZING DATUM (2026-08-27, llama-3.2-3B Q8 GPU_PROF): the 3B prefill window is ~99% + GPU-busy - on small dense models the lever is per-GEMM kernel rate (item 11), not + overlap; overlap pays where per-token CPU still rides the chain (the MoE/chunked + tier, long multi-window prefill, and the O0-class boxes the acceptance test prices). + SHIPPED 2026-08-27 (both halves measured, REBAR=0 protocol, debug-jit): + (a) chunked submits (`DASLLAMA_VK_OVERLAP`, 1,2,4,8-layer ramp, cmd ring, one fence on + the last chunk) - tinyllama pp 18074 -> 18463 (+2.2%), 3B +0.7%: exactly the record + wall, as the GPU_PROF datum predicted; the O0/O3 pair had read 34%/15% CPU share but + most of that is O0-inflated record cost. + (b) device-side embed gather (`DASLLAMA_VK_GPU_EMBED`, the ids-form prefill seam + + engine embed gate with CPU backfill; the q8 arm gathers from the tied cls plane (a + tied Q8 table only), the f32 arm uploads the raw fblob table, 512 MB cap + + budget-guarded) - 3B pp 6664 -> 7062 (+6.0%), tinyllama 18158 -> 19527 (+6.8%); + tg untouched (a one-run tinyllama tg dip re-measured as box state). Footprint of the + trade, by construction: a tied q8 model places nothing (the cls plane is reused - the 3B + case); a raw-f32 table costs vocab x dim x 4 bytes of device memory (tinyllama: 32000 x + 2048 x 4 = 262 MB) and the residency plan counts it before it picks the context cap, so + a box that cannot afford it keeps the CPU embed rather than a shorter context. Decision: + taken - the +6% pp buys the table on every box the plan clears. + (c) the prefill batch ar+rq fusion (`ClsArAddRmsRqB`, one wg per row, verbatim + reduce/amax fold - bit-exact vs the split pair by suite gate; rides `DASLLAMA_VK_FUSE`; + both sites, the last layer keeps split ar for fin_rq's xb) - tinyllama pp 19527 -> + 19989 (+2.4%), 3B a wash (its elementwise share was already small). + DAY-END STANDINGS vs the reference exe b10659 (same box, back-to-back): tinyllama + 19989 +/- 60 pp / 291.9 tg vs 20277 +/- 260 / 291.4 = **98.6% pp (inside their row + noise), tg AT PAR** - the llama family is effectively closed on this box; 3B + 7071 +/- 84 / 105.4 vs 7691 / 110.0 = 91.9% pp / 95.8% tg - the 3B residual is + per-GEMM rate (this item's (e)/(f) counters), not chain shape. Still-serial per + window: cos rows + their upload, prep (~0.45 ms total - the last ~1.4% of tinyllama). + +24. **The cm2 tiles stamp from one class template (ruled 2026-08-28 at the vkclass PR round: + a follow-up PR, not this one).** `Q8Cm2LBatch`/`Q8Cm2MBatch`, `K4Cm2LBatch`/`K4Cm2MBatch` + and `K6Cm2LBatch`/`K6Cm2MBatch` are six hand-stamped bodies over two axes (tile width + 128/256, decode format); `REVIEW_GPU.md`'s twin rule asks for one `class template` with a + `@template_constant` for the width, typedefs for the block/coopmat types, and a + `def override decode_*` per format - the shape `harness/vk_gemm_probe.das`'s `K6PxBase` + already proves. Gate: the six oracle cells in `tests/test_vulkan_kernels.das` stay 0-off, + the probe's l/m rows stay within noise. The k5/q40 stamps (item 11's NEXT) land on the + template, not as two more copies. diff --git a/modules/dasLLAMA/harness/vk_gemm_probe.das b/modules/dasLLAMA/harness/vk_gemm_probe.das new file mode 100644 index 0000000000..e3981b25b4 --- /dev/null +++ b/modules/dasLLAMA/harness/vk_gemm_probe.das @@ -0,0 +1,1367 @@ +options gen2 +options stack = 524288 +options _dasllama_internal = true + +// ATTRIBUTION SWEEP - isolated per-shape rates of the prefill GEMM tiles. +// RefGemmArgs mirrors that blob's push block byte for byte - its constant fields are dead by design. +// Every arm records once, then times over repeated submits; the serving kernels' oracle gate +// is tests/test_vulkan_kernels.das. +// Args: gate|down|q|kv|kvm|qkv|tl pin a shape (the long-window ngfx GPU Trace vehicle); +// k4|k6 the K-quant tiles; cm2x|k6x|ref the bisect and reference arms. +// Shapes default to Llama-3.2-3B Q8 geometry (dim 3072, hidden 8192, kv_dim 1024). + +require math +require vulkan +require vulkan/vulkan_boost +require spirv/spirv_shader +require dasllama/dasllama_vulkan_common +require dasllama/dasllama_vulkan_dispatch +require dasllama/dasllama_vulkan_classes +require dasllama/dasllama_kqformat + +let private MM_TILE = 128 +let private DISPATCHES = 16 +let private SUBMITS = 100 + +def private hash_word(i : uint) : uint { + var x = i * 2654435761u + x ^= x >> 16u + x *= 2246822519u + x ^= x >> 13u + return x +} + +def private cool_f16_pair(i : uint) : uint { + // two packed f16 scales per word, small positive values (the kernel-suite fixture shape) + let h = 0x2c00u + (i % 64u) + return h | (h << 16u) +} + +struct private ShapeBufs { + wq : array + ws : array + xq : array + xs : array + xf : array +} + +def private fill_fixture(var b : ShapeBufs; totblk, rows, nbb : int) { + b.wq |> resize(totblk * 8) + b.ws |> resize(totblk / 2 + 1) + b.xq |> resize(rows * nbb * 8) + b.xs |> resize(rows * nbb) + b.xf |> resize(rows * nbb * 16) + for (i in range(totblk * 8)) { + b.wq[i] = hash_word(uint(i) + 3u) + } + for (i in range(totblk / 2 + 1)) { + b.ws[i] = cool_f16_pair(uint(i) + 7u) + } + for (i in range(rows * nbb * 8)) { + b.xq[i] = hash_word(uint(i) * 13u + 1u) + } + for (i in range(rows * nbb)) { + // cool inputs: the mm family accumulates in f16, hot fixtures overflow 65504 + b.xs[i] = (0.375 + float(i % 9) * 0.0625) * 0.0625 + } + for (i in range(rows * nbb * 16)) { + b.xf[i] = cool_f16_pair(uint(i) * 5u + 11u) + } +} + +// ===== cm2 decode-cost bisect (arg "cm2x") - the shipped l-tile body with the decode method +// varied: nil = constant A (zero decode cost ceiling), flat = quant only (no scale-plane read), +// lit = full decode minus the wg_blk0 shared-memory read. Fast-path no-split arm only ===== + +[vk_dispatch(name = "cm2px_nil", grid = "wgs", params = "wgs : int64")] +class Cm2PxNil : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @role = "alias" @readonly wsh : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode] + def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { + return float16(0.05) + } + + [spirv_kernel(local_size_x = 256, name = "cm2px_nil_spv")] + def run { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 255u) / 256u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + } +} + +[vk_dispatch(name = "cm2px_flat", grid = "wgs", params = "wgs : int64")] +class Cm2PxFlat : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @role = "alias" @readonly wsh : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode] + def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return float16(0.05) * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 256, name = "cm2px_flat_spv")] + def run { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 255u) / 256u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + } +} + +// push = full decode with the scale base off a PUSH CONSTANT (pa.ksplit, 0 here) instead of +// the wg_blk0 shared read - prices the dense-dispatch candidate against lit's literal base +[vk_dispatch(name = "cm2px_push", grid = "wgs", params = "wgs : int64")] +class Cm2PxPush : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @role = "alias" @readonly wsh : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode] + def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return wsh[pa.ksplit + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 256, name = "cm2px_push_spv")] + def run { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 255u) / 256u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + } +} + +[vk_dispatch(name = "cm2px_lit", grid = "wgs", params = "wgs : int64")] +class Cm2PxLit : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @role = "alias" @readonly wsh : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode] + def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return wsh[bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 256, name = "cm2px_lit_spv")] + def run { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 255u) / 256u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + } +} + +// k4 (Q4_K) cm2 decode-in-load vs the serving kq tile - per-shape rate on the qwen role shapes +def private run_k4_shape(name : string; d, n, cnt : int) { // nolint:STYLE038 — one linear measurement sweep + let nsb = n / 256 + let totsb = d * nsb + var wqh : array + var wsuh : array + var xfh : array + var xqh : array + var axsh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize(cnt * n / 2) + xqh |> resize(cnt * nsb * 64) + axsh |> resize(cnt * nsb) + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.00005 * float(1 + sb % 5))) + for (wi in range(4)) { + wsuh[sb * 5 + 1 + wi] = hash_word(uint(sb * 4 + wi) + 613u) + } + } + for (i in range(cnt * n / 2)) { + xfh[i] = cool_f16_pair(uint(i) * 3u + 5u) + } + for (i in range(cnt * nsb * 64)) { + xqh[i] = hash_word(uint(i) * 13u + 1u) + } + for (i in range(cnt * nsb)) { + axsh[i] = 0.0625 + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = long_length(wsuh) * 4l + let xf_bytes = int64(cnt * n) * 2l + let xq_bytes = long_length(xqh) * 4l + let axs_bytes = int64(cnt * nsb) * 4l + let y_bytes = int64(cnt * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let xqd = make_device_buf(xq_bytes) + let axsd = make_device_buf(axs_bytes) + let yd = make_device_buf(y_bytes) + let flop = 2.0lf * double(cnt) * double(d) * double(n) + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + upload_region_at(xqd, 0l, addr(xqh[0]), xq_bytes) + upload_region_at(axsd, 0l, addr(axsh[0]), axs_bytes) + let vnames = fixed_array("k4l ", "k4m ", "k4lnb", "kqnb ") // kqnb: the kq tile dispatches un-barriered, like k4lnb + for (v in range(4)) { + let ttile = v == 1 ? 128 : (v == 3 ? 32 : 256) + let wtile = v == 3 ? 32 : 128 + let wgs = ((cnt + ttile - 1) / ttile) * ((d + wtile - 1) / wtile) + var sched : array + sched |> resize(4 + wgs) + sched[2] = uint(cnt) + let sc_bytes = int64(4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + delete sched + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var hz : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + if (v == 3) { + var s = kq_batch_cls_set_for(int(KqFmt.k4), false, + fixed_array(wqd, wsd, scd, xqd, axsd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xq_bytes, axs_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 8u, 16u)) + for (_k in range(DISPATCHES)) { + var hz2 : VkHaz + enc_kq_batch_k4_cls(raw, hz2, s, pc, int64(wgs)) + } + } else { + var s = (v == 1 + ? set_kq_batch_k4_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k4_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u))) + for (_k in range(DISPATCHES)) { + if (v == 2) { + var hz2 : VkHaz + enc_kq_batch_k4_cm2l_cls(raw, hz2, s, pc, int64(wgs)) + } elif (v == 1) { + enc_kq_batch_k4_cm2m_cls(raw, hz, s, pc, int64(wgs)) + } else { + enc_kq_batch_k4_cm2l_cls(raw, hz, s, pc, int64(wgs)) + } + } + } + vk_check(vkEndCommandBuffer(raw), null) + let us = usec_per_dispatch(raw) + let tf = flop / (us * 1000000.0lf) + print("{name} {vnames[v]}: {us / 1000.0lf} ms/dispatch {tf} TFLOP/s timing-only\n") + } + } + delete wqh + delete wsuh + delete xfh + delete xqh + delete axsh +} + +// ===== k6 decode-spelling bisect (arg "k6x") - the shipped k6 l-tile body with the decode +// varied: nil = constant (ceiling), flat = current compose w/o scale reads, ql = nibble only +// (no qh stream), pair = upstream's 16-bit-pair compose + ONE unpack8 select (v4 form), +// pair16 = the same compose selected through a 2-lane byte2 (the Q8-winning select width) ===== + +// the shared probe base: members + the single-region fast-path l-tile body; leaves override +// decode_k6 (the kq-family tile_shell/stage_w pattern) +class K6PxBase : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @role = "alias" @readonly wsu : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + + [spirv_decode] + def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + return float16(0.05) + } + + def body { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 255u) / 256u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + var tv : tensorView2Dt + tensorViewCreate(tv) + barrier() + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 256u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 8u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + } +} + +[vk_dispatch(name = "k6px_nil", grid = "wgs", params = "wgs : int64")] +class K6PxNil : K6PxBase { + [spirv_kernel(local_size_x = 256, name = "k6px_nil_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "k6px_flat", grid = "wgs", params = "wgs : int64")] +class K6PxFlat : K6PxBase { + [spirv_decode] + def override decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let lo = uint(int(unpack8(blk.ql[int((bu * 16u + j) >> 1u)])[int(j & 1u)])) & 0xFFu + let hby = uint(int(unpack8(blk.qh[int(((bu >> 2u) * 32u + hh * 16u + j) >> 1u)])[int(j & 1u)])) & 0xFFu + let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + return float16(0.001) * float16(float(q6)) + } + + [spirv_kernel(local_size_x = 256, name = "k6px_flat_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "k6px_ql", grid = "wgs", params = "wgs : int64")] +class K6PxQl : K6PxBase { + [spirv_decode] + def override decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let lo = uint(int(unpack8(blk.ql[int((bu * 16u + j) >> 1u)])[int(j & 1u)])) & 0xFFu + let q6 = int((lo >> (hh * 4u)) & 0xFu) - 8 + return float16(0.001) * float16(float(q6)) + } + + [spirv_kernel(local_size_x = 256, name = "k6px_ql_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "k6px_pair", grid = "wgs", params = "wgs : int64")] +class K6PxPair : K6PxBase { + [spirv_decode] + def override decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let qlw = (uint(int(blk.ql[int((bu * 16u + j) >> 1u)])) >> (hh * 4u)) & 0x0F0Fu + let qhw = ((uint(int(blk.qh[int(((bu >> 2u) * 32u + hh * 16u + j) >> 1u)])) >> ((bu & 3u) * 2u)) & 0x0303u) << 4u + let q = int(unpack8(int(qlw | qhw))[int(j & 1u)]) - 32 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let sidx = e >> 4u + let sc = unpack8(int(wsu[srow + (sidx >> 2u)]))[int(sidx & 3u)] + let d = unpackHalf2x16(wsu[srow + 4u]).x + return float16(d * float(int(sc)) * float(q)) + } + + [spirv_kernel(local_size_x = 256, name = "k6px_pair_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "k6px_pair16", grid = "wgs", params = "wgs : int64")] +class K6PxPair16 : K6PxBase { + [spirv_decode] + def override decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let qlw = (uint(int(blk.ql[int((bu * 16u + j) >> 1u)])) >> (hh * 4u)) & 0x0F0Fu + let qhw = ((uint(int(blk.qh[int(((bu >> 2u) * 32u + hh * 16u + j) >> 1u)])) >> ((bu & 3u) * 2u)) & 0x0303u) << 4u + let q = int(unpack8(int16(int(qlw | qhw)))[int(j & 1u)]) - 32 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let sidx = e >> 4u + let sc = unpack8(int(wsu[srow + (sidx >> 2u)]))[int(sidx & 3u)] + let d = unpackHalf2x16(wsu[srow + 4u]).x + return float16(d * float(int(sc)) * float(q)) + } + + [spirv_kernel(local_size_x = 256, name = "k6px_pair16_spv")] + def run { + body() + } +} + +// k6 (Q6_K) cm2 decode-in-load vs the serving kq tile - same instrument as run_k4_shape; +// bisect adds the k6px decode-spelling variants +[arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] +def private run_k6_shape(name : string; d, n, cnt : int; bisect : bool = false) { // nolint:STYLE037,STYLE038 — one linear measurement sweep + let nsb = n / 256 + let totsb = d * nsb + var wqh : array + var wsuh : array + var xfh : array + var xqh : array + var axsh : array + wqh |> resize(totsb * 48) + wsuh |> resize(totsb * 5) + xfh |> resize(cnt * n / 2) + xqh |> resize(cnt * nsb * 64) + axsh |> resize(cnt * nsb) + for (i in range(totsb * 48)) { + wqh[i] = hash_word(uint(i) + 41u) + } + for (sb in range(totsb)) { + for (wi in range(4)) { + wsuh[sb * 5 + wi] = hash_word(uint(sb * 4 + wi) + 811u) + } + wsuh[sb * 5 + 4] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) + } + for (i in range(cnt * n / 2)) { + xfh[i] = cool_f16_pair(uint(i) * 3u + 5u) + } + for (i in range(cnt * nsb * 64)) { + xqh[i] = hash_word(uint(i) * 13u + 1u) + } + for (i in range(cnt * nsb)) { + axsh[i] = 0.0625 + } + let wq_bytes = int64(totsb) * 192l + let ws_bytes = long_length(wsuh) * 4l + let xf_bytes = int64(cnt * n) * 2l + let xq_bytes = long_length(xqh) * 4l + let axs_bytes = int64(cnt * nsb) * 4l + let y_bytes = int64(cnt * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let xqd = make_device_buf(xq_bytes) + let axsd = make_device_buf(axs_bytes) + let yd = make_device_buf(y_bytes) + let flop = 2.0lf * double(cnt) * double(d) * double(n) + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + upload_region_at(xqd, 0l, addr(xqh[0]), xq_bytes) + upload_region_at(axsd, 0l, addr(axsh[0]), axs_bytes) + let vnames = fixed_array("k6l ", "k6m ", "k6lnb", "kqnb ", "xnil ", "xflat", "xql ", "xpair", "xpr16") // the shipped k6 decode IS the sfix spelling the bisect ended on + for (v in range(bisect ? 9 : 4)) { + let ttile = v == 1 ? 128 : (v == 3 ? 32 : 256) + let wtile = v == 3 ? 32 : 128 + let wgs = ((cnt + ttile - 1) / ttile) * ((d + wtile - 1) / wtile) + var sched : array + sched |> resize(4 + wgs) + sched[2] = uint(cnt) + let sc_bytes = int64(4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + delete sched + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var hz : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + if (v >= 4) { + let bufs = fixed_array(wqd, wsd, scd, xfd, yd) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + for (_k in range(DISPATCHES)) { + if (v == 4) { + var s = set_k6px_nil(bufs, sizes, gbits) + enc_k6px_nil(raw, hz, s, pc, int64(wgs)) + } elif (v == 5) { + var s = set_k6px_flat(bufs, sizes, gbits) + enc_k6px_flat(raw, hz, s, pc, int64(wgs)) + } elif (v == 6) { + var s = set_k6px_ql(bufs, sizes, gbits) + enc_k6px_ql(raw, hz, s, pc, int64(wgs)) + } elif (v == 7) { + var s = set_k6px_pair(bufs, sizes, gbits) + enc_k6px_pair(raw, hz, s, pc, int64(wgs)) + } else { + var s = set_k6px_pair16(bufs, sizes, gbits) + enc_k6px_pair16(raw, hz, s, pc, int64(wgs)) + } + } + } elif (v == 3) { + var s = kq_batch_cls_set_for(int(KqFmt.k6), false, + fixed_array(wqd, wsd, scd, xqd, axsd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xq_bytes, axs_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 8u, 16u)) + for (_k in range(DISPATCHES)) { + var hz2 : VkHaz + enc_kq_batch_k6_cls(raw, hz2, s, pc, int64(wgs)) + } + } else { + var s = (v == 1 + ? set_kq_batch_k6_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k6_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u))) + for (_k in range(DISPATCHES)) { + if (v == 2) { + var hz2 : VkHaz + enc_kq_batch_k6_cm2l_cls(raw, hz2, s, pc, int64(wgs)) + } elif (v == 1) { + enc_kq_batch_k6_cm2m_cls(raw, hz, s, pc, int64(wgs)) + } else { + enc_kq_batch_k6_cm2l_cls(raw, hz, s, pc, int64(wgs)) + } + } + } + vk_check(vkEndCommandBuffer(raw), null) + let us = usec_per_dispatch(raw) + let tf = flop / (us * 1000000.0lf) + print("{name} {vnames[v]}: {us / 1000.0lf} ms/dispatch {tf} TFLOP/s timing-only\n") + } + } + delete wqh + delete wsuh + delete xfh + delete xqh + delete axsh +} + +// two warmup submits (pipeline compile + clocks settle), then the timed window +def private usec_per_dispatch(var raw : VkCommandBuffer) : double { + submit_wait(raw) + submit_wait(raw) + let t0 = ref_time_ticks() + for (_s in range(SUBMITS)) { + submit_wait(raw) + } + return double(get_time_usec(t0)) / double(SUBMITS * DISPATCHES) +} + +def private run_shape(name : string; d, n, cnt : int) { // nolint:STYLE038 — one linear measurement sweep + let nbb = n / 32 + let rows = cnt + let wtiles = (d + MM_TILE - 1) / MM_TILE + let wgs = ((cnt + MM_TILE - 1) / MM_TILE) * wtiles + let totblk = d * nbb + var b : ShapeBufs + fill_fixture(b, totblk, rows, nbb) + let wq_bytes = int64(totblk) * 32l + let ws_bytes = int64(totblk) * 2l + let xq_bytes = int64(rows * nbb) * 32l + let xs_bytes = int64(rows * nbb) * 4l + let xf_bytes = int64(rows * nbb) * 64l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xqd = make_device_buf(xq_bytes) + let xsd = make_device_buf(xs_bytes) + let xfd = make_device_buf(xf_bytes) + let yd = make_device_buf(y_bytes) + var sched : array + sched |> resize(4 + wgs) + sched[2] = uint(cnt) // one region: [wblk0 = 0, row0 = 0, cnt, wg0 = 0], map all zeros + var per_us : double + var cm2l_us = 0.0lf + var cm2m_us = 0.0lf + let lwgs = ((cnt + 255) / 256) * ((d + 127) / 128) + let mwgs = ((cnt + 127) / 128) * ((d + 127) / 128) + unsafe { + upload_region_at(wqd, 0l, addr(b.wq[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(b.ws[0]), ws_bytes) + upload_region_at(xqd, 0l, addr(b.xq[0]), xq_bytes) + upload_region_at(xsd, 0l, addr(b.xs[0]), xs_bytes) + upload_region_at(xfd, 0l, addr(b.xf[0]), xf_bytes) + let sc_bytes = int64(4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + let bufs = fixed_array(wqd, wsd, scd, xqd, xsd, yd) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xq_bytes, xs_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 8u, 16u) + var sc = set_q8_batch_mm_a_cls(bufs, sizes, gbits) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var hz : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + for (_k in range(DISPATCHES)) { + enc_q8_batch_mm_a_cls(raw, hz, sc, pc, int64(wgs)) + } + vk_check(vkEndCommandBuffer(raw), null) + per_us = usec_per_dispatch(raw) + if (g_gpu.has_coopmat2) { + // the cm2 tiles at the serving no-split config (f16-fed, decode-in-load) + let cbufs = fixed_array(wqd, wsd, scd, xfd, yd) + let csizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let cgbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var scl = set_q8_batch_cm2l_cls(cbufs, csizes, cgbits) + var rawl = alloc_cmd() + vk_check(vkBeginCommandBuffer(rawl, begin), null) + var hzl : VkHaz + for (_k in range(DISPATCHES)) { + enc_q8_batch_cm2l_cls(rawl, hzl, scl, pc, int64(lwgs)) + } + vk_check(vkEndCommandBuffer(rawl), null) + cm2l_us = usec_per_dispatch(rawl) + var scm = set_q8_batch_cm2m_cls(cbufs, csizes, cgbits) + var rawm = alloc_cmd() + vk_check(vkBeginCommandBuffer(rawm, begin), null) + var hzm : VkHaz + for (_k in range(DISPATCHES)) { + enc_q8_batch_cm2m_cls(rawm, hzm, scm, pc, int64(mwgs)) + } + vk_check(vkEndCommandBuffer(rawm), null) + cm2m_us = usec_per_dispatch(rawm) + } + } + let flop = 2.0lf * double(rows) * double(d) * double(n) + let tflops = flop / (per_us * 1000000.0lf) + let ms = per_us / 1000.0lf + print("{name}: d={d} K={n} cnt={cnt} wgs={wgs} {ms} ms/dispatch {tflops} TFLOP/s timing-only\n") + if (cm2l_us > 0.0lf) { + let ltf = flop / (cm2l_us * 1000000.0lf) + let mtf = flop / (cm2m_us * 1000000.0lf) + print("{name}: cm2l wgs={lwgs} {cm2l_us / 1000.0lf} ms/dispatch {ltf} TFLOP/s timing-only\n") + print("{name}: cm2m wgs={mwgs} {cm2m_us / 1000.0lf} ms/dispatch {mtf} TFLOP/s timing-only\n") + } + delete sched + delete b +} + +// the decode-cost bisect: the shipped cm2l against the three probe variants, one shape +[arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] +def private run_cm2x_shape(name : string; d, n, cnt : int) { // nolint:STYLE038 — one linear measurement sweep + let nbb = n / 32 + let rows = cnt + let totblk = d * nbb + var b : ShapeBufs + fill_fixture(b, totblk, rows, nbb) + let wq_bytes = int64(totblk) * 32l + let ws_bytes = int64(totblk) * 2l + let xf_bytes = int64(rows * nbb) * 64l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd = make_device_buf(y_bytes) + let lwgs = ((cnt + 255) / 256) * ((d + 127) / 128) + var sched : array + sched |> resize(4 + lwgs) + sched[2] = uint(cnt) + let flop = 2.0lf * double(rows) * double(d) * double(n) + unsafe { + upload_region_at(wqd, 0l, addr(b.wq[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(b.ws[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(b.xf[0]), xf_bytes) + let sc_bytes = int64(4 + lwgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + let cbufs = fixed_array(wqd, wsd, scd, xfd, yd) + let csizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let cgbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + let wfd = make_device_buf(wq_bytes * 2l) + upload_region_at(wfd, 0l, addr(b.xf[0]), min(xf_bytes, wq_bytes * 2l)) + let fbufs = fixed_array(wfd, scd, xfd, yd) + let fsizes = fixed_array(wq_bytes * 2l, sc_bytes, xf_bytes, y_bytes) + let fgbits = fixed_array(1u, 0u, 4u, 16u) + let vnames = fixed_array("cm2l", "lit ", "flat", "nil ", "f16a", "nobar", "mmnb ", "push") + let mm_wgs = ((cnt + 127) / 128) * ((d + 127) / 128) + var mmsched : array + mmsched |> resize(4 + mm_wgs) + mmsched[2] = uint(cnt) + let msc_bytes = int64(4 + mm_wgs) * 4l + let mscd = make_device_buf(msc_bytes) + upload_region_at(mscd, 0l, addr(mmsched[0]), msc_bytes) + let xq_bytes = int64(rows * nbb) * 32l + let xs_bytes = int64(rows * nbb) * 4l + let xqd = make_device_buf(xq_bytes) + let xsd = make_device_buf(xs_bytes) + upload_region_at(xqd, 0l, addr(b.xq[0]), xq_bytes) + upload_region_at(xsd, 0l, addr(b.xs[0]), xs_bytes) + delete mmsched + for (v in range(8)) { + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var hz : VkHaz + if (v == 0) { + var s = set_q8_batch_cm2l_cls(cbufs, csizes, cgbits) + for (_k in range(DISPATCHES)) { + enc_q8_batch_cm2l_cls(raw, hz, s, pc, int64(lwgs)) + } + } elif (v == 1) { + var s = set_cm2px_lit(cbufs, csizes, cgbits) + for (_k in range(DISPATCHES)) { + enc_cm2px_lit(raw, hz, s, pc, int64(lwgs)) + } + } elif (v == 2) { + var s = set_cm2px_flat(cbufs, csizes, cgbits) + for (_k in range(DISPATCHES)) { + enc_cm2px_flat(raw, hz, s, pc, int64(lwgs)) + } + } elif (v == 3) { + var s = set_cm2px_nil(cbufs, csizes, cgbits) + for (_k in range(DISPATCHES)) { + enc_cm2px_nil(raw, hz, s, pc, int64(lwgs)) + } + } elif (v == 4) { + var s = set_cm2px_f16a(fbufs, fsizes, fgbits) + for (_k in range(DISPATCHES)) { + enc_cm2px_f16a(raw, hz, s, pc, int64(lwgs)) + } + } elif (v == 5) { + // no-barrier arm: a fresh hazard rail per enc drops the y write-write barrier, + // so consecutive dispatches overlap on device - sizes the per-dispatch drain + var s = set_q8_batch_cm2l_cls(cbufs, csizes, cgbits) + for (_k in range(DISPATCHES)) { + var hz2 : VkHaz + enc_q8_batch_cm2l_cls(raw, hz2, s, pc, int64(lwgs)) + } + } elif (v == 6) { + // the serving mm kernel, same no-barrier treatment - the fair kernel-rate twin + var s = set_q8_batch_mm_a_cls( + fixed_array(wqd, wsd, mscd, xqd, xsd, yd), + fixed_array(wq_bytes, ws_bytes, msc_bytes, xq_bytes, xs_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 8u, 16u)) + for (_k in range(DISPATCHES)) { + var hz2 : VkHaz + enc_q8_batch_mm_a_cls(raw, hz2, s, pc, int64(mm_wgs)) + } + } else { + var s = set_cm2px_push(cbufs, csizes, cgbits) + for (_k in range(DISPATCHES)) { + enc_cm2px_push(raw, hz, s, pc, int64(lwgs)) + } + } + vk_check(vkEndCommandBuffer(raw), null) + let us = usec_per_dispatch(raw) + let tf = flop / (us * 1000000.0lf) + print("{name} {vnames[v]}: {us / 1000.0lf} ms/dispatch {tf} TFLOP/s timing-only\n") + } + } + delete sched + delete b +} + +[vk_dispatch(name = "cm2px_f16a", grid = "wgs", params = "wgs : int64")] +class Cm2PxF16A : MoeCmBase { + @ssbo @binding = 0 @readonly wf : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + + [spirv_kernel(local_size_x = 256, name = "cm2px_f16a_spv")] + def run { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 255u) / 256u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + var tv : tensorView2Dt + tensorViewCreate(tv) + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + let t0 = row0 + xt * 256u + let m0 = wt * 128u + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n & ~7u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, row0 + cnt, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensor(a, wf, 0u, fla, m0, 128u, k, 64u) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensor(a, wf, 0u, fla, m0, 128u, k, 64u) + coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) + } +} + +// ===== the reference coopmat2 GEMM blob in our harness (arg "ref") - the probe class only +// shapes the pipeline layout (3 SSBOs + its 17-word push block); DASLLAMA_VK_SPV_OVERRIDE serves +// the glslc-built, spec-const-patched reference shader in place of the dummy body ===== + +struct RefGemmArgs { + m : uint + n : uint + k : uint + sa : uint + sb : uint + sd : uint + bsa : uint + bsb : uint + bsd : uint + bz : uint + nb : uint + ks : uint + ne02 : uint + ne12 : uint + b2 : uint + b3 : uint + pn : uint +} + +[vk_dispatch(name = "cm2px_ref", grid = "wgs", params = "wgs : int64")] +class Cm2PxRef { + @ssbo @binding = 0 @readonly aq : array + @ssbo @binding = 1 @readonly bx : array + @ssbo @binding = 2 dy : array + @push_constant pa : RefGemmArgs + + [spirv_kernel(local_size_x = 256, name = "cm2px_ref_spv")] + def run { + let i = gl_GlobalInvocationID.x + if (i < pa.m) { + dy[i] = float(bx[i]) + float(aq[i]) + } + } +} + +[vk_dispatch(name = "cm2px_ref2", grid = "wgs", params = "wgs : int64")] +class Cm2PxRef2 { + @ssbo @binding = 0 @readonly aq : array + @ssbo @binding = 1 @readonly bx : array + @ssbo @binding = 2 dy : array + @ssbo @binding = 3 @readonly sc : array + @push_constant pa : RefGemmArgs + + [spirv_kernel(local_size_x = 256, name = "cm2px_ref2_spv")] + def run { + let i = gl_GlobalInvocationID.x + if (i < pa.m) { + dy[i] = float(bx[i]) + float(aq[i]) + float(sc[i]) + } + } +} + +// the minimal our-emitter kernel: fast-path loop only, 256 rows literal, wg.x = weight tile, +// two-plane decode without the workgroup base (the two-plane reference form). Recovery here pins the +// cm2l deficit to the preamble/branch structure; no recovery pins it to per-op emission +[vk_dispatch(name = "cm2px_min", grid = "wgs", params = "wgs : int64")] +class Cm2PxMin { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @readonly wsh : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @push_constant pa : BatchArgs + + [spirv_decode] + def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return wsh[bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 256, name = "cm2px_min_spv")] + def run { + let m0 = gl_WorkGroupID.x * 128u + var tv : tensorView2Dt + tensorViewCreate(tv) + var a : coopmatWgA_f16_128x64 + var b : coopmatWgB_f16_64x256 + var acc : coopmatWgAcc_f16_128x256 + var fla : tensorLayout2D + tensorLayoutCreate(fla) + tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetDimension(fla, pa.d, pa.n) + tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + var flb : tensorLayout2D + tensorLayoutCreate(flb) + tensorLayoutSetDimension(flb, 256u, pa.n) + tensorLayoutSetStride(flb, pa.n & ~7u, 1u) + var flo : tensorLayout2D + tensorLayoutCreate(flo) + tensorLayoutSetDimension(flo, 256u, pa.d) + tensorLayoutSetStride(flo, pa.d & ~7u, 1u) + var k = 0u + for (_i in range(int(pa.n / 512u))) { + for [unroll] (_j in range(8)) { + coopmatLoadTensorDecode(a, wq, 0u, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, 0u, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + } + while (k < pa.n) { + coopmatLoadTensorDecode(a, wq, 0u, fla, m0, 128u, k, 64u, self.decode_q8) + coopmatLoadTensor(b, xf16, 0u, flb, 0u, 256u, k, 64u, tv) + acc = coopmatMulAdd(a, b, acc) + k += 64u + } + coopmatClamp(acc, -65504.0, 65504.0) + var accw : coopmatWgAcc_f32_128x256 + coopmatConvert(accw, acc) + coopmatStoreTensor(accw, y, 0u, flo, 0u, 256u, m0, 128u, tv) + } +} + +// gate shape at cnt = 256 (the reference grid is 2-D; one BN=256 column tile keeps it 1-D): +// the reference interleaved q8 A plane, drain-free timing, our cm2l + mm rows alongside +[arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] +def private run_ref_gemm(d, n, cnt : int) { // nolint:STYLE038 — one linear measurement sweep + let nbb = n / 32 + let totblk = d * nbb + var aw : array + aw |> resize(totblk * 17) + for (blk in range(totblk)) { + aw[blk * 17] = uint16(0x2c00u + (uint(blk) % 64u)) + for (w in range(16)) { + let h = hash_word(uint(blk * 16 + w) + 3u) + aw[blk * 17 + 1 + w] = uint16(h & 0xffffu) + } + } + var xf : array + xf |> resize(cnt * nbb * 16) + for (i in range(cnt * nbb * 16)) { + xf[i] = cool_f16_pair(uint(i) * 5u + 11u) + } + let a_bytes = int64(totblk) * 34l + let b_bytes = int64(cnt * nbb) * 64l + let y_bytes = int64(cnt * d) * 4l + let ad = make_device_buf(a_bytes) + let bd = make_device_buf(b_bytes) + let yd = make_device_buf(y_bytes) + let flop = 2.0lf * double(cnt) * double(d) * double(n) + let wgs = (d + 127) / 128 + unsafe { + upload_region_at(ad, 0l, addr(aw[0]), a_bytes) + upload_region_at(bd, 0l, addr(xf[0]), b_bytes) + var pc = RefGemmArgs(m = uint(d), n = uint(cnt), k = uint(n), + sa = uint(n), sb = uint(n), sd = uint(d), + bsa = uint(d) * uint(n), bsb = uint(cnt) * uint(n), bsd = uint(cnt) * uint(d), + bz = 0u, nb = 1u, ks = uint(n), ne02 = 1u, ne12 = 1u, b2 = 1u, b3 = 1u, pn = uint(cnt)) + var scw : array + scw |> resize(totblk / 2 + 1) + for (i in range(totblk / 2 + 1)) { + scw[i] = cool_f16_pair(uint(i) + 7u) + } + let sc_bytes = int64(totblk) * 2l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(scw[0]), sc_bytes) + delete scw + var s = set_cm2px_ref( + fixed_array(ad, bd, yd), + fixed_array(a_bytes, b_bytes, y_bytes), + fixed_array(1u, 2u, 4u)) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + for (_k in range(DISPATCHES)) { + var hz : VkHaz + enc_cm2px_ref(raw, hz, s, pc, int64(wgs)) + } + vk_check(vkEndCommandBuffer(raw), null) + let us = usec_per_dispatch(raw) + let tf = flop / (us * 1000000.0lf) + print("ref cm2 (d={d} K={n} cnt={cnt} wgs={wgs}): {us / 1000.0lf} ms/dispatch {tf} TFLOP/s timing-only\n") + var s2 = set_cm2px_ref2( + fixed_array(ad, bd, yd, scd), + fixed_array(a_bytes, b_bytes, y_bytes, sc_bytes), + fixed_array(1u, 2u, 4u, 8u)) + var raw2 = alloc_cmd() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + for (_k in range(DISPATCHES)) { + var hz : VkHaz + enc_cm2px_ref2(raw2, hz, s2, pc, int64(wgs)) + } + vk_check(vkEndCommandBuffer(raw2), null) + let us2 = usec_per_dispatch(raw2) + let tf2 = flop / (us2 * 1000000.0lf) + print("ref+2plane: {us2 / 1000.0lf} ms/dispatch {tf2} TFLOP/s timing-only\n") + var s3 = set_cm2px_min( + fixed_array(ad, scd, bd, yd), + fixed_array(a_bytes, sc_bytes, b_bytes, y_bytes), + fixed_array(1u, 2u, 4u, 8u)) + var raw3 = alloc_cmd() + vk_check(vkBeginCommandBuffer(raw3, begin), null) + var pcm = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + for (_k in range(DISPATCHES)) { + var hz : VkHaz + enc_cm2px_min(raw3, hz, s3, pcm, int64(wgs)) + } + vk_check(vkEndCommandBuffer(raw3), null) + let us3 = usec_per_dispatch(raw3) + let tf3 = flop / (us3 * 1000000.0lf) + print("our minimal (min): {us3 / 1000.0lf} ms/dispatch {tf3} TFLOP/s timing-only\n") + } + delete aw + delete xf +} + +[arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] +def private run_probe { // nolint:STYLE037,STYLE038 — the flat per-arg shape dispatcher + if (!ensure_q8_batch_cls()) { + print("no Vulkan device\n") + return + } + if (g_gpu.coopmat_mode < 3) { + print("coopmat mode {g_gpu.coopmat_mode} < 3 - mm tiles not routed on this device\n") + return + } + if (!ensure_q8_batch_mm_a_cls()) { + print("mm_a pipeline failed\n") + return + } + if (g_gpu.has_coopmat2) { + if (!ensure_q8_batch_cm2l_cls() || !ensure_q8_batch_cm2m_cls()) { + print("cm2 pipelines failed\n") + return + } + } + var only = "" + for (a in get_command_line_arguments()) { + if (a == "gate" || a == "down" || a == "q" || a == "kv" || a == "kvm" || a == "qkv" || a == "cm2x" || a == "ref" || a == "tl" || a == "k4" || a == "k6" || a == "k6x") { + only = a + } + } + if (only == "k6x") { + if (!g_gpu.has_coopmat2) { + print("no coopmat2 on this device\n") + return + } + verify(ensure_kq_batch_k6_cm2l_cls() && ensure_kq_batch_k6_cm2m_cls() + && kq_batch_cls_ensure(int(KqFmt.k6), false) + && ensure_k6px_nil() && ensure_k6px_flat() && ensure_k6px_ql() + && ensure_k6px_pair() && ensure_k6px_pair16(), "k6x probe pipelines must engage") + run_k6_shape("q6k gate", 9728, 2560, 512, true) + run_k6_shape("q6k down", 2560, 9728, 512, true) + return + } + if (only == "k6") { + if (!g_gpu.has_coopmat2) { + print("no coopmat2 on this device\n") + return + } + verify(ensure_kq_batch_k6_cm2l_cls() && ensure_kq_batch_k6_cm2m_cls() + && kq_batch_cls_ensure(int(KqFmt.k6), false), "k6 probe pipelines must engage") + run_k6_shape("q6k gate", 9728, 2560, 512) + run_k6_shape("q6k down", 2560, 9728, 512) + run_k6_shape("q6k kv ", 1024, 2560, 512) + return + } + if (only == "k4") { + // Qwen3-4B role shapes (dim 2560, hidden 9728, qd 4096, kvd 1024) - k4 cm2 vs the kq tile + if (!g_gpu.has_coopmat2) { + print("no coopmat2 on this device\n") + return + } + verify(ensure_kq_batch_k4_cm2l_cls() && ensure_kq_batch_k4_cm2m_cls() + && kq_batch_cls_ensure(int(KqFmt.k4), false), "k4 probe pipelines must engage") + run_k4_shape("q4k gate", 9728, 2560, 512) + run_k4_shape("q4k down", 2560, 9728, 512) + run_k4_shape("q4k q/wo", 4096, 2560, 512) + run_k4_shape("q4k kv ", 2048, 2560, 512) + return + } + if (only == "ref") { + if (!g_gpu.has_coopmat2) { + print("no coopmat2 on this device\n") + return + } + verify(ensure_cm2px_ref() && ensure_cm2px_ref2() && ensure_cm2px_min(), "ref-blob pipelines must engage") + run_ref_gemm(8192, 3072, 256) + return + } + if (only == "tl") { + // tinyllama-1.1B role shapes (dim 2048, hidden 5632, kvd 256) - the small-model tile pick data + run_shape("tl gate", 5632, 2048, 512) + run_shape("tl down", 2048, 5632, 512) + run_shape("tl q/wo", 2048, 2048, 512) + run_shape("tl k/v ", 256, 2048, 512) + return + } + if (only == "cm2x") { + if (!g_gpu.has_coopmat2) { + print("no coopmat2 on this device\n") + return + } + verify(ensure_cm2px_nil() && ensure_cm2px_flat() && ensure_cm2px_lit() && ensure_cm2px_f16a() + && ensure_cm2px_push(), "cm2 probe pipelines must engage") + run_cm2x_shape("gate/up", 8192, 3072, 512) + run_cm2x_shape("down ", 3072, 8192, 512) + return + } + if (only == "gate" || only == "") { + run_shape("gate/up", 8192, 3072, 512) + } + if (only == "down" || only == "") { + run_shape("down ", 3072, 8192, 512) + } + if (only == "q" || only == "") { + run_shape("q/wo ", 3072, 3072, 512) + } + if (only == "kv" || only == "") { + run_shape("k/v ", 1024, 3072, 512) + } + if (only == "kvm" || only == "") { + run_shape("k+v mrg", 2048, 3072, 512) + } + if (only == "qkv" || only == "") { + run_shape("qkv mrg", 5120, 3072, 512) + } +} + +[export] +def main { + run_probe() +} diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index ebf478355d..7207ac95cb 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -92,6 +92,73 @@ def q8f16_gemm_oracle(wq : array; wsw : array; xf : array; } } +// Q4_K x f16 GEMM oracle (the cm2 k4 tile's shape): per weight d*sc*q - dmin*mn off the +// repacked planes, rounded f16 as the decode callback rounds, f16 activations — approx bars +def k4f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dm = unpackHalf2x16(wsu[sb * 5]) + for (g in range(8)) { + let sc = float(int((wsu[sb * 5 + 1 + g / 4] >> uint((g % 4) * 8)) & 0xFFu)) + let mn = float(int((wsu[sb * 5 + 3 + g / 4] >> uint((g % 4) * 8)) & 0xFFu)) + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = float((by >> ((e / 16) * 4)) & 0xF) + let w = float(float16(dm.x * sc * q - dm.y * mn)) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// Q6_K x f16 GEMM oracle (the cm2 k6 tile's shape): 6-bit compose - 32, per-16 signed +// sub-scale times the superblock d, rounded f16 as the decode rounds — approx bars +def k6f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dd = unpackHalf2x16(wsu[sb * 5 + 4]).x + for (e in range(256)) { + let bu = e / 32 + let hh = (e / 16) % 2 + let j = e % 16 + let lo = byte_u8(wq, sb * 192 + bu * 16 + j) + let hby = byte_u8(wq, sb * 192 + 128 + (bu / 4) * 32 + hh * 16 + j) + let q6 = (((lo >> (hh * 4)) & 0xF) | (((hby >> ((bu % 4) * 2)) & 3) << 4)) - 32 + let sidx = e / 16 + let scb = int((wsu[sb * 5 + sidx / 4] >> uint((sidx % 4) * 8)) & 0xFFu) + let sc = scb >= 128 ? scb - 256 : scb + let w = float(float16(dd * float(sc) * float(q6))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + e) + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // a kq class instance with plane members bound — its blk_contrib IS the per-block dequant // bit-math under test, callable on the CPU (no subgroup machinery inside) def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvBase? { diff --git a/modules/dasLLAMA/tests/test_gpu_tier.das b/modules/dasLLAMA/tests/test_gpu_tier.das index 728c74207f..57fbbd4647 100644 --- a/modules/dasLLAMA/tests/test_gpu_tier.das +++ b/modules/dasLLAMA/tests/test_gpu_tier.das @@ -68,6 +68,33 @@ def private d_read(l, pos : int64; kp : uint8?; vp : uint8?) {} [unused_argument(l, kp, vp, npos)] def private d_read_bulk(l : int64; kp : uint8?; vp : uint8?; npos : int64) {} +// ===== the ids-form prefill seat's doubles (installed separately from the driver bundle) ===== + +var private g_ids_npos = -1l +var private g_ids_scale = 0.0 +var private g_emb_block = -1l +var private g_emb_f32_vocab = -1l + +[unused_argument(tokens, cos_batch)] +def private d_prefill_ids(tokens : array; emb_scale : float; cos_batch : array; npos : int64; var logits : array) { + g_ids_npos = npos + g_ids_scale = emb_scale + if (!empty(logits)) { + logits[0] = 43.0 + } +} + +def private d_set_emb(emb_block : int64) : bool { + g_emb_block = emb_block + return emb_block >= 0l +} + +[unused_argument(fblob, off, dim)] +def private d_upload_emb_f32(fblob : array; off, vocab, dim : int64) : bool { + g_emb_f32_vocab = vocab + return vocab > 0l +} + def private arm_on() : bool => true def private arm_off() : bool => false @@ -145,3 +172,31 @@ def test_gpu_tier_contract(t : T?) { t |> success(!moe_gpu_backend_registered(), "registration witness resets with the hook") } } + +[test] +def test_rdec_prefill_ids_seam(t : T?) { + t |> run("the ids-form prefill seat: unset declines, installed doubles receive the forwarded values") @(t : T?) { + t |> success(!rdec_prefill_ids_installed(), "the seat starts uninstalled") + t |> success(!rdec_set_emb(3l), "an unset set_emb declines - the engine keeps the CPU embed") + let nofblob : array + t |> success(!rdec_upload_emb_f32(nofblob, 0l, 16l, 8l), "an unset f32 upload declines") + t |> equal(rdec_emb_f32_bytes(1000l, 256l), 1000l * 256l * 4l) + t |> equal(rdec_emb_f32_bytes(1l << 20l, 4096l), 0l) // past RDEC_EMB_F32_CAP the arm declines, so the plan counts nothing + install_rdec_prefill_ids(@@d_prefill_ids, @@d_set_emb, @@d_upload_emb_f32) + t |> success(rdec_prefill_ids_installed(), "install must flip the witness") + t |> success(rdec_set_emb(9l), "the double's answer forwards") + t |> equal(g_emb_block, 9l) + t |> success(!rdec_set_emb(-1l), "the double's decline forwards") + t |> success(rdec_upload_emb_f32(nofblob, 4l, 32000l, 2048l), "the f32 double's answer forwards") + t |> equal(g_emb_f32_vocab, 32000l) + var ids : array + ids |> push(5l) + let cosb : array + var logits : array + logits |> resize(4) + rdec_prefill_ids(ids, 1.5, cosb, 1l, logits) + t |> equal(g_ids_npos, 1l) + t |> equal(g_ids_scale, 1.5) + t |> equal(logits[0], 43.0) + } +} diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index f8c485a347..e85be3dfd5 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -11,6 +11,7 @@ require ?vulkan _vkd_oracles require ?vulkan vulkan require ?vulkan vulkan/vulkan_boost require ?vulkan dasllama/dasllama_vulkan_common +require ?vulkan dasllama/dasllama_env // g_env_vulkan.coopmat - the resolved-default ladder cell pins and restores it require ?vulkan dasllama/dasllama_vulkan_classes require ?vulkan dasllama/dasllama_vulkan_seams require ?vulkan dasllama/dasllama_kqformat @@ -1501,6 +1502,250 @@ def test_vkd_cm2l_batch(t0 : T?) { } } +[test] +def test_vkd_k4_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q4_K l/m tiles == the CPU oracle (decode-in-load over the repacked planes)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k4 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k4_cm2l_cls() && ensure_kq_batch_k4_cm2m_cls(), "k4 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize(rows * n / 2) + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary nibble bytes + } + for (sb in range(totsb)) { + let dv = 0.0002 * float(1 + sb % 7) // cool scales: the f16 acc must stay far from 65504 + let dmv = 0.00005 * float(1 + sb % 5) + wsuh[sb * 5] = packHalf2x16(float2(dv, dmv)) + for (wi in range(4)) { + wsuh[sb * 5 + 1 + wi] = hash_word(uint(sb * 4 + wi) + 613u) // sc / mn bytes + } + } + for (i in range(rows * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64(rows * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(2)) { + let ttile = ml == 0 ? 256 : 128 + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k4_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k4_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k4_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k4_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k4f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k4 {ml == 0 ? "l" : "m"}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k4 {ml == 0 ? "l" : "m"}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k4 bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_k6_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q6_K l/m tiles == the CPU oracle (6-bit compose decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k6 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k6_cm2l_cls() && ensure_kq_batch_k6_cm2m_cls(), "k6 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 48) + wsuh |> resize(totsb * 5) + xfh |> resize(rows * n / 2) + for (i in range(totsb * 48)) { + wqh[i] = hash_word(uint(i) + 41u) // arbitrary ql/qh bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { + wsuh[sb * 5 + wi] = hash_word(uint(sb * 4 + wi) + 811u) // signed sub-scale bytes + } + wsuh[sb * 5 + 4] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d: the f16 acc must stay far from 65504 + } + for (i in range(rows * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) + } + let wq_bytes = int64(totsb) * 192l + let ws_bytes = int64(totsb * 5) * 4l + let xf_bytes = int64(rows * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(2)) { + let ttile = ml == 0 ? 256 : 128 + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k6_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k6_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k6_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k6_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k6f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k6 {ml == 0 ? "l" : "m"}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k6 {ml == 0 ? "l" : "m"}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k6 bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_cm2m_batch(t0 : T?) { t0 |> run("cm2 m-tile fmt-0 batch class kernel == the CPU oracle") <| @(t : T?) { @@ -1909,6 +2154,7 @@ def test_vkd_fa_cm2(t0 : T?) { return } verify(ensure_fa_cm2_h64_cls() && ensure_fa_cm2_h128_cls(), "fa class rails must engage on an fa-capable device") + verify(ensure_fa_cm2_h64_f16_cls() && ensure_fa_cm2_h128_f16_cls(), "fa f16-out twins must engage on an fa-capable device") let nheads = 4 let kv_mul = 2 let rows = 77 // partial 64-tile: q0=64 has 13 valid rows — clamp loads + store discard run @@ -1934,19 +2180,34 @@ def test_vkd_fa_cm2(t0 : T?) { let q_bytes = int64(rows * qd) * 4l let m_bytes = int64(kpos * kvd) * 4l let sh_bytes = int64(kpos * kvd) * 2l + let h16_bytes = int64(rows * qd) * 2l let qdev = make_device_buf(q_bytes) let kdev = make_device_buf(m_bytes) let vdev = make_device_buf(m_bytes) let khdev = make_device_buf(sh_bytes) let vhdev = make_device_buf(sh_bytes) let yd = make_device_buf(q_bytes) + let yh16_fused = make_device_buf(h16_bytes) // the f16-out twin's plane + let yh16_split = make_device_buf(h16_bytes) // the split pair's: f32 out -> f16cvt var host = make_host_buf(q_bytes, true, [cached = true]) + var host16 = make_host_buf(h16_bytes * 2l, true, [cached = true]) var y_cls : array + var y16_fused : array + var y16_split : array y_cls |> resize(rows * qd) + y16_fused |> resize(rows * qd) + y16_split |> resize(rows * qd) + // two DIFFERENT sentinels on the compared f16 planes: a dead leg on either side reads as a mismatch + for (i in range(rows * qd)) { + y16_fused[i] = float16(-777.5) + y16_split[i] = float16(-999.25) + } unsafe { upload_region_at(qdev, 0l, addr(qh[0]), q_bytes) upload_region_at(kdev, 0l, addr(kh[0]), m_bytes) upload_region_at(vdev, 0l, addr(vh[0]), m_bytes) + upload_region_at(yh16_fused, 0l, addr(y16_fused[0]), h16_bytes) + upload_region_at(yh16_split, 0l, addr(y16_split[0]), h16_bytes) var sck = set_f16cvt_cls(fixed_array(kdev, khdev), fixed_array(m_bytes, sh_bytes), fixed_array(2u, 16u)) var scv = set_f16cvt_cls(fixed_array(vdev, vhdev), @@ -1954,6 +2215,11 @@ def test_vkd_fa_cm2(t0 : T?) { var sfa = set_fa_cm2_cls(fixed_array(qdev, khdev, vhdev, yd), fixed_array(q_bytes, sh_bytes, sh_bytes, q_bytes), fixed_array(1u, 16u, 32u, 4u)) + var sfa16 = set_fa_cm2_cls(fixed_array(qdev, khdev, vhdev, yh16_fused), + fixed_array(q_bytes, sh_bytes, sh_bytes, h16_bytes), + fixed_array(1u, 16u, 32u, 64u)) + var scy = set_f16cvt_cls(fixed_array(yd, yh16_split), + fixed_array(q_bytes, h16_bytes), fixed_array(4u, 128u)) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -1967,14 +2233,23 @@ def test_vkd_fa_cm2(t0 : T?) { let fawgs = int64(nheads * ((rows + 63) / 64)) if (hs == 64) { enc_fa_cm2_h64_cls(raw2, h2, sfa, pcf, fawgs) + enc_fa_cm2_h64_f16_cls(raw2, h2, sfa16, pcf, fawgs) } else { enc_fa_cm2_h128_cls(raw2, h2, sfa, pcf, fawgs) + enc_fa_cm2_h128_f16_cls(raw2, h2, sfa16, pcf, fawgs) } - vhz_dep(raw2, h2, 4u, 0u, true) + // the split pair's own convert: f16cvt over the f32 out - the fused twin's true peer + var pcy = ActArgs(nelem = uint(rows * qd), gelu = 0u, nblk = 0u) + enc_f16cvt_cls(raw2, h2, scy, pcy, int64((rows * qd / 4 + 255) / 256)) + vhz_dep(raw2, h2, 4u | 64u | 128u, 0u, true) cmd_copy_whole(raw2, yd, host.buf, q_bytes) + cmd_copy_range(raw2, yh16_fused, 0l, host16.buf, 0l, h16_bytes) + cmd_copy_range(raw2, yh16_split, 0l, host16.buf, h16_bytes, h16_bytes) vk_check(vkEndCommandBuffer(raw2), null) submit_wait(raw2) memcpy(addr(y_cls[0]), host.mapped, q_bytes) + memcpy(addr(y16_fused[0]), host16.mapped, h16_bytes) + memcpy(addr(y16_split[0]), reinterpret(intptr(host16.mapped) + uint64(h16_bytes)), h16_bytes) } // oracle mirrors the kernel's f16 grid: q pre-scales then rounds (the kernel's // MatrixTimesScalar-then-convert), K/V round through the shadow; softmax stays f32 @@ -2001,6 +2276,13 @@ def test_vkd_fa_cm2(t0 : T?) { } let bad = mismatch_approx(y_cls, y_ref) t |> success(bad == 0, "fa_cm2 hs {hs} matches the CPU oracle ({bad} of {rows * qd} off)") + var bad16 = 0 + for (i in range(rows * qd)) { + if (float(y16_fused[i]) != float(y16_split[i])) { + bad16++ + } + } + t |> success(bad16 == 0, "fa_cm2 hs {hs} f16-out twin == the split pair's f16cvt ({bad16} of {rows * qd} off)") delete qh delete kh delete vh @@ -2009,6 +2291,8 @@ def test_vkd_fa_cm2(t0 : T?) { delete vh2 delete y_ref delete y_cls + delete y16_fused + delete y16_split } } else { feint("dasVulkan not present - skipping\n") @@ -2688,7 +2972,7 @@ def test_vkd_rope_b_pair(t0 : T?) { var pc = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(half), neox = uint(neox), kpair0 = uint(kpair0), ppp = uint(ppp), layerbase = uint(layerbase), npos = uint(npos), - pos0 = uint(pos0), voff = uint(voff)) + pos0 = uint(pos0), voff = uint(voff), kstride = uint(kvd)) enc_rope_kv_b_cls(raw, h, sc, pc, int64(wgs)) vhz_dep(raw, h, 8u, 0u, true) cmd_copy_whole(raw, qd2, host.buf, q_bytes) @@ -2747,6 +3031,59 @@ def test_vkd_rope_b_pair(t0 : T?) { } } t |> success(bad == 0, "rope_kv_store_b matches the CPU oracle ({bad} off)") + // the merged k|v projection layout (the DASLLAMA_VK_KV_MERGE default): [k | v] per position at + // stride 2*kvd, v at +kvd - the same oracle rows must land; the mirrors take a sentinel first + var kvm_h : array + kvm_h |> resize(npos * 2 * kvd) + for (p in range(npos)) { + for (c in range(kvd)) { + kvm_h[p * 2 * kvd + c] = kvh[p * kvd + c] + kvm_h[p * 2 * kvd + kvd + c] = kvh[voff + p * kvd + c] + } + } + var msent : array + msent |> resize(mirn) + for (i in range(mirn)) { + msent[i] = -999.25 + } + unsafe { + upload_region_at(qd2, 0l, addr(q0h[0]), q_bytes) + upload_region_at(kvdv, 0l, addr(kvm_h[0]), kv_bytes) + upload_region_at(km2, 0l, addr(msent[0]), m_bytes) + upload_region_at(vm2, 0l, addr(msent[0]), m_bytes) + var scm = set_rope_kv_b_cls(fixed_array(qd2, kvdv, km2, vm2, csd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes), + fixed_array(1u, 16u, 2u, 4u, 8u)) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pc = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), + half = uint(half), neox = uint(neox), kpair0 = uint(kpair0), + ppp = uint(ppp), layerbase = uint(layerbase), npos = uint(npos), + pos0 = uint(pos0), voff = uint(kvd), kstride = uint(2 * kvd)) + enc_rope_kv_b_cls(raw, h, scm, pc, int64(wgs)) + vhz_dep(raw, h, 8u, 0u, true) + cmd_copy_whole(raw, qd2, host.buf, q_bytes) + cmd_copy_range(raw, km2, 0l, host.buf, q_bytes, m_bytes) + cmd_copy_range(raw, vm2, 0l, host.buf, q_bytes + m_bytes, m_bytes) + vk_check(vkEndCommandBuffer(raw), null) + submit_wait(raw) + memcpy(addr(q_cls[0]), host.mapped, q_bytes) + memcpy(addr(m_cls[0]), reinterpret(intptr(host.mapped) + uint64(q_bytes)), 2l * m_bytes) + } + var badm = mismatch_approx(q_cls, q_ref) + for (i in range(npos * kvd)) { + if (!approx(m_cls[w0 + i], mk_ref[w0 + i])) { + badm++ + } + if (!approx(m_cls[mirn + w0 + i], mv_ref[w0 + i])) { + badm++ + } + } + t |> success(badm == 0, "rope_kv_store_b over merged k|v rows (kstride 2*kvd, voff kvd) matches the CPU oracle ({badm} off)") + delete kvm_h + delete msent delete q0h delete kvh delete csh @@ -3732,7 +4069,7 @@ def test_vkd_kv16_writers(t0 : T?) { var pc = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(half), neox = uint(neox), kpair0 = uint(kpair0), ppp = uint(ppp), layerbase = uint(layerbase), npos = uint(npos), - pos0 = uint(pos0), voff = uint(voff)) + pos0 = uint(pos0), voff = uint(voff), kstride = uint(kvd)) enc_rope_kv_b_f16_cls(raw, h, sc, pc, int64(wgs)) vhz_dep(raw, h, 8u, 0u, true) cmd_copy_range(raw, km2, 0l, host.buf, 0l, m_bytes) @@ -3778,6 +4115,58 @@ def test_vkd_kv16_writers(t0 : T?) { } } t |> success(bad == 0, "rope_kv_b_f16 matches the CPU oracle ({bad} off)") + // the merged k|v projection layout (the DASLLAMA_VK_KV_MERGE default): [k | v] per position at + // stride 2*kvd, v at +kvd - the same oracle rows must land; the mirrors take a sentinel first + var kvm_h : array + kvm_h |> resize(npos * 2 * kvd) + for (p in range(npos)) { + for (c in range(kvd)) { + kvm_h[p * 2 * kvd + c] = kvh[p * kvd + c] + kvm_h[p * 2 * kvd + kvd + c] = kvh[voff + p * kvd + c] + } + } + var msent : array + msent |> resize(mirn) + for (i in range(mirn)) { + msent[i] = float16(-999.25) + } + unsafe { + upload_region_at(qd2, 0l, addr(q0h[0]), q_bytes) + upload_region_at(kvdv, 0l, addr(kvm_h[0]), kv_bytes) + upload_region_at(km2, 0l, addr(msent[0]), m_bytes) + upload_region_at(vm2, 0l, addr(msent[0]), m_bytes) + var scm = set_rope_kv_b_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes), + fixed_array(1u, 16u, 2u, 4u, 8u)) + var raw = alloc_cmd() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pc = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), + half = uint(half), neox = uint(neox), kpair0 = uint(kpair0), + ppp = uint(ppp), layerbase = uint(layerbase), npos = uint(npos), + pos0 = uint(pos0), voff = uint(kvd), kstride = uint(2 * kvd)) + enc_rope_kv_b_f16_cls(raw, h, scm, pc, int64(wgs)) + vhz_dep(raw, h, 8u, 0u, true) + cmd_copy_range(raw, km2, 0l, host.buf, 0l, m_bytes) + cmd_copy_range(raw, vm2, 0l, host.buf, m_bytes, m_bytes) + vk_check(vkEndCommandBuffer(raw), null) + submit_wait(raw) + memcpy(addr(m16[0]), host.mapped, 2l * m_bytes) + } + var m_clsm <- unpack_f16(m16) + var badm = 0 + for (i in range(npos * kvd)) { + if (!approx(m_clsm[wb0 + i], mk_ref[wb0 + i])) { + badm++ + } + if (!approx(m_clsm[mirn + wb0 + i], mv_ref[wb0 + i])) { + badm++ + } + } + t |> success(badm == 0, "rope_kv_b_f16 over merged k|v rows (kstride 2*kvd, voff kvd) matches the CPU oracle ({badm} off)") + delete kvm_h + delete msent + delete m_clsm delete q0h delete kvh delete csh @@ -4171,7 +4560,394 @@ def test_vkd_kv16_readers(t0 : T?) { feint("no coopmat on this device - h128 f16 arm skipped\n") } } else { - feint("dasVulkan not present - skipping\n") + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_emb_gather_pair(t0 : T?) { + t0 |> run("embed-gather class kernels (q8 + f32) == the CPU dequant/copy") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_emb_gather_cls() || !ensure_emb_gather_f32_cls()) { + t |> skip("no Vulkan device") + return + } + let dim = 64 + let nbb = dim / 32 + let vocab = 16 + let npos = 8 + let scale = 1.25 + let wblk0 = 3 // the plane sits 3 blocks into its slab - the serving shape, never block 0 + let totblk = wblk0 + vocab * nbb + var wqh : array + var wsh16 : array + var idsh : array + wqh |> resize(totblk * 8) + wsh16 |> resize(totblk) + idsh |> resize(npos) + for (i in range(totblk * 8)) { + wqh[i] = hash_word(uint(i) + 5u) + } + for (i in range(totblk)) { + wsh16[i] = float16(0.0625 + float(i % 7) * 0.03125) + } + for (i in range(npos)) { + idsh[i] = uint((i * 5 + 3) % 6) // a 6-cycle over 8 rows: tokens 3 and 2 gather twice + } + // f32 twin's table: a distinct pattern so a crossed binding cannot pass + var wfh : array + wfh |> resize(vocab * dim) + for (i in range(vocab * dim)) { + wfh[i] = float(i % 97) * 0.125 - 6.0 + } + let wq_bytes = int64(totblk) * 32l + let ws_bytes = int64(totblk) * 2l + let ids_bytes = int64(npos) * 4l + let x_bytes = int64(npos * dim) * 4l + let wf_bytes = int64(vocab * dim) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let idd = make_device_buf(ids_bytes) + let xd = make_device_buf(x_bytes) + let wfd = make_device_buf(wf_bytes) + var host = make_host_buf(x_bytes * 2l, true, [cached = true]) + var y_cls : array + var yf_cls : array + y_cls |> resize(npos * dim) + yf_cls |> resize(npos * dim) + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsh16[0]), ws_bytes) + upload_region_at(idd, 0l, addr(idsh[0]), ids_bytes) + upload_region_at(wfd, 0l, addr(wfh[0]), wf_bytes) + var scq = set_emb_gather_cls(fixed_array(wqd, wsd, idd, xd), + fixed_array(wq_bytes, ws_bytes, ids_bytes, x_bytes), + fixed_array(1u, 2u, 4u, 8u)) + var scf = set_emb_gather_f32_cls(fixed_array(wfd, idd, xd), + fixed_array(wf_bytes, ids_bytes, x_bytes), + fixed_array(1u, 4u, 8u)) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pc = EmbArgs(npos = uint(npos), dim = uint(dim), wblk0 = uint(wblk0), embed_scale = scale) + enc_emb_gather_cls(raw, h, scq, pc, int64((npos * dim / 4 + 255) / 256)) + vhz_dep(raw, h, 8u, 0u, true) + cmd_copy_range(raw, xd, 0l, host.buf, 0l, x_bytes) + enc_emb_gather_f32_cls(raw, h, scf, pc, int64((npos * dim / 4 + 255) / 256)) + vhz_dep(raw, h, 8u, 0u, true) + cmd_copy_range(raw, xd, 0l, host.buf, x_bytes, x_bytes) + vk_check(vkEndCommandBuffer(raw), null) + submit_wait(raw) + memcpy(addr(y_cls[0]), host.mapped, x_bytes) + memcpy(addr(yf_cls[0]), reinterpret(intptr(host.mapped) + uint64(x_bytes)), x_bytes) + } + var y_ref : array + var yf_ref : array + y_ref |> resize(npos * dim) + yf_ref |> resize(npos * dim) + for (r in range(npos)) { + let tok = int(idsh[r]) + for (col in range(dim)) { + let ib = wblk0 + tok * nbb + col / 32 + let by = col % 32 + let w = wqh[ib * 8 + by / 4] + let q = int8(w >> uint(8 * (by % 4))) + y_ref[r * dim + col] = float(int(q)) * (float(wsh16[ib]) * scale) + yf_ref[r * dim + col] = wfh[tok * dim + col] * scale + } + } + let bad = mismatch_exact(y_cls, y_ref) + t |> success(bad == 0, "q8 embed gather matches the CPU dequant ({bad} of {npos * dim} off)") + let badf = mismatch_exact(yf_cls, yf_ref) + t |> success(badf == 0, "f32 embed gather matches the CPU copy ({badf} of {npos * dim} off)") + delete wqh + delete wsh16 + delete idsh + delete wfh + delete y_cls + delete yf_cls + delete y_ref + delete yf_ref + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_ar_rq_fused(t0 : T?) { + t0 |> run("fused batch ar+rq == the split cls_ar + cls_dn_rq pair, bit-exact") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_cls_ar() || !ensure_cls_dn_rq() || !ensure_cls_ar_rq_b()) { + feint("no Vulkan device - skipping\n") + return + } + let dim = 128 + let nrows = 6 + let woff = 64 + let nblk = nrows * (dim / 32) + var xh : array + var aah : array + var wnh : array + xh |> resize(nrows * dim) + aah |> resize(nrows * dim) + wnh |> resize(woff + dim) + for (i in range(nrows * dim)) { + xh[i] = 0.375 + float(i % 13) * 0.0625 - float(i % 5) * 0.125 + aah[i] = float(i % 7) * 0.25 - 0.5 + } + for (i in range(woff + dim)) { + wnh[i] = 0.75 + float(i % 9) * 0.03125 + } + let row_bytes = int64(nrows * dim) * 4l + let wn_bytes = int64(woff + dim) * 4l + let q_bytes = int64(nblk) * 32l + let s_bytes = int64(nblk) * 4l + let x_split = make_device_buf(row_bytes) + let x_fused = make_device_buf(row_bytes) + let aad = make_device_buf(row_bytes) + let wnd = make_device_buf(wn_bytes) + let yod = make_device_buf(row_bytes) + let q_split = make_device_buf(q_bytes) + let s_split = make_device_buf(s_bytes) + let q_fused = make_device_buf(q_bytes) + let s_fused = make_device_buf(s_bytes) + // two DIFFERENT sentinels on the compared planes: a dead leg on either side reads as a mismatch + var sent_split : array + var sent_fused : array + sent_split |> resize(nrows * dim) + sent_fused |> resize(nrows * dim) + for (i in range(nrows * dim)) { + sent_split[i] = -999.25 + sent_fused[i] = -777.5 + } + var host = make_host_buf((q_bytes + s_bytes) * 2l + row_bytes * 2l, true, [cached = true]) + unsafe { + upload_region_at(x_split, 0l, addr(xh[0]), row_bytes) + upload_region_at(x_fused, 0l, addr(xh[0]), row_bytes) + upload_region_at(aad, 0l, addr(aah[0]), row_bytes) + upload_region_at(wnd, 0l, addr(wnh[0]), wn_bytes) + upload_region_at(q_split, 0l, addr(sent_split[0]), q_bytes) + upload_region_at(s_split, 0l, addr(sent_split[0]), s_bytes) + upload_region_at(q_fused, 0l, addr(sent_fused[0]), q_bytes) + upload_region_at(s_fused, 0l, addr(sent_fused[0]), s_bytes) + var sca = set_cls_ar(fixed_array(x_split, aad, wnd, yod), + fixed_array(row_bytes, row_bytes, wn_bytes, row_bytes), + fixed_array(1u, 2u, 0u, 4u)) + var scr = set_rq_cls(fixed_array(yod, q_split, s_split), + fixed_array(row_bytes, q_bytes, s_bytes), + fixed_array(4u, 8u, 8u)) + var scf = set_cls_ar_rq_b(fixed_array(x_fused, aad, wnd, q_fused, s_fused), + fixed_array(row_bytes, row_bytes, wn_bytes, q_bytes, s_bytes), + fixed_array(16u, 2u, 0u, 32u, 32u)) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pca = ArArgs(dim = uint(dim), add_on = 1u, woff = uint(woff), eps = 1e-5, ascale = 1.0) + enc_cls_ar(raw, h, sca, pca, int64(nrows)) + var pcr = RqArgs(inbase = 0u, nblk = uint(nblk)) + enc_cls_dn_rq(raw, h, scr, pcr, int64((nrows * dim / 4 + 255) / 256)) + enc_cls_ar_rq_b(raw, h, scf, pca, int64(nrows)) + vhz_dep(raw, h, 8u | 32u | 16u | 1u, 0u, true) + cmd_copy_range(raw, q_split, 0l, host.buf, 0l, q_bytes) + cmd_copy_range(raw, s_split, 0l, host.buf, q_bytes, s_bytes) + cmd_copy_range(raw, q_fused, 0l, host.buf, q_bytes + s_bytes, q_bytes) + cmd_copy_range(raw, s_fused, 0l, host.buf, q_bytes * 2l + s_bytes, s_bytes) + cmd_copy_range(raw, x_split, 0l, host.buf, (q_bytes + s_bytes) * 2l, row_bytes) + cmd_copy_range(raw, x_fused, 0l, host.buf, (q_bytes + s_bytes) * 2l + row_bytes, row_bytes) + vk_check(vkEndCommandBuffer(raw), null) + submit_wait(raw) + var q_split_h : array + var q_fused_h : array + var s_split_h : array + var s_fused_h : array + var x_split_h : array + var x_fused_h : array + q_split_h |> resize(nblk * 8) + q_fused_h |> resize(nblk * 8) + s_split_h |> resize(nblk) + s_fused_h |> resize(nblk) + x_split_h |> resize(nrows * dim) + x_fused_h |> resize(nrows * dim) + let hp = intptr(host.mapped) + memcpy(addr(q_split_h[0]), host.mapped, q_bytes) + memcpy(addr(s_split_h[0]), reinterpret(hp + uint64(q_bytes)), s_bytes) + memcpy(addr(q_fused_h[0]), reinterpret(hp + uint64(q_bytes + s_bytes)), q_bytes) + memcpy(addr(s_fused_h[0]), reinterpret(hp + uint64(q_bytes * 2l + s_bytes)), s_bytes) + memcpy(addr(x_split_h[0]), reinterpret(hp + uint64((q_bytes + s_bytes) * 2l)), row_bytes) + memcpy(addr(x_fused_h[0]), reinterpret(hp + uint64((q_bytes + s_bytes) * 2l + row_bytes)), row_bytes) + let badq = mismatch_qbytes(q_split_h, q_fused_h, 0) + t |> success(badq == 0, "fused quant words == split ({badq} of {nblk * 8} off)") + let bads = mismatch_exact(s_split_h, s_fused_h) + t |> success(bads == 0, "fused scales == split ({bads} of {nblk} off)") + // the residual plane is IN-PLACE, so it cannot hold a sentinel - the liveness pair is its witness + t |> success(x_split_h[0] != xh[0], "the split ar wrote the residual stream") + t |> success(x_fused_h[0] != xh[0], "the fused ar wrote the residual stream") + let badx = mismatch_exact(x_split_h, x_fused_h) + t |> success(badx == 0, "fused residual update == split ({badx} of {nrows * dim} off)") + delete q_split_h + delete q_fused_h + delete s_split_h + delete s_fused_h + delete x_split_h + delete x_fused_h + } + delete sent_split + delete sent_fused + delete xh + delete aah + delete wnh + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vk_coopmat_default_and_tile_pick(t0 : T?) { + t0 |> run("the resolved coopmat default ladder and the cm2 tile pick (pure functions)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!empty(g_env_vulkan.coopmat) && g_env_vulkan.coopmat != "auto") { + t |> skip("DASLLAMA_COOPMAT is set - the resolved default ladder is not what this box runs") + return + } + let saved_force = g_coopmat_mode_force + g_coopmat_mode_force = -1 + t |> equal(resolve_coopmat_mode(true, true), 4) // cm2 where the device has it + t |> equal(resolve_coopmat_mode(true, false), 3) // else mm + t |> equal(resolve_coopmat_mode(false, false), 0) // else sdot4 + g_coopmat_mode_force = saved_force + if (!ensure_q8_batch_cls()) { + t |> skip("no Vulkan device for the tile pick (it reads the device's SM count)") + return + } + let saved_sm = g_gpu.sm_count + g_gpu.sm_count = 36 + t |> equal(cm2_tile_cols(3072l, 128l), 128l) // a short window never takes the half-empty l column + t |> equal(cm2_tile_cols(3072l, 512l), 128l) // the 3B q/down shape: m on a strict wave win (96 tiles in 3 waves vs 48 in 2) + t |> equal(cm2_tile_cols(8192l, 512l), 256l) // the 3B gate/up shape: a wave tie goes to l + t |> equal(cm2_tile_cols(5632l, 512l), 128l) // the tinyllama gate shape: m on a strict win + g_gpu.sm_count = 0 + t |> equal(cm2_tile_cols(3072l, 64l), 128l) // unknown SM count: the short-window rule still fires + t |> equal(cm2_tile_cols(3072l, 512l), 256l) // unknown SM count: l, never a split + g_gpu.sm_count = saved_sm + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_ar_f16_fused(t0 : T?) { + t0 |> run("fused batch ar+f16 == the split cls_ar + f16cvt_cls pair, bit-exact") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_cls_ar() || !ensure_f16cvt_cls() || !ensure_cls_ar_f16_b()) { + t |> skip("no Vulkan device") + return + } + let dim = 128 + let nrows = 6 + let woff = 64 + var xh : array + var aah : array + var wnh : array + xh |> resize(nrows * dim) + aah |> resize(nrows * dim) + wnh |> resize(woff + dim) + for (i in range(nrows * dim)) { + xh[i] = 0.375 + float(i % 13) * 0.0625 - float(i % 5) * 0.125 + aah[i] = float(i % 7) * 0.25 - 0.5 + } + for (i in range(woff + dim)) { + wnh[i] = 0.75 + float(i % 9) * 0.03125 + } + let row_bytes = int64(nrows * dim) * 4l + let wn_bytes = int64(woff + dim) * 4l + let h_bytes = int64(nrows * dim) * 2l + let x_split = make_device_buf(row_bytes) + let x_fused = make_device_buf(row_bytes) + let aad = make_device_buf(row_bytes) + let wnd = make_device_buf(wn_bytes) + let yod = make_device_buf(row_bytes) + let f16_split = make_device_buf(h_bytes) + let f16_fused = make_device_buf(h_bytes) + // two DIFFERENT sentinels on the compared planes: a dead leg on either side reads as a mismatch + var sent_split : array + var sent_fused : array + sent_split |> resize(nrows * dim) + sent_fused |> resize(nrows * dim) + for (i in range(nrows * dim)) { + sent_split[i] = float16(-999.25) + sent_fused[i] = float16(-777.5) + } + var host = make_host_buf(h_bytes * 2l + row_bytes * 2l, true, [cached = true]) + unsafe { + upload_region_at(x_split, 0l, addr(xh[0]), row_bytes) + upload_region_at(x_fused, 0l, addr(xh[0]), row_bytes) + upload_region_at(aad, 0l, addr(aah[0]), row_bytes) + upload_region_at(wnd, 0l, addr(wnh[0]), wn_bytes) + upload_region_at(f16_split, 0l, addr(sent_split[0]), h_bytes) + upload_region_at(f16_fused, 0l, addr(sent_fused[0]), h_bytes) + var sca = set_cls_ar(fixed_array(x_split, aad, wnd, yod), + fixed_array(row_bytes, row_bytes, wn_bytes, row_bytes), + fixed_array(1u, 2u, 0u, 4u)) + var scc = set_f16cvt_cls(fixed_array(yod, f16_split), + fixed_array(row_bytes, h_bytes), fixed_array(4u, 8u)) + var scf = set_cls_ar_f16_b(fixed_array(x_fused, aad, wnd, f16_fused), + fixed_array(row_bytes, row_bytes, wn_bytes, h_bytes), + fixed_array(16u, 2u, 0u, 32u)) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pca = ArArgs(dim = uint(dim), add_on = 1u, woff = uint(woff), eps = 1e-5, ascale = 1.0) + enc_cls_ar(raw, h, sca, pca, int64(nrows)) + var pcc = ActArgs(nelem = uint(nrows * dim), gelu = 0u, nblk = 0u) + enc_f16cvt_cls(raw, h, scc, pcc, int64((nrows * dim / 4 + 255) / 256)) + enc_cls_ar_f16_b(raw, h, scf, pca, int64(nrows)) + vhz_dep(raw, h, 8u | 32u | 16u | 1u, 0u, true) + cmd_copy_range(raw, f16_split, 0l, host.buf, 0l, h_bytes) + cmd_copy_range(raw, f16_fused, 0l, host.buf, h_bytes, h_bytes) + cmd_copy_range(raw, x_split, 0l, host.buf, h_bytes * 2l, row_bytes) + cmd_copy_range(raw, x_fused, 0l, host.buf, h_bytes * 2l + row_bytes, row_bytes) + vk_check(vkEndCommandBuffer(raw), null) + submit_wait(raw) + let nw = nrows * dim / 2 + var f16_split_h : array + var f16_fused_h : array + var x_split_h : array + var x_fused_h : array + f16_split_h |> resize(nw) + f16_fused_h |> resize(nw) + x_split_h |> resize(nrows * dim) + x_fused_h |> resize(nrows * dim) + let hp = intptr(host.mapped) + memcpy(addr(f16_split_h[0]), host.mapped, h_bytes) + memcpy(addr(f16_fused_h[0]), reinterpret(hp + uint64(h_bytes)), h_bytes) + memcpy(addr(x_split_h[0]), reinterpret(hp + uint64(h_bytes * 2l)), row_bytes) + memcpy(addr(x_fused_h[0]), reinterpret(hp + uint64(h_bytes * 2l + row_bytes)), row_bytes) + let badh = mismatch_qbytes(f16_split_h, f16_fused_h, 0) + t |> success(badh == 0, "fused f16 rows == split ({badh} of {nw} words off)") + // the residual plane is IN-PLACE, so it cannot hold a sentinel - the liveness pair is its witness + t |> success(x_split_h[0] != xh[0], "the split ar wrote the residual stream") + t |> success(x_fused_h[0] != xh[0], "the fused ar wrote the residual stream") + let badx = mismatch_exact(x_split_h, x_fused_h) + t |> success(badx == 0, "fused residual update == split ({badx} of {nrows * dim} off)") + delete f16_split_h + delete f16_fused_h + delete x_split_h + delete x_fused_h + } + delete sent_split + delete sent_fused + delete xh + delete aah + delete wnh + } else { + t |> skip("dasVulkan not present") } } } diff --git a/modules/dasLLAMA/tests/test_vulkan_tier.das b/modules/dasLLAMA/tests/test_vulkan_tier.das index b54a96c283..2aa69c8964 100644 --- a/modules/dasLLAMA/tests/test_vulkan_tier.das +++ b/modules/dasLLAMA/tests/test_vulkan_tier.das @@ -27,7 +27,7 @@ require math // Without dasVulkan (or with no Vulkan device) every check feints cleanly. // The CPU-reference arms' tolerances were derived against the sdot4 reference kernels — pin -// that mode ahead of the router's fastest-path default (mm) so the oracles stay deterministic. +// that mode ahead of the router's resolved default (cm2 or mm) so the oracles stay deterministic. // The coopmat kernels' numeric envelope is validated by the drift-class model gates, not here. [init] def private pin_reference_gemm_mode { @@ -2271,6 +2271,33 @@ def test_vulkan_attention(t : T?) { t |> success(cy.maxdiff <= AT_REL_TOL * cy.maxref, "{tag}: out rows within tolerance (maxdiff {cy.maxdiff} vs {AT_REL_TOL * cy.maxref} bar, maxref {cy.maxref})") let ck = compare_gout(kout, kp) + if (ck.maxdiff > AT_REL_TOL * ck.maxref) { + // a run of bad rows at a window boundary names the racing dispatch + var frow = -1l + var lrow = -1l + var nbad = 0l + for (rw in range64(npos)) { + var bad = false + for (c in range64(ATKVD)) { + if (abs(kout[int(rw * ATKVD + c)] - kp[int(rw * ATKVD + c)]) > AT_REL_TOL * ck.maxref) { + bad = true + break + } + } + if (bad) { + if (frow < 0l) { + frow = rw + } + lrow = rw + nbad++ + } + } + to_log(LOG_ERROR, "roped-k forensics: bad rows {nbad} first {frow} last {lrow} (npos {npos})\n") + if (frow >= 0l) { + let first_bad_off = frow * ATKVD + to_log(LOG_ERROR, "roped-k row {frow} gpu [{kout[int(first_bad_off)]}, {kout[int(first_bad_off + 1l)]}] ref [{kp[int(first_bad_off)]}, {kp[int(first_bad_off + 1l)]}]\n") + } + } t |> success(ck.maxdiff <= AT_REL_TOL * ck.maxref, "{tag}: roped k rows within tolerance (maxdiff {ck.maxdiff} vs {AT_REL_TOL * ck.maxref} bar)") let cv = compare_gout(vout, vy) diff --git a/modules/dasSpirv/ARCHITECTURE.md b/modules/dasSpirv/ARCHITECTURE.md index 86e597fffd..8516ccdac9 100644 --- a/modules/dasSpirv/ARCHITECTURE.md +++ b/modules/dasSpirv/ARCHITECTURE.md @@ -48,7 +48,7 @@ compute test as a ready-made end-to-end gate. raises it to 1.4 because `SPV_EXT_mesh_shader` requires it, and a few subgroup and cooperative-matrix ops raise it to 1.5. Every other stage stays at 1.3. -## 3. Files and emission mechanism +## 3. Files and emission mechanism {#files-and-emission} `modules/dasSpirv` is **pure daslang** (mirrors dasGlsl: a `spirv/` subdir of `.das` files + CMake resolver rows derived from `.das_module`; no `.shared_module`, no C++). dasVulkan @@ -90,6 +90,42 @@ without macro plumbing. parameters. The method form erases the das-level `self` from it, so the decode body still reads its class members - a separate scale plane, push constants, `@workgroup` staging. +**Cooperative-matrix element loops carry `Unroll`.** `coopmatClamp` walks a coopmat local +element by element through a hand-emitted structured loop bounded by +`OpCooperativeMatrixLengthKHR`, and its `OpLoopMerge` sets loop control `Unroll` - the control +glslang emits for `[[unroll]]`. Rolled, the dynamic per-element `OpAccessChain` index demotes +the accumulator out of tensor-register form into addressable storage for the whole kernel, not +only for the loop. On an RTX 5060 Ti (driver 610.74) the cm2 l-tile min-kernel runs 34.2 TFLOP/s +rolled and 57.8 unrolled. + +### 3.1 The 8/16-bit small-integer surface {#small-int-surface} + +A shader reads AND writes `int8`/`uint8`/`int16`/`uint16`/`float16` SSBO elements and struct +members, and the write direction costs the emitter no arm of its own. A narrowing daslang cast +(`int8(v)`, `uint16(u)`) is one of the conversion opcodes `convert_op` already picks for any +narrowing pair, and a store reaches its element through the same width-aware std430 access chain +a load reads - so `ensure_member_storage_caps`, pulling `StorageBuffer8BitAccess` / +`StorageBuffer16BitAccess` per member width, serves both directions from one call site. That is +what lets a shader write quantized data - int8 quants beside an f16 scale - instead of packing +32-bit words by hand. + +`unpack8` extends the same way. The `daslib/shader_lingua_franca.das` overloads add +`int16 -> byte2` and `uint16 -> ubyte2` beside the 32-bit pair, and every one of them lowers +through the single `OpBitcast` the emitter already emits for the name, so a 16-bit quant read +needs no emitter change at all. The `byte4`/`ubyte4` type factory pulls the `Int8` capability, +and widening an unpacked lane (`int4(b4)`) is a same-class `OpSConvert`, which gives sign +extension for free. + +### 3.2 A cm2 tile shape is one struct declaration {#cm2-tile-markers} + +The workgroup-scope cooperative-matrix tiles are marker structs in `spirv_builtins.das` whose +NAMES carry their geometry - `coopmatWg{A|B|Acc}_{f16|f32|s8|s32}_{R}x{C}` - and +`coopmat_wg_info` parses that name instead of looking the struct up in a table. Adding a tile +shape is therefore one struct declaration plus the overload that types the das call +(`coopmatMulAdd` for a multiply tile, `coopmatConvert` for an accumulator-only tile): no +emitter arm changes, because every cm2 arm reads rows, columns and component width out of the +parse. + ## 4. Test architecture - "every emitted instruction has a test" The behavioral layers, then the enforcement gates (all in main-tree `tests/spirv/` except the diff --git a/modules/dasSpirv/REVIEW.md b/modules/dasSpirv/REVIEW.md index 3b97084670..1d1a9bebeb 100644 --- a/modules/dasSpirv/REVIEW.md +++ b/modules/dasSpirv/REVIEW.md @@ -1,25 +1,29 @@ # dasSpirv Code Review Checklist -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist. Shared -emitter rules: `modules/REVIEW_SHADER_EMITTERS.md` - apply that list with this one.** -Architecture doc: `ARCHITECTURE.md`. +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +doc: `ARCHITECTURE.md`. Shared emitter rules: `modules/REVIEW_SHADER_EMITTERS.md` - apply that +list with this one. -- **A diff that adds an emitter capability also adds its tests under `tests/spirv/` (repo - root), in the same change:** a `_golden/` disassembly or equivalence fixture, and presence - in the opcode census. +**A diff that adds an emitter capability - a name the emitter recognizes, an opcode it emits, +or a type it accepts - also adds its fixture under `tests/spirv/` (repo root), in the same +change: a kernel in `tests/spirv/_spirv_common.das` that uses the capability, with a row in +`tests/spirv/_gen_golden.das` and its `_golden/.txt`, or an assertion on the emitted +words.** A capability no fixture exercises is emitted by nothing the suite runs. -- **A diff that adds a rejection path also adds its fixture under `tests/spirv/_fail_closed/` - (repo root) and asserts that fixture's error text in `tests/spirv/test_fail_closed.das`, in - the same change.** A rejection path is emitter code that refuses a construct with a compile - error. +**A diff that adds a rejection path also adds its fixture under `tests/spirv/_fail_closed/` +(repo root) and asserts that fixture's error text in `tests/spirv/test_fail_closed.das`, in +the same change.** A rejection path is emitter code that refuses a construct with a compile +error. -- **A diff that adds a capability a downstream consumer uses also adds a test there, in the - same change** - in dasVulkan `tests/integration` or the dasLLAMA vulkan kernel suite - that - runs the kernel on a device and compares the result against the same computation run on the - CPU. +**A diff that adds a capability a downstream consumer uses also adds a test there, in the +same change** - in dasVulkan `tests/integration` or the dasLLAMA vulkan kernel suite - that +runs the kernel on a device. -- **A diff that leaves a fixture emitting an opcode the opcode census does not declare, or - leaves the census declaring an opcode no fixture emits, is a defect.** +**A device test's oracle is the same computation run on the CPU** - the kernel body, or the +CPU twin the emitter's builtins mirror - not an expectation re-spelled inline in the test. -- **A diff under `modules/dasSpirv` that edits a file under `modules/dasGlsl` or - `modules/dasOpenGL` is a defect** - dasSpirv copies dasGlsl's design, not its code. +**Weakening `tests/spirv/test_census.das` is a defect** - it holds every fixture opcode +declared and every declared opcode emitted, in both directions. + +**A diff under `modules/dasSpirv` that edits a file under `modules/dasGlsl` or +`modules/dasOpenGL` is a defect** - dasSpirv copies dasGlsl's design, not its code. diff --git a/modules/dasSpirv/spirv/spirv_builtins.das b/modules/dasSpirv/spirv/spirv_builtins.das index 68e130033b..67ee0d454e 100644 --- a/modules/dasSpirv/spirv/spirv_builtins.das +++ b/modules/dasSpirv/spirv/spirv_builtins.das @@ -412,8 +412,7 @@ def public coopmatMulAdd(a : coopmatA_s8_16x32; b : coopmatB_s8_32x16; c : coopm // ===== cooperative matrices v2 (SPV_NV_cooperative_matrix2 — tensor addressing, workgroup scope) ===== // A cm2 tile is WORKGROUP-scoped (the whole workgroup owns one MxN tile; the driver manages staging) // and is addressed through tensor LAYOUT objects (dims + strides + a per-load slice window + an -// optional q8-style block size) instead of a flat (idx, stride) pair. The wg markers are name-parsed: -// coopmatWg{A|B|Acc}_{f16|f32|s8|s32}_{R}x{C} — declare a new struct here and coopmat_info picks it up. +// optional q8-style block size) instead of a flat (idx, stride) pair. struct coopmatWgA_f16_64x16 {} struct coopmatWgB_f16_16x16 {} struct coopmatWgB_f16_16x32 {} @@ -444,6 +443,8 @@ struct coopmatWgB_f16_32x32 {} struct coopmatWgB_f16_32x128 {} struct coopmatWgB_f16_64x64 {} struct coopmatWgB_f16_128x32 {} +[arch(at="../ARCHITECTURE.md#cm2-tile-markers")] +struct coopmatWgAcc_f16_64x128 {} struct coopmatWgAcc_f32_64x128 {} // the int8 twins (q8 x q8 -> s32 at workgroup scope; K-tile 32) struct coopmatWgA_s8_64x32 {} diff --git a/modules/dasSpirv/spirv/spirv_emit.das b/modules/dasSpirv/spirv/spirv_emit.das index 2c582f1577..8e992bec07 100644 --- a/modules/dasSpirv/spirv/spirv_emit.das +++ b/modules/dasSpirv/spirv/spirv_emit.das @@ -338,6 +338,7 @@ def private is_rt_stage(stage : ShaderStage) : bool { // Pull the StorageBuffer8/16BitAccess capability for a sub-32-bit member reachable through an // @ssbo (std430) layout. std140 blocks never see sub-32 members (compute_block_layout rejects // them), so std140 calls are no-ops by construction. +[arch(at="../ARCHITECTURE.md#small-int-surface")] def private ensure_member_storage_caps(var m : SpirvModule; bt : Type; rules : BlockLayoutRules) { if (rules != BlockLayoutRules.std430) return let w = scalar_width(bt) @@ -525,9 +526,8 @@ def private is_subpass_input_type(t : TypeDecl?) : bool { return "{t.structType.name}" == "subpassInput" } -// Parse a WORKGROUP-scope cm2 marker name: coopmatWg{A|B|Acc}_{f16|f32|s8|s32}_{R}x{C}. The caller -// already verified the "coopmatWg" prefix. Name-parsed (not a table) so adding a tile shape is one -// struct declaration in spirv_builtins.das. One peek_data pass; `good` fails the parse closed. +// The caller already verified the "coopmatWg" prefix. One peek_data pass; `good` fails the parse closed. +[arch(at="../ARCHITECTURE.md#cm2-tile-markers")] def private coopmat_wg_info(n : string) : tuple { var ok = false var isf = false @@ -2221,6 +2221,7 @@ def private scalar_type_name_class(name : string) : tuplef32) / SConvert / UConvert (SPIR-V narrowing keeps low bits = the das // C-truncation); class changes are value conversions (width-agnostic opcodes). A cross-sign // cross-width int cast is supported only when it NARROWS; widening fails closed. +[arch(at="../ARCHITECTURE.md#small-int-surface")] def private convert_op(src_cls, src_w, dst_cls, dst_w : int) : tuple { if (src_cls == dst_cls) { if (src_w == dst_w) return (ok = false, code = SpvOp.Nop) // no-op — caller's case @@ -4487,6 +4488,7 @@ class SpirvEmit : AstVisitor { // ----- calls: texture / vector constructors / dot / GLSL.std.450 math (args pre-visited) ----- // Mirrors emit_call but reads operand ids via value_of (children already visited). for-source // range()/urange() calls are consumed by preVisitExprForBody and skipped here. + [arch(at="../ARCHITECTURE.md#files-and-emission"), arch(at="../ARCHITECTURE.md#small-int-surface")] def override visitExprCall(var expr : ExprCall?) : ExpressionPtr { if (key_exists(for_src, intptr(expr))) return expr var bm & = unsafe(*m) @@ -5032,8 +5034,7 @@ class SpirvEmit : AstVisitor { return expr } // coopmatClamp(mat, lo, hi): per-element FClamp over a float accumulator tile — the - // f16-accumulate ±65504 overflow guard. OpCooperativeMatrixLengthKHR bounds a hand-emitted - // structured loop that AccessChains each element of the Function-storage tile local. + // f16-accumulate ±65504 overflow guard. if (origin_name == "coopmatClamp") { if (ctx.stage != ShaderStage.Compute) { errs |> push("coopmatClamp is only valid in a compute shader") @@ -5089,7 +5090,7 @@ class SpirvEmit : AstVisitor { let bt = type_bool(bm) let cnd = alloc_id(bm) emit(bm, SEC_FUNCS, SpvOp.ULessThan, bt, cnd, iv, len) - emit(bm, SEC_FUNCS, SpvOp.LoopMerge, merge, cont, 0u) + emit(bm, SEC_FUNCS, SpvOp.LoopMerge, merge, cont, uint(SpvLoopControl.Unroll)) emit(bm, SEC_FUNCS, SpvOp.BranchConditional, cnd, body, merge) emit(bm, SEC_FUNCS, SpvOp.Label, body) let ept = type_pointer(bm, SpvStorageClass.Function, comp_t) @@ -5525,9 +5526,6 @@ class SpirvEmit : AstVisitor { e2id[k] = res return expr } - // unpack8 / pack32: reinterpret one 32-bit scalar as four 8-bit lanes (and back) — a single - // OpBitcast; the byte4/ubyte4 type factory pulls the Int8 capability. Widening the unpacked - // lanes (`int4(b4)`) rides the existing OpSConvert path, giving sign extension for free. if (name == "unpack8" || name == "pack32") { if (length(expr.arguments) != 1) { errs |> push("{name} expects 1 argument") diff --git a/modules/dasVulkan/ARCHITECTURE.md b/modules/dasVulkan/ARCHITECTURE.md new file mode 100644 index 0000000000..3a99b73d51 --- /dev/null +++ b/modules/dasVulkan/ARCHITECTURE.md @@ -0,0 +1,269 @@ +# dasVulkan - the Vulkan binding and its boost layer + +**Read `ARCHITECTURE_COMMON.md` (repo root) first - its contract binds this document.** The +checklist that binds a diff here is `REVIEW.md` (this folder); `generator/` and `tutorials/` +carry their own. Planned work: `ROADMAP.md`. Agent instructions - build, run, test, where +things live: `CLAUDE.md`. The original boost-layer design plan is archived at +`history/dasVulkan/ORIGINAL_PLAN.md`. + +## 1. The two layers + +dasVulkan binds [Vulkan](https://www.vulkan.org/) from the Khronos `vk.xml` registry and lives +in-tree at `modules/dasVulkan/`. It builds by default: the root CMake option +`DAS_VULKAN_DISABLED` defaults to `OFF`, and the headers plus volk are vendored, so building +needs no Vulkan SDK. + +- **`vulkan`** - the raw binding: the full API, core and extensions, generated as a daslang + C++ module dispatching through [volk](https://github.com/zeux/volk). It mirrors the C API + 1:1. +- **`vulkan_boost`** - the ergonomic layer, pure daslang: RAII handle wrappers, view structs + taking `array` with auto-filled `sType`, named and defaulted arguments, block brackets, + and windowing. + +The split is what keeps the ergonomic half soft. Everything hard - ABI, dispatch, extension +loading - is generated C++ behind a rebuild; everything ergonomic is `.das` a user reads and +edits in place, with no rebuild at all. + +One build produces both halves: `libDasModuleVulkan` (static, for `daslang_static` and +embedders) and the `dasModuleVulkan.shared_module` twin that the dynamic `daslang` / +`daslang-live` host loads through `.das_module`. The shared twin has ~55 generated translation +units that are template-heavy enough to exhaust a 7 GB CI runner at full parallelism, so every +lane builds that one target at `--parallel 2`. + +`daspkg` treats `require_package("dasVulkan")` as in-tree and reports *part of this daslang +tree - nothing to install*, so in-repo example `.das_package` manifests do not declare it. + +## 2. The generator and the skip ratchet + +`generator/*.das` parses the `vk.xml` vendored under `vendor/` at its SDK tag, using +`dasPUGIXML`, and emits both layers: C++ into `src/*.gen.*` and boost `.das` into +`daslib/vulkan_*.das`. Both are committed, following the dasGlfw and dasSQLITE convention, so +a checkout builds without running the generator. + +A full run also writes `generator/skip_report.txt`: every struct and command the emitter could +not handle, with its reason, sorted. The committed copy is the golden baseline. CI regenerates +into scratch directories and diffs the fresh report against it, so an emitter change that +quietly grows the skip tail - dropping bound surface - turns the lane red. The generated +sources themselves are not byte-diffed, because their churn is large and uninteresting; the +skip tail is the part whose growth is a loss. + +The generated files are lint-clean by construction: the emitter writes conforming code, so a +lint finding in one of them points at the emitter rather than at the file. + +## 3. Boost file layout + +The boost `.das` files form an acyclic graph. `vulkan_runtime` (hand-written) and +`vulkan_ctors` (generated) are the roots; `vulkan_handles` sits on `vulkan_runtime`; +`vulkan_structs` on `vulkan_ctors` and `vulkan_handles`; `vulkan_commands` (generated +creators) and `vulkan_cmds` (generated plain commands) on `vulkan_handles` and +`vulkan_structs`; `vulkan_boost` (hand-written) requires all of them and re-exports them; +`vulkan_window` (hand-written) sits on `vulkan_boost`. Every file declares `module ` and +`require vulkan public`. + +`vulkan_runtime` exists so the generated files can reach `vk_check`, `array_addr` and +`weak_copy` without a cycle: `vulkan_boost` re-exports the generated files, so it sits above +them and cannot also sit below them. + +A daslib file registers in exactly one place - the `boost_paths` list in `.das_module`. CMake +derives the compiled-in resolver rows from that descriptor at configure time +(`ADD_MODULE_DAS_FROM_DESCRIPTOR(vulkan daslib)` in `CMakeLists.txt`), and `REVIEW.das` +censuses the descriptor against the folder, so a file the list misses and a listed name whose +file is gone both fail review. + +## 4. Handles are stored as uint64 {#handles-uint64} + +A boost wrapper stores its Vulkan handle as `uint64`, not as the handle pointer type, and +reinterprets at the C boundary. Vulkan handles are const-tracked pointers, so copying a const +handle into a non-const struct slot raises `error[30915]`; `uint64` is their ABI form and +copies without friction. This is the systemic fix for every const-pointer-copy problem in the +layer, not a local workaround. + +A wrapper holds its parent the same way - a raw `uint64` field such as `Buffer._device` - +never as a nested wrapper, so a child never owns its parent. + +## 5. Ownership {#ownership} + +A boost wrapper carries `_needs_delete`, and its generated `finalize` destroys the handle only +when that flag is set. Ownership travels with the value and the scope machinery does the +freeing; there is no GC safety net behind a Vulkan handle, because a handle is a raw pointer. + +An owner is declared `var inscope`, which runs `finalize` at scope exit in reverse declaration +order - the order Vulkan requires, children before parents. A plain `var x <- create_*()` +leaks. + +`weak_copy(x)` is the intentional non-owning alias: it copies the wrapper and clears +`_needs_delete`, so only the original frees. The copy is a plain struct copy because a wrapper +stores its handle as `uint64` (sec.4). Handle fields inside a view struct take a `weak_copy`, +because the `create_*` that produced the handle keeps ownership. + +A composite struct that owns several handles - `OffscreenTarget`, `OffscreenDepth`, +`HostBuffer`, `AddressBuffer`, `ShaderBindingTable`, `AccelStructure`, `Swapchain`, +`FrameSync` - carries a hand-written `finalize` that deletes its members in reverse dependency +order, so the composite behaves like a single owner to whoever holds it. + +## 6. Filling a boost view struct + +Boost view-struct fields keep the C spelling - `renderPass`, `pAttachments`, +`queueFamilyIndex`: camelCase with the Hungarian `p`, not `render_pass` or `attachments`. The +generated marshalling maps boost field to raw `Vk*` field by position, so the names are free to +change, but changing them is a churning public-API rename entangled with two other decisions +(`ROADMAP.md`). `pNext` is the one renamed field: `next : void?`, a raw escape hatch. + +A CreateInfo view is filled through its named-argument constructor - +`Foo(scalar = x, handle = weak_copy(h), arr <- [..])`. Non-copyable array fields are +move-initialized with `<-`. + +Two field kinds cannot go in that constructor and are assigned after it: + +- **Nested raw `Vk*` struct fields.** `extent`, `subresourceRange` and `imageSubresource` are + the native `VkExtent3D` / `VkImageSubresourceRange` types, not boost wrappers, so a nested + `Extent3D(..)` constructor fails `error[30915]`. Write `ci.extent.width = ..` after. +- **Bitfield fields.** `usage.transfer_dst = true`, `samples._1 = true`, + `aspectMask.color = true` - a bitfield has no named-argument constructor. + +Leading with a non-empty constructor is what keeps those residual assignments lint-clean: +STYLE013 fires on field-by-field filling after an empty or default init, and not after a +constructor that already initialized something. + +## 7. Count fields + +Count fields are derived from array length and are not part of the public surface. The +exceptions are the optional and `noautovalidity` arrays - `descriptorCount` without samplers is +the standing case - which stay settable boost fields under the independent-count model: the +view emits `count != 0 ? count : max(length of the referencing arrays)`. An explicit non-zero +count therefore wins, and zero means "derive it". + +## 8. Raw-layer out-parameters + +In the raw `vulkan` binding a single out-handle - a command with no `len` parameter - is +by-reference: pass `var h`. An array out-handle - a command that has a `len`, even when the +count is 1 - is a double pointer: pass `addr(h)`. The boost creators and commands hide this +distinction; it is visible only when calling the raw layer directly. + +## 9. Portability: macOS and MoltenVK {#portability-subset} + +macOS works through MoltenVK with no opt-in. The host needs one setup step, +`brew install molten-vk vulkan-loader vulkan-tools`; four pieces then make it work, all of them +platform-agnostic in the code: + +- **Loader discovery.** `das_volkInitialize` (`src/dasVULKAN.main.cpp`, `__APPLE__` branch) + falls back to dlopen of the loader from `$VULKAN_SDK` or the Homebrew prefix when the + built-in volk search misses it, then wires volk through `volkInitializeCustom`. +- **Instance portability.** A portability driver rejects `vkCreateInstance` with + `ERROR_INCOMPATIBLE_DRIVER` unless `VK_KHR_portability_enumeration` is enabled and the + matching create flag is set. `create_instance` adds both when the loader advertises the + extension, so the same user code runs unchanged everywhere. +- **Device portability.** The spec requires enabling `VK_KHR_portability_subset` on any device + that advertises it, or `vkCreateDevice` fails. `append_portability_subset` appends it, and + every boost device creator routes through that one helper. +- **Metal surface.** `vk_surface_from_native` has a Metal arm that creates a `VkSurfaceKHR` + from a `CAMetalLayer` through `vkCreateMetalSurfaceEXT`; the Cocoa and QuartzCore code is + isolated to `src/dasVULKAN.metal.mm`. + +A windowed application calls `glfwInitVulkanLoader(vk_get_instance_proc_addr())` before +`glfwInit`, so GLFW finds the same loader the module found. dasGlfw binds that call. + +Code that calls raw `vkCreateDevice` instead of a boost creator gets none of this: nothing +appends the extension on its behalf. + +## 10. The 8/16-bit storage set {#storage-8-16} + +`shaderInt16` is a core `VkPhysicalDeviceFeatures` bit, set on `f2.features` directly - it is +not in the Vulkan11 or Vulkan12 chain that carries `storageBuffer16BitAccess`, +`storageBuffer8BitAccess`, `shaderFloat16` and `shaderInt8`. It belongs with them all the same: +a shader declaring an `int16` SSBO member pulls the SPIR-V `Int16` capability, which needs +`shaderInt16`. Those five bits are one set, and the layer treats them as one. + +`storage_8_16_supported` probes the whole set. Each `create_device_storage_8_16*` creator +enables every bit that probe reads, threading `VkPhysicalDeviceVulkan11Features` into +`VkPhysicalDeviceVulkan12Features` through the features2 overload and setting `shaderInt16` on +the core block. `cooperative_matrix2_supported` probes the same core `shaderInt16` bit, because +the cm2 device its name gates is a superset of this one. + +The probe and the creators are two halves of one claim. A bit that one side names and the other +misses either fails `vkCreateDevice` or lets a kernel use a feature the device never enabled. + +## 11. The Vulkan 1.3 subgroup pair {#subgroup-pair} + +`subgroupSizeControl` and `computeFullSubgroups` are a pair: a compute pipeline may set +`REQUIRE_FULL_SUBGROUPS` only on a device where both are enabled. +`compute_full_subgroups_supported` reports the pair, returning false below API 1.3 before it +reads any feature bit, because the struct carrying them is 1.3 core. + +Every optional feature block joins a creator's pNext chain only when its extension is enabled +and the device reports the bit: a struct for an extension the device did not enable fails +`vkCreateDevice`, so each creator queries first and chains second. + +Every ladder creator that can enable the pair does so opportunistically: it queries the pair +first, and chains a `VkPhysicalDeviceVulkan13Features` onto the tail of its pNext chain only +when the device reports both bits, after whichever optional struct that creator chained last. A +device missing the pair degrades to the same device without it rather than failing to create. +That tail position is why the chaining is a per-creator ladder: only the creator knows which of +its own optional blocks is last. `create_device_coopmat_full_subgroups`, which predates the +ladder, is the other shape: it chains its `VkPhysicalDeviceVulkan13Features` unconditionally in +mid-chain and copies the reported bits into it - a 1.3 core struct, so the chain is legal +either way, and a missing pair leaves the bits off. + +`create_compute_pipeline_full_subgroups` is the consumer: `create_compute_pipeline` with +`REQUIRE_FULL_SUBGROUPS` set on the stage, which lets the shader compiler drop its +partial-subgroup guards. A pipeline it returns is valid only on a device from a creator that +enabled the pair, and only for a workgroup width that is a multiple of the subgroup size. + +## 12. Documentation pipeline + +`utils/vulkan2rst.das` documents the ergonomic layer into the generated stdlib pages of the +main Sphinx tree by RTTI introspection, modeled on the `imgui2rst` of dasImgui. The hand-filled +module intros are tracked; the generated pages are not. Tutorial pages live at +`doc/source/reference/tutorials/vulkan/`. + +A helper reaches a page only through a `group_by_regex` group in `vulkan2rst.das`. A helper in +no group is emitted nowhere and nothing turns red - this is the one part of the pipeline with +no gate behind it, which is why `REVIEW.md` carries the duty. + +A generated page emits a `:ref:` for every type it mentions. A type the boost layer does not +own needs a label in `doc/source/stdlib/vulkan_external_types.rst` (repo root) or that `:ref:` +dangles; the `-W` Sphinx build in `doc.yml` is paths-filtered on `modules/dasVulkan/**`, so it +catches a missing label per PR. + +The raw `vulkan` binding and the generated `vulkan_structs` (~2000 symbols), `vulkan_cmds` and +`vulkan_ctors` mirror Vulkan 1:1 and are deliberately not re-documented: the overview page +explains the patterns and points at the spec, which stays correct as the registry grows. + +Doc snippets are not compile-checked. + +## 13. Tutorial units + +A tutorial is a self-contained unit under `tutorials//`: an offscreen module, its +`[compute_shader]` or `[shader]` blob, a pixel-oracle `[test]` that CI gates, and a +`recording/` driver. Its windowed viewer lives at `/window/show_.das`, which is +where the tutorials `.das_test` skips it - the lavapipe CI daslang build is +`-DDAS_GLFW_DISABLED=ON`, so it has no display and no GLFW. + +Shared helpers are copied per tutorial rather than factored out, because daslang `require` +cannot parse an unquoted path segment starting with a digit: +`require ../../02_mandelbrot/window/x.das` fails with `error[30151] unexpected integer +constant`. The `mandelbrot_compute` of 02 and the `resident_compute` of 03 are the same generic +resident single-float-pushconstant compute-to-image builder, held as two copies for that +reason. A non-digit shared path such as `tutorials/common/` is what a third windowed compute +tutorial would need. + +## 14. CI gates + +- `.github/workflows/vulkan_checks.yml` - the per-PR gate, paths-filtered to + `modules/dasVulkan/**` so an unrelated PR pays nothing. It carries the two cheap correctness + gates: the generator skip ratchet, and the module-wide lint over every `.das` here. +- `.github/workflows/nightly_vulkan.yml` - the render suite. Full integration plus tutorial + pixel-oracle tests on Mesa lavapipe (a software ICD, no GPU) on Linux, plus a build and + loader-discovery smoke on macOS. Nightly and on demand only. + +Windows has no lane: it needs a software ICD wired up (`ROADMAP.md`). The macOS lane is a build +and smoke gate rather than a render gate, because GitHub-hosted macOS runners expose only a +paravirtualized GPU that MoltenVK cannot render the suite on. + +`tests/integration/` is in-process dastest - offscreen render to an image with pixel readback, +and compute into a storage buffer. No window, no subprocess. A test body calls +`volkInitialize()` itself, because nothing in the harness does it. + +## 15. Exception ledger + +Empty. No rule in `REVIEW.md` has a ruled-acceptable case here yet. diff --git a/modules/dasVulkan/CLAUDE.md b/modules/dasVulkan/CLAUDE.md index 4735b8c8bc..5e1e3a61ad 100644 --- a/modules/dasVulkan/CLAUDE.md +++ b/modules/dasVulkan/CLAUDE.md @@ -1,20 +1,25 @@ # dasVulkan module instructions -dasVulkan is the daslang binding + ergonomic boost layer for [Vulkan](https://www.vulkan.org/), generated from the Khronos `vk.xml` registry, **in-tree at `modules/dasVulkan/`** and built by default (root CMake option `DAS_VULKAN_DISABLED`, default `OFF`; headers + volk are vendored, so no Vulkan SDK is needed to build). Two layers: +dasVulkan is the daslang binding + ergonomic boost layer for [Vulkan](https://www.vulkan.org/), +in-tree at `modules/dasVulkan/`. **How it is built and why - the two layers, the generator, the +ownership model, the boost conventions, the macOS mechanics - is `ARCHITECTURE.md` beside this +file. Read the section you are about to work in before writing code here.** The rules binding a +diff are `REVIEW.md`; postponed work is `ROADMAP.md`. -- **`vulkan`** - the raw binding: the full Vulkan API (core + extensions), generated as a daslang C++ module dispatching through [volk](https://github.com/zeux/volk). Mirrors the C API 1:1. -- **`vulkan_boost`** - the ergonomic layer (pure daslang, no rebuild to edit): RAII handle wrappers, idiomatic `array` structs with auto-filled `sType`, named/defaulted args, block brackets, windowing. - -The old standalone repo (borisbat/dasVulkan) is archived with full history. `daspkg` recognizes `require_package("dasVulkan")` as in-tree and reports *part of this daslang tree - nothing to install*; in-repo example `.das_package` manifests do NOT declare it. - -Follow the daslang **gen2** conventions (the root `CLAUDE.md` rules apply to every `.das` file). This file captures only the dasVulkan-specific truths. +Follow the daslang **gen2** conventions - the root `CLAUDE.md` rules apply to every `.das` file +here. ## Locations -- Module source: `modules/dasVulkan/` (`src/`, `daslib/`, `generator/`, `examples/`, `tutorials/`, `utils/`, `vendor/`) -- Tests: `modules/dasVulkan/tests/integration/` - nightly CI lane `.github/workflows/nightly_vulkan.yml` -- Docs: stdlib section + generated pages in the main Sphinx tree; tutorials at `doc/source/reference/tutorials/vulkan/` -- Recordings: intermediates under `tutorials/**/recording/` (gitignored); MP4 deliverables on the rolling `docs-assets` GitHub release +- Module source: `modules/dasVulkan/` (`src/`, `daslib/`, `generator/`, `examples/`, + `tutorials/`, `utils/`, `vendor/`) +- Tests: `modules/dasVulkan/tests/integration/` +- CI: `.github/workflows/vulkan_checks.yml` (per-PR), `.github/workflows/nightly_vulkan.yml` + (render suite) +- Docs: stdlib section + generated pages in the main Sphinx tree; tutorials at + `doc/source/reference/tutorials/vulkan/` +- Recordings: intermediates under `tutorials/**/recording/` (gitignored); MP4 deliverables on + the rolling `docs-assets` GitHub release ## Skill files (REQUIRED) @@ -26,85 +31,49 @@ Follow the daslang **gen2** conventions (the root `CLAUDE.md` rules apply to eve ## Build & run -The boost layer is pure daslang - **editing `daslib/*.das` needs no rebuild**. Only C++ or generator changes need the native module rebuilt, and then it is the ordinary in-tree build: +The boost layer is pure daslang - **editing `daslib/*.das` needs no rebuild**. Only C++ or +generator changes need the native module rebuilt, and then it is the ordinary in-tree build: ``` cmake --build build --config Release ``` -Both halves come out of that one build: `libDasModuleVulkan` (static, for `daslang_static` and embedders) and the `dasModuleVulkan.shared_module` twin the default dynamic `daslang` / `daslang-live` host loads via `.das_module`. Run anything from the repo root: +**Build the shared twin at `--parallel 2`, not unbounded** - unbounded exhausts a 7 GB runner +(`ARCHITECTURE.md` sec.1): ``` -bin/Release/daslang -project_root . modules/dasVulkan/examples/offscreen_triangle_boost.das +cmake --build ./build --parallel 2 --target dasModuleVulkan ``` -**Build the shared twin at `--parallel 2`, not unbounded** - the ~55 template-heavy generated TUs OOM a 7 GB CI runner under full parallelism. Both dasVulkan CI lanes do exactly this (`cmake --build ./build --parallel 2 --target dasModuleVulkan`). - -## The generator - -`generator/*.das` parses `vk.xml` (vendored under `vendor/` at the SDK tag) with `dasPUGIXML` and emits both layers: - -- C++ -> `src/*.gen.*` (committed, per the dasGlfw/dasSQLITE convention). -- boost -> `daslib/vulkan_*.das`. - -`daslang generator/generate.das` regenerates everything; **`--no-cpp` regenerates only the boost** (the fast iteration loop - no C++ rebuild). `--boost-out` defaults to `daslib`. +Run anything from the repo root: -**The skip ratchet.** Full runs also write `generator/skip_report.txt` - every struct/command the generator could not emit, with its reason, sorted. The committed copy is the golden baseline: CI re-runs the generator into scratch dirs and diffs the regenerated report against it, so an emitter change that silently grows the skip tail (drops generated surface) fails the gate. A justified change regenerates the report in the same commit. (Full byte-diff of the generated sources themselves is NOT gated - regen locally and check the git diff when touching the emitter.) - -## Boost file layout (acyclic) - -`vulkan_runtime` (hand) <- `vulkan_ctors` (gen) <- `vulkan_handles` (gen) <- `vulkan_structs` (gen) <- `vulkan_commands` (gen creators) <- `vulkan_cmds` (gen plain commands) / `vulkan_boost` (hand) / `vulkan_window` (hand). Each file is `module ` + `require vulkan public`. - -**A daslib file registers in ONE place: the `boost_paths` list in `.das_module`.** CMake derives the compiled-in resolver rows from the descriptor at configure (`ADD_MODULE_DAS_FROM_DESCRIPTOR(vulkan daslib)` in `CMakeLists.txt`), and `REVIEW.das`'s descriptor census fails review on a daslib `.das` the list misses or a listed name whose file is gone. - -## Docs +``` +bin/Release/daslang -project_root . modules/dasVulkan/examples/offscreen_triangle_boost.das +``` -`utils/vulkan2rst.das` (RTTI introspection, modeled on dasImgui's `imgui2rst`) documents the ergonomic layer into the main Sphinx tree's generated stdlib pages; the hand-filled module intros are tracked, the generated pages are not. Tutorial pages live at `doc/source/reference/tutorials/vulkan/`. +## Regenerating -- A new public `vulkan_boost` / `vulkan_window` helper must land in a `group_by_regex` group in `vulkan2rst.das` **in the same change**, or it is silently undocumented. A type it exposes that the boost layer does not own needs a label in `doc/source/stdlib/vulkan_external_types.rst` (repo root), or the `:ref:` dangles and the `-W` docs build goes red. -- The raw `vulkan` binding and the generated `vulkan_structs` (~2000 symbols), `vulkan_cmds`, `vulkan_ctors` mirror Vulkan 1:1 and are deliberately **not** re-documented - the overview page explains the patterns and points at the spec. -- Doc snippets are not compile-checked - verify field names against the real `examples/` before writing one (the boost field names are not what you'd guess - see below). +`daslang generator/generate.das` regenerates both layers. **`--no-cpp` regenerates only the +boost** - the fast iteration loop, no C++ rebuild. `--boost-out` defaults to `daslib`, and a +full run rewrites `generator/skip_report.txt`. ## Tests -`tests/integration/` is in-process dastest (offscreen render to image + pixel readback; compute to a storage buffer - no window, no subprocess). CI renders on Mesa lavapipe (software ICD, no GPU). Run from the repo root: +Run from the repo root, so the cwd-relative shader paths resolve: ``` bin/Release/daslang -project_root . dastest/dastest.das -- \ --test modules/dasVulkan/tests/integration --isolated-mode --isolated-mode-threads 4 ``` -Run from the repo root so the cwd-relative shader paths resolve. Test bodies must call `volkInitialize()` themselves. - -## CI - -- **`.github/workflows/vulkan_checks.yml`** - the per-PR gate, paths-filtered to `modules/dasVulkan/**` so unrelated PRs pay nothing. Two cheap correctness gates: the generator skip ratchet and the module-wide lint. -- **`.github/workflows/nightly_vulkan.yml`** - the render suite: full integration + tutorial pixel-oracle tests on lavapipe (Linux) plus a build + loader-discovery smoke on macOS. Nightly and on-demand only. Windows has no lane (needs lavapipe/SwiftShader wiring - see `ROADMAP.md`). - -**Lint is mandatory and gated.** The whole module is lint-clean - keep it that way. The **generated** `daslib/vulkan_*.das` files are lint-clean *by construction*: fix the emitter in `generator/vk_emit_boost.das` and regenerate, never hand-edit. - -## Tutorials - -Each tutorial is a self-contained unit under `tutorials//`: the offscreen module + its `[compute_shader]`/`[shader]` blob + a pixel-oracle `[test]` (the CI gate) + a `recording/` driver. +The gates a change here answers to: -The windowed driver at `tutorials//window/show_.das`: - -- calls `glfwInitVulkanLoader(vk_get_instance_proc_addr())` **before** `glfwInit` so GLFW finds the same loader on every platform (see the macOS gotcha below); -- lives in `window/` so the tutorials `.das_test` skips it in CI (the lavapipe CI daslang build is `-DDAS_GLFW_DISABLED=ON` - no display, no GLFW). - -**Cross-tutorial requires don't work**: daslang `require` can't parse an unquoted path segment starting with a digit, so `require ../../02_mandelbrot/window/x.das` fails with `error[30151] unexpected integer constant`. Keep shared helpers tutorial-local - 02's `mandelbrot_compute` and 03's `resident_compute` are the same generic resident single-float-pushconstant compute-to-image builder, copied per tutorial. Factor to a non-digit shared path (e.g. `tutorials/common/`) only if a third windowed compute tutorial appears. - -## Key gotchas / API truths - -- **Handles are stored as `uint64` inside wrappers**, not the pointer type. Vulkan handles are const-tracked pointers; copying a const handle into a non-const struct slot is `error[30915]`. `uint64` (their ABI form) copies friction-free; `reinterpret` at the C boundary. This is the systemic fix for all const-pointer-copy pain. -- **Ownership:** declare every owner `var inscope` so `finalize` destroys it in reverse order. A plain `var x <- create_*()` leaks (handles are raw pointers, no GC safety net). Parents are stored as raw handles inside wrappers, never as nested wrappers. `weak_copy(x)` makes an intentional non-owning alias (clears `_needs_delete`). -- **Boost view-struct field names keep the C spelling** - `renderPass`, `pAttachments`, `queueFamilyIndex` (camelCase + Hungarian `p`), NOT `render_pass`/`attachments`. `pNext` -> `next : void?` (raw escape hatch). Stripping the `p` and typed pNext chains are deferred (see `ROADMAP.md`). -- **Filling a CreateInfo view:** use the named-argument constructor - `Foo(scalar = x, handle = weak_copy(h), arr <- [..])`. Handle fields take a `weak_copy` (a non-owning alias; the `create_*` keeps ownership); non-copyable array fields are move-initialized with `<-`. **Two field kinds cannot go in the ctor** and stay as field-assigns *after* it: (1) **nested raw `Vk*` struct fields** - `extent`, `subresourceRange`, `imageSubresource` are the native `VkExtent3D`/`VkImageSubresourceRange`/... not boost wrappers, so a nested `Extent3D(..)` ctor fails `error[30915]`; write `ci.extent.width = ..` after; (2) **bitfield fields** - `usage.transfer_dst = true`, `samples._1 = true`, `aspectMask.color = true` (bitfields have no named-arg ctor). A non-empty ctor init silences STYLE013 for those residual assigns, so the lint stays clean. Field-by-field-only (empty/default init) trips STYLE013 - the linter is a CI gate, so always lead with the ctor. -- **Count fields are mostly auto-derived** from array length. The exceptions (optional / `noautovalidity` arrays, e.g. `descriptorCount` without samplers) are settable boost fields under the independent-count model: the view sets `count != 0 ? count : max(referencing-array lengths)`. -- **Raw layer out-params:** single out-handle (no `len`) is by-ref (pass `var h`); array out-handle (has `len`, even count 1) is a double-pointer (pass `addr(h)`). The boost creators/commands hide this. -- **Block trailing syntax** is `f(...) $(cmd) { ... }` or `f(...) { ... }` - NO `<|` (STYLE001). -- **macOS works via MoltenVK** (no opt-in). One-time host setup: `brew install molten-vk vulkan-loader vulkan-tools`. `vk_surface_from_native` has a Metal arm (`src/dasVULKAN.metal.mm`, `vkCreateMetalSurfaceEXT` from a `CAMetalLayer`); `das_volkInitialize` finds the loader that volk's built-in macOS search misses, by dlopen'ing the Homebrew/SDK paths; `create_instance` auto-enables `VK_KHR_portability_enumeration`; and every boost-layer device creator auto-enables `VK_KHR_portability_subset` when the device advertises it (the spec requires enabling it, or `vkCreateDevice` fails). Raw `vkCreateDevice` call sites (some examples/tutorials) must append it themselves on portability devices. Windowed apps call `glfwInitVulkanLoader(vk_get_instance_proc_addr())` before `glfwInit`. See `ROADMAP.md`. +``` +bin/Release/daslang utils/lint/main.das -- modules/dasVulkan -q -j 0 +bin/Release/daslang modules/dasVulkan/REVIEW.das +``` ## Workflow -Work here goes through the repo's own PR flow (`skills/internal/make_pr.md`) and its gates; there is no module-local hook or lint recipe. `ROADMAP.md` holds postponed work, with enough context to pick each item up cold; the original boost-layer design plan is archived at `history/dasVulkan/ORIGINAL_PLAN.md`. +Work here goes through the repo PR flow (`skills/internal/make_pr.md`) and its gates; there is +no module-local hook or lint recipe. diff --git a/modules/dasVulkan/REVIEW.md b/modules/dasVulkan/REVIEW.md index 36e90130b3..97107f6307 100644 --- a/modules/dasVulkan/REVIEW.md +++ b/modules/dasVulkan/REVIEW.md @@ -1,7 +1,7 @@ # dasVulkan Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: -`CLAUDE.md`. Planned work: `ROADMAP.md`. A tutorial - a `tutorials//` unit, and any +`ARCHITECTURE.md`. Planned work: `ROADMAP.md`. A tutorial - a `tutorials//` unit, and any `record_*.das` recording driver wherever the diff puts it - answers to the `tutorials/` subfolder's checklist. A generator change - any `generator/*.das` file or `generator/skip_report.txt` - answers to the `generator/` subfolder's checklist. @@ -13,3 +13,27 @@ itself to see what it checks. **A diff touching any `src/dasVULKAN.gen*` file, or any `daslib/*.das` whose first line reads `// generated by generator/generate.das -- DO NOT EDIT`, with no accompanying `generator/` change is a defect - change `generator/` and regenerate instead.** + +**A diff that changes which feature bits `storage_8_16_supported` probes, or which bits any +`create_device_storage_8_16*` creator enables (`daslib/vulkan_boost.das`), changes both sides +in the same change.** A mismatch either fails `vkCreateDevice` or lets a kernel use a feature +the device never enabled. + +**A diff that adds a public helper to a hand-written `daslib/*.das` (`vulkan_boost.das`, +`vulkan_window.das`, `vulkan_runtime.das`) also adds it to a `group_by_regex` group in +`utils/vulkan2rst.das`, in the same change.** A helper in no group is documented nowhere, and +nothing turns red to say so. + +**A diff that adds or edits a call to raw `vkCreateDevice`, wherever the diff puts it, appends +`VK_KHR_portability_subset` to that call's extension list when the device advertises it - or +calls a `create_device*` boost creator instead, which appends it.** The Vulkan spec fails +`vkCreateDevice` on a device that advertises the extension without enabling it. + +**A diff that puts the result of a `create_*` call into a local the function neither returns +nor moves into a container it deletes declares that local `var inscope`.** A plain +`var x <- create_*()` leaks: the wrapper owns a raw Vulkan handle, and nothing frees it without +the scope-exit `finalize`. + +**A diff that adds a `[test]` under `tests/integration/` reaches `volkInitialize()` before its +first Vulkan call - in the `[test]` function, or in a helper that function calls first.** +Nothing in the harness calls it, and every Vulkan entry point is null until it runs. diff --git a/modules/dasVulkan/daslib/vulkan_boost.das b/modules/dasVulkan/daslib/vulkan_boost.das index 6ae9f759af..23d1860762 100644 --- a/modules/dasVulkan/daslib/vulkan_boost.das +++ b/modules/dasVulkan/daslib/vulkan_boost.das @@ -144,6 +144,7 @@ def public mesh_shader_supported(phys : VkPhysicalDevice) : bool { return mf.meshShader != 0u && mf.taskShader != 0u } +[arch(at="../ARCHITECTURE.md#portability-subset")] def public create_instance(application_name : string; api_version : uint; extensions : array) : Instance { var ai = VkApplicationInfo() ai.apiVersion = api_version @@ -154,9 +155,6 @@ def public create_instance(application_name : string; api_version : uint; extens unsafe { ci.pApplicationInfo = addr(ai) } - // Portability drivers (MoltenVK on macOS) reject vkCreateInstance with ERROR_INCOMPATIBLE_DRIVER - // unless VK_KHR_portability_enumeration is enabled and the matching create flag is set. Added - // transparently when the loader advertises it, so the same user code runs unchanged everywhere. var exts <- [for (e in extensions); e] if (instance_extension_available("VK_KHR_portability_enumeration")) { if (find_index(exts, "VK_KHR_portability_enumeration") < 0) { @@ -184,9 +182,7 @@ def public create_instance(application_name : string = ""; api_version : uint = return <- create_instance(application_name, api_version, no_ext) } -// Portability devices (MoltenVK) advertise VK_KHR_portability_subset and the spec REQUIRES -// enabling it whenever advertised — vkCreateDevice fails otherwise. Appended transparently at -// every raw vkCreateDevice site (create_instance's portability_enumeration twin). +[arch(at="../ARCHITECTURE.md#portability-subset")] def private append_portability_subset(phys : VkPhysicalDevice; var exts : array) { if (device_extension_available(phys, "VK_KHR_portability_subset") && find_index(exts, "VK_KHR_portability_subset") < 0) { @@ -385,9 +381,7 @@ def public create_device(phys : VkPhysicalDevice; queue_families : array; return <- b } -//! True iff ``phys`` supports the 8/16-bit SSBO storage set the dasSpirv storage wave emits: -//! ``storageBuffer16BitAccess`` (Vulkan11Features) + ``storageBuffer8BitAccess`` + ``shaderFloat16`` + -//! ``shaderInt8`` (Vulkan12Features). Gate 8/16-bit kernels — old/software drivers can miss any bit. +[arch(at="../ARCHITECTURE.md#storage-8-16")] def public storage_8_16_supported(phys : VkPhysicalDevice) : bool { var f11 = VkPhysicalDeviceVulkan11Features() var f12 = VkPhysicalDeviceVulkan12Features() @@ -398,10 +392,10 @@ def public storage_8_16_supported(phys : VkPhysicalDevice) : bool { } vkGetPhysicalDeviceFeatures2(phys, f2) return (f11.storageBuffer16BitAccess != 0u && f12.storageBuffer8BitAccess != 0u - && f12.shaderFloat16 != 0u && f12.shaderInt8 != 0u) + && f12.shaderFloat16 != 0u && f12.shaderInt8 != 0u && f2.features.shaderInt16 != 0u) } -//! Create a device with the 8/16-bit SSBO storage set enabled (the four bits storage_8_16_supported +//! Create a device with the 8/16-bit SSBO storage set enabled (the five bits storage_8_16_supported //! probes), threaded as a Vulkan11Features -> Vulkan12Features chain through the features2 overload. //! Pairs with dasSpirv's ``float16``/``int8`` SSBO members and ``unpack8``/``pack32``; gate first. def public create_device_storage_8_16(phys : VkPhysicalDevice; queue_family : uint) : Device { @@ -409,6 +403,7 @@ def public create_device_storage_8_16(phys : VkPhysicalDevice; queue_family : ui return <- create_device_storage_8_16(phys, queue_family, no_ext) } +[arch(at="../ARCHITECTURE.md#storage-8-16")] def public create_device_storage_8_16(phys : VkPhysicalDevice; queue_family : uint; extensions : array) : Device { var f11 = VkPhysicalDeviceVulkan11Features() var f12 = VkPhysicalDeviceVulkan12Features() @@ -417,6 +412,7 @@ def public create_device_storage_8_16(phys : VkPhysicalDevice; queue_family : ui f12.shaderFloat16 = 1u f12.shaderInt8 = 1u var f2 = VkPhysicalDeviceFeatures2() + f2.features.shaderInt16 = 1u unsafe { f2.pNext = addr(f11) f11.pNext = addr(f12) @@ -476,6 +472,7 @@ def public pipeline_exec_props_supported(phys : VkPhysicalDevice) : bool { //! create_device_cooperative_matrix PLUS the Vulkan 1.3 subgroup-size-control pair so pipelines can pin //! ``requiredSubgroupSize``; each bit is enabled only when the device reports it (degrades to the plain //! coopmat device). Instance must be api 1.3; opportunistically enables VK_KHR_pipeline_executable_properties. +[arch(at="../ARCHITECTURE.md#subgroup-pair")] def public create_device_coopmat_full_subgroups(phys : VkPhysicalDevice; queue_family : uint) : Device { var q13 = VkPhysicalDeviceVulkan13Features() var q2 = VkPhysicalDeviceFeatures2() @@ -491,7 +488,6 @@ def public create_device_coopmat_full_subgroups(phys : VkPhysicalDevice; queue_f f11.storageBuffer16BitAccess = 1u f12.shaderFloat16 = 1u f12.vulkanMemoryModel = 1u - // optional 1.3 bits: request only what the queried features report f13.subgroupSizeControl = q13.subgroupSizeControl f13.computeFullSubgroups = q13.computeFullSubgroups fcm.cooperativeMatrix = 1u @@ -553,6 +549,7 @@ def public integer_dot_product_supported(phys : VkPhysicalDevice) : bool { //! Create a device with the 8/16-bit SSBO storage set AND ``shaderIntegerDotProduct`` (a Vulkan11 -> //! Vulkan12 -> ShaderIntegerDotProduct chain through features2); pre-1.3 devices get the //! VK_KHR_shader_integer_dot_product string automatically. Pairs with ``sdot4``; gate with both probes. +[arch(at="../ARCHITECTURE.md#storage-8-16"), arch(at="../ARCHITECTURE.md#subgroup-pair")] def public create_device_storage_8_16_int_dot(phys : VkPhysicalDevice; queue_family : uint) : Device { var f11 = VkPhysicalDeviceVulkan11Features() var f12 = VkPhysicalDeviceVulkan12Features() @@ -562,11 +559,19 @@ def public create_device_storage_8_16_int_dot(phys : VkPhysicalDevice; queue_fam f12.shaderFloat16 = 1u f12.shaderInt8 = 1u fdot.shaderIntegerDotProduct = 1u + let with_fullsg = compute_full_subgroups_supported(phys) + var f13 = VkPhysicalDeviceVulkan13Features() + f13.subgroupSizeControl = 1u + f13.computeFullSubgroups = 1u var f2 = VkPhysicalDeviceFeatures2() + f2.features.shaderInt16 = 1u unsafe { f2.pNext = addr(f11) f11.pNext = addr(f12) f12.pNext = addr(fdot) + if (with_fullsg) { + fdot.pNext = addr(f13) + } } var props : VkPhysicalDeviceProperties vkGetPhysicalDeviceProperties(phys, props) @@ -597,6 +602,7 @@ def private push_memory_management_extensions(var ext : array; with_memp } } +[arch(at="../ARCHITECTURE.md#storage-8-16"), arch(at="../ARCHITECTURE.md#subgroup-pair")] def public create_device_storage_8_16_int_dot_coopmat(phys : VkPhysicalDevice; queue_family : uint; transfer_family : int = -1) : Device { var q12 = VkPhysicalDeviceVulkan12Features() var qmp = VkPhysicalDeviceMemoryPriorityFeaturesEXT() @@ -628,7 +634,12 @@ def public create_device_storage_8_16_int_dot_coopmat(phys : VkPhysicalDevice; q fcm.cooperativeMatrix = 1u fmp.memoryPriority = 1u fpg.pageableDeviceLocalMemory = 1u + let with_fullsg = compute_full_subgroups_supported(phys) + var f13 = VkPhysicalDeviceVulkan13Features() + f13.subgroupSizeControl = 1u + f13.computeFullSubgroups = 1u var f2 = VkPhysicalDeviceFeatures2() + f2.features.shaderInt16 = 1u unsafe { f2.pNext = addr(f11) f11.pNext = addr(f12) @@ -640,6 +651,15 @@ def public create_device_storage_8_16_int_dot_coopmat(phys : VkPhysicalDevice; q if (with_pageable) { fmp.pNext = addr(fpg) } + if (with_fullsg) { + if (with_pageable) { + fpg.pNext = addr(f13) + } elif (with_memprio) { + fmp.pNext = addr(f13) + } else { + fcm.pNext = addr(f13) + } + } } var props : VkPhysicalDeviceProperties vkGetPhysicalDeviceProperties(phys, props) @@ -676,7 +696,8 @@ def public memory_priority_supported(phys : VkPhysicalDevice) : bool { //! True iff ``phys`` can create the cm2 device: VK_NV_cooperative_matrix2 (+ its VK_KHR_cooperative_matrix //! dependency), the cm2 feature set the dasSpirv rails emit (wg scope, flexible dims, tensor addressing, block -//! loads), and every bit create_device_storage_8_16_int_dot_coopmat2 enables (``cooperativeMatrix``, ``bufferDeviceAddress``). +//! loads), and the bits create_device_storage_8_16_int_dot_coopmat2 enables (``cooperativeMatrix``, ``bufferDeviceAddress``, ``shaderInt16``). +[arch(at="../ARCHITECTURE.md#storage-8-16")] def public cooperative_matrix2_supported(phys : VkPhysicalDevice) : bool { if (!device_extension_available(phys, "VK_NV_cooperative_matrix2") || !device_extension_available(phys, "VK_KHR_cooperative_matrix")) { @@ -692,7 +713,7 @@ def public cooperative_matrix2_supported(phys : VkPhysicalDevice) : bool { fcm.pNext = addr(fcm2) } vkGetPhysicalDeviceFeatures2(phys, f2) - if (fcm.cooperativeMatrix == 0u || f12.bufferDeviceAddress == 0u) return false + if (fcm.cooperativeMatrix == 0u || f12.bufferDeviceAddress == 0u || f2.features.shaderInt16 == 0u) return false return (fcm2.cooperativeMatrixWorkgroupScope != 0u && fcm2.cooperativeMatrixFlexibleDimensions != 0u && fcm2.cooperativeMatrixTensorAddressing != 0u && fcm2.cooperativeMatrixBlockLoads != 0u) } @@ -763,9 +784,8 @@ def public select_transfer_queue_family(phys : VkPhysicalDevice) : int { //! Superset of create_device_storage_8_16_int_dot_coopmat that ALSO enables VK_NV_cooperative_matrix2 + //! ``bufferDeviceAddress`` (gate with ``cooperative_matrix2_supported``), plus the same opportunistic set //! (coopvec, residency levers, timelineSemaphore, external_memory_host) and ``transfer_family`` second queue. +[arch(at="../ARCHITECTURE.md#storage-8-16"), arch(at="../ARCHITECTURE.md#subgroup-pair")] def public create_device_storage_8_16_int_dot_coopmat2(phys : VkPhysicalDevice; queue_family : uint; transfer_family : int = -1) : Device { // nolint:STYLE038 — linear feature/extension pNext-chain assembly, one hop per capability - // query the optional feature bits first — a pNext struct may only join the chain when its - // extension is actually enabled, so each optional block is gated on ext + reported bit var q12q = VkPhysicalDeviceVulkan12Features() var qcv = VkPhysicalDeviceCooperativeVectorFeaturesNV() var qmp = VkPhysicalDeviceMemoryPriorityFeaturesEXT() @@ -814,7 +834,12 @@ def public create_device_storage_8_16_int_dot_coopmat2(phys : VkPhysicalDevice; fcv.cooperativeVector = 1u fmp.memoryPriority = 1u fpg.pageableDeviceLocalMemory = 1u + let with_fullsg = compute_full_subgroups_supported(phys) + var f13 = VkPhysicalDeviceVulkan13Features() + f13.subgroupSizeControl = 1u + f13.computeFullSubgroups = 1u var f2 = VkPhysicalDeviceFeatures2() + f2.features.shaderInt16 = 1u unsafe { f2.pNext = addr(f11) f11.pNext = addr(f12) @@ -834,6 +859,17 @@ def public create_device_storage_8_16_int_dot_coopmat2(phys : VkPhysicalDevice; if (with_pageable) { fmp.pNext = addr(fpg) } + if (with_fullsg) { + if (with_pageable) { + fpg.pNext = addr(f13) + } elif (with_memprio) { + fmp.pNext = addr(f13) + } elif (with_coopvec) { + fcv.pNext = addr(f13) + } else { + fcm2.pNext = addr(f13) + } + } } var props : VkPhysicalDeviceProperties vkGetPhysicalDeviceProperties(phys, props) @@ -1707,6 +1743,42 @@ def public create_compute_pipeline(device : Device; layout : PipelineLayout; sha return <- b } +[arch(at="../ARCHITECTURE.md#subgroup-pair")] +def public compute_full_subgroups_supported(phys : VkPhysicalDevice) : bool { + var props : VkPhysicalDeviceProperties + vkGetPhysicalDeviceProperties(phys, props) + if (props.apiVersion < make_api_version(1u, 3u, 0u)) { + return false + } + var q13 = VkPhysicalDeviceVulkan13Features() + var q2 = VkPhysicalDeviceFeatures2() + unsafe { + q2.pNext = addr(q13) + } + vkGetPhysicalDeviceFeatures2(phys, q2) + return q13.subgroupSizeControl != 0u && q13.computeFullSubgroups != 0u +} + +[arch(at="../ARCHITECTURE.md#subgroup-pair")] +def public create_compute_pipeline_full_subgroups(device : Device; layout : PipelineLayout; shader : ShaderModule; entry : string = "main") : Pipeline { + var stage = VkPipelineShaderStageCreateInfo() + stage.stage.compute = true + stage.flags.require_full_subgroups = true + stage.module_ = boost_value_to_vk(shader) + stage.pName = entry + var cp = VkComputePipelineCreateInfo() + cp.stage = stage + cp.layout = boost_value_to_vk(layout) + var h : VkPipeline + unsafe { + vk_check(vkCreateComputePipelines(boost_value_to_vk(device), null, 1u, addr(cp), null, addr(h)), null) + } + var b = vk_value_to_boost(h) + b._needs_delete = true + b._device = unsafe(reinterpret(boost_value_to_vk(device))) + return <- b +} + //! Record a single-image pipeline barrier that transitions `image` from `old_layout` to `new_layout`, //! with the given access masks and pipeline stages (color aspect, 1 mip, 1 layer). The building block //! for storage-image / transfer layout moves; used by compute_to_storage_image below. @@ -1946,6 +2018,7 @@ struct public OffscreenTarget { view : ImageView } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var t : OffscreenTarget) { delete t.view delete t.image @@ -2004,6 +2077,7 @@ struct public OffscreenDepth { view : ImageView } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var d : OffscreenDepth) { delete d.view delete d.image @@ -2072,6 +2146,7 @@ struct public HostBuffer { size : uint64 } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var hb : HostBuffer) { delete hb.buffer delete hb.memory @@ -2515,6 +2590,7 @@ struct public AddressBuffer { address : uint64 } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var ab : AddressBuffer) { delete ab.buffer delete ab.memory @@ -2666,6 +2742,7 @@ struct public ShaderBindingTable { callable : StridedDeviceAddressRegionKHR } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var sbt : ShaderBindingTable) { delete sbt.buffer } @@ -2757,6 +2834,7 @@ struct public AccelStructure { address : uint64 } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var a : AccelStructure) { delete a.handle delete a.buffer diff --git a/modules/dasVulkan/daslib/vulkan_runtime.das b/modules/dasVulkan/daslib/vulkan_runtime.das index d6eff9f605..cf78f1679f 100644 --- a/modules/dasVulkan/daslib/vulkan_runtime.das +++ b/modules/dasVulkan/daslib/vulkan_runtime.das @@ -32,9 +32,7 @@ def public array_addr(var a : array) : TT? { return unsafe(addr(a[0])) } -//! Intentional non-owning copy of an ownership wrapper: copy the struct, then -//! clear its `_needs_delete` so only the original frees the handle. Wrapper -//! structs store handles as uint64, so the copy is friction-free. +[arch(at="../ARCHITECTURE.md#ownership")] def public weak_copy(b : auto(TT)) : TT { var c = b static_if (typeinfo has_field<_needs_delete>(c)) { diff --git a/modules/dasVulkan/daslib/vulkan_window.das b/modules/dasVulkan/daslib/vulkan_window.das index c2c4ccd52b..aaf6c686b1 100644 --- a/modules/dasVulkan/daslib/vulkan_window.das +++ b/modules/dasVulkan/daslib/vulkan_window.das @@ -54,6 +54,7 @@ struct public Swapchain { readback_supported : bool //!< true if the images carry TRANSFER_SRC (host readback is possible) } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var sc : Swapchain) { for (i in range(length(sc.framebuffers))) { delete sc.framebuffers[i] @@ -198,6 +199,7 @@ struct public FrameSync { render_finished : Semaphore } +[arch(at="../ARCHITECTURE.md#ownership")] def public finalize(var f : FrameSync) { delete f.image_available delete f.render_finished diff --git a/modules/dasVulkan/generator/REVIEW.md b/modules/dasVulkan/generator/REVIEW.md index cd8da21ceb..bab3468402 100644 --- a/modules/dasVulkan/generator/REVIEW.md +++ b/modules/dasVulkan/generator/REVIEW.md @@ -1,7 +1,7 @@ # dasVulkan Generator Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -doc: `../CLAUDE.md`. +doc: `../ARCHITECTURE.md`. **A diff that grows `skip_report.txt` states in its commit message why losing the newly skipped structs and commands is acceptable.** CI checks only that the committed report matches a fresh diff --git a/modules/dasVulkan/generator/vk_emit_boost.das b/modules/dasVulkan/generator/vk_emit_boost.das index a30fcafa02..979defc7c7 100644 --- a/modules/dasVulkan/generator/vk_emit_boost.das +++ b/modules/dasVulkan/generator/vk_emit_boost.das @@ -154,7 +154,7 @@ def public derive_owners(reg : VkRegistry; em : EmitModel) : array { return <- owners } -[unused_argument(reg, em)] +[unused_argument(reg, em), arch(at="../ARCHITECTURE.md#handles-uint64")] def private emit_handles_boost(reg : VkRegistry; em : EmitModel; out_dir : string; owners : array) { fopen("{out_dir}/vulkan_handles.das", "wb") $(f) { panic("can't write vulkan_handles.das") if (f == null) @@ -162,7 +162,6 @@ def private emit_handles_boost(reg : VkRegistry; em : EmitModel; out_dir : strin fprint(f, "options gen2\noptions _comment_hygiene = true\noptions indenting = 4\n\nmodule vulkan_handles shared\n\n") fprint(f, "require vulkan public\nrequire vulkan/vulkan_runtime public\n\n") for (oi in owners) { - // handles stored as uint64 (their ABI form) so wrapper copies are friction-free fprint(f, "struct {oi.boost} \{\n") fprint(f, " _vk : uint64\n") fprint(f, " _needs_delete : bool\n") diff --git a/modules/dasVulkan/tests/integration/test_compute_features.das b/modules/dasVulkan/tests/integration/test_compute_features.das index 6b3d309125..c77512fa1d 100644 --- a/modules/dasVulkan/tests/integration/test_compute_features.das +++ b/modules/dasVulkan/tests/integration/test_compute_features.das @@ -65,6 +65,7 @@ def cmt_kernel { coopmatStore(acc, cmt_c, 0, 16, 1) } +//! packs to 34 bytes: f16 ``d`` at +0, 32 int8 ``qs`` at +2 struct GBlockQ8 { d : float16 qs : int8[32] @@ -85,6 +86,30 @@ def gq8_kernel { gq8_y[b] = acc * float(gq8_blocks[b].d) } +// the store counterpart to gq8_*: narrowing casts and 8/16-bit stores +var @ssbo @binding = 0 gst_blocks : array +var @ssbo @binding = 1 gst_i16 : array +var @ssbo @binding = 2 gst_u16 : array +var @ssbo @binding = 3 gst_src : array +var @ssbo @binding = 4 gst_i8 : array // one byte per invocation: four adjacent lanes share each dword + +[compute_shader(local_size_x=64, name="gst_spv")] +def gst_kernel { + let gi = gl_GlobalInvocationID.x + let v = gst_src[gi] + gst_i8[gi] = int8(v) + gst_i16[gi] = int16(v) + gst_u16[gi] = uint16(uint(v) * 3u) + if (gi < 16u) { + gst_blocks[gi].d = float16(0.25 * float(1u + gi % 3u)) + var k = 0u + while (k < 32u) { + gst_blocks[gi].qs[int(k)] = int8(int(gi * 32u + k) - 200) + k++ + } + } +} + // ===== a multi-SSBO compute runner ===== // One host-visible coherent STORAGE buffer per binding, uploaded from / read back into `bufs` @@ -371,6 +396,76 @@ def test_storage_8_16_gpu(t : T?) { delete expected } +[test] +def test_storage_8_16_store_gpu(t : T?) { + let sup = probe_support() + if (!sup.storage) { + feint("device lacks 8/16-bit SSBO storage features; skipping\n") + return + } + let n = 64 + var bufs : array> + var blocks : array + blocks |> resize(16 * 34) + bufs |> emplace(blocks) + var i16b : array + i16b |> resize(n * 2) + bufs |> emplace(i16b) + var u16b : array + u16b |> resize(n * 2) + bufs |> emplace(u16b) + var srcb : array + srcb |> resize(n * 4) + for (i in range(n)) { + let v = uint(i * 517 + 3) + srcb[i * 4] = uint8(v & 0xFFu) + srcb[i * 4 + 1] = uint8((v >> 8u) & 0xFFu) + srcb[i * 4 + 2] = uint8((v >> 16u) & 0xFFu) + srcb[i * 4 + 3] = uint8((v >> 24u) & 0xFFu) + } + bufs |> emplace(srcb) + var i8b : array + i8b |> resize(n) + bufs |> emplace(i8b) + var words <- clone_to_move(gst_spv) + let no_spec : array + run_multi(words, bufs, 1u, no_spec, true) + var bad16 = 0 + for (i in range(n)) { + let v = uint(i * 517 + 3) + let want16 = v & 0xFFFFu + let got16 = uint(bufs[1][i * 2]) | (uint(bufs[1][i * 2 + 1]) << 8u) + let wantu = (v * 3u) & 0xFFFFu + let gotu = uint(bufs[2][i * 2]) | (uint(bufs[2][i * 2 + 1]) << 8u) + if (got16 != want16 || gotu != wantu) { + bad16++ + } + } + t |> success(bad16 == 0, "int16/uint16 element stores land exact bytes ({bad16} of {n} wrong)") + var badblk = 0 + for (b in range(16)) { + let dbits = packHalf2x16(float2(0.25 * float(1 + b % 3), 0.0)) & 0xFFFFu + if (uint(bufs[0][b * 34]) != (dbits & 0xFFu) || uint(bufs[0][b * 34 + 1]) != ((dbits >> 8u) & 0xFFu)) { + badblk++ + } + for (k in range(32)) { + if (int(bufs[0][b * 34 + 2 + k]) != ((b * 32 + k - 200) & 0xFF)) { + badblk++ + } + } + } + t |> success(badblk == 0, "int8/f16 struct-member stores land exact bytes ({badblk} wrong)") + var bad8 = 0 + for (i in range(n)) { + if (int(bufs[4][i]) != ((i * 517 + 3) & 0xFF)) { + bad8++ + } + } + t |> success(bad8 == 0, "int8 element stores from adjacent invocations land exact bytes ({bad8} of {n} wrong)") + delete words + delete bufs +} + [test] def test_coopmat_gpu(t : T?) { let sup = probe_support() diff --git a/modules/dasVulkan/tutorials/REVIEW.md b/modules/dasVulkan/tutorials/REVIEW.md index 565f8bb376..3382bef93a 100644 --- a/modules/dasVulkan/tutorials/REVIEW.md +++ b/modules/dasVulkan/tutorials/REVIEW.md @@ -1,7 +1,7 @@ # dasVulkan Tutorials Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -doc: `../CLAUDE.md`. A recording-pipeline change - a `record_*.das` driver, wherever the diff puts +doc: `../ARCHITECTURE.md`. A recording-pipeline change - a `record_*.das` driver, wherever the diff puts it, or the shared `recording/` harness - is reviewed with `skills/internal/vulkan_recording.md` (repo root). diff --git a/modules/dasVulkan/utils/vulkan2rst.das b/modules/dasVulkan/utils/vulkan2rst.das index 7025ab3a08..bb8ab91ec3 100644 --- a/modules/dasVulkan/utils/vulkan2rst.das +++ b/modules/dasVulkan/utils/vulkan2rst.das @@ -68,7 +68,7 @@ def document_module_vulkan_boost() { var groups <- array( group_by_regex("Device selection", mod, %regex~^(select_physical_device|select_graphics_queue_family|find_memory_type)$%%), group_by_regex("Instance & device", mod, %regex~^(create_instance|create_device|create_device_descriptor_indexing|create_device_draw_parameters|get_device_queue|instance_extension_available|device_extension_available)$%%), - group_by_regex("Shaders & pipelines", mod, %regex~^(create_shader_module|create_pipeline_layout|create_graphics_pipeline_simple|create_graphics_pipeline_v3d|create_graphics_pipeline_dyn|create_compute_pipeline)$%%), + group_by_regex("Shaders & pipelines", mod, %regex~^(create_shader_module|create_pipeline_layout|create_graphics_pipeline_simple|create_graphics_pipeline_v3d|create_graphics_pipeline_dyn|create_compute_pipeline|create_compute_pipeline_full_subgroups)$%%), group_by_regex("Mesh shaders", mod, %regex~^(create_device_mesh_shader|mesh_shader_supported|create_mesh_pipeline)$%%), group_by_regex("Hardware ray tracing", mod, %regex~^(rt_supported|create_device_ray_tracing|create_address_buffer|get_ray_tracing_properties|get_ray_tracing_shader_group_handles|create_ray_tracing_pipeline|build_shader_binding_table|cmd_trace_rays|make_accel_instance|build_blas|build_tlas|write_descriptor_acceleration_structure)$%%), group_by_regex("Render passes", mod, %regex~^create_render_pass_(single_color|color_depth|depth_only)$%%), @@ -76,7 +76,7 @@ def document_module_vulkan_boost() { group_by_regex("Memory & brackets", mod, %regex~^(run_cmd_sync|record_render_pass|record_rendering|map_memory_to_array|read_memory|with_mapped_memory|with_push_staging|write_field|upload_bytes)$%%), group_by_regex("Command recording", mod, %regex~^(cmd_bind_pipeline|cmd_bind_vertex_buffer|cmd_draw|cmd_push_constants|copy_image_to_buffer|transition_image|transition_depth_image|transition_image_aspect)$%%), group_by_regex("Conveniences", mod, %regex~^(full_area|clear_color|clear_depth)$%%), - group_by_regex("Compute tier & device capabilities", mod, %regex~^(storage_8_16_supported|subgroup_properties|subgroup_compute_ops_supported|integer_dot_product_supported|cooperative_matrix_supported|cooperative_matrix2_supported|cooperative_matrix2_fa_supported|cooperative_matrix2_properties|cooperative_vector_supported|memory_priority_supported|timeline_semaphore_supported|external_memory_host_supported|external_memory_host_min_alignment|select_transfer_queue_family|create_device_(storage_8_16.*|cooperative_matrix|coopmat_full_subgroups))$%%), + group_by_regex("Compute tier & device capabilities", mod, %regex~^(storage_8_16_supported|compute_full_subgroups_supported|subgroup_properties|subgroup_compute_ops_supported|integer_dot_product_supported|cooperative_matrix_supported|cooperative_matrix2_supported|cooperative_matrix2_fa_supported|cooperative_matrix2_properties|cooperative_vector_supported|memory_priority_supported|timeline_semaphore_supported|external_memory_host_supported|external_memory_host_min_alignment|select_transfer_queue_family|create_device_(storage_8_16.*|cooperative_matrix|coopmat_full_subgroups))$%%), group_by_regex("Pipeline introspection", mod, %regex~^(pipeline_exec_props_supported|dump_pipeline_executables)$%%), hide_group(group_by_regex("Finalizers", mod, %regex~^finalize$%%)) ) diff --git a/skills/internal/make_pr.md b/skills/internal/make_pr.md index 37a2d3b12f..102d68b8b6 100644 --- a/skills/internal/make_pr.md +++ b/skills/internal/make_pr.md @@ -26,7 +26,7 @@ kills the chain's JIT loads, AOT links, and spawned tools mid-suite). |---|---|---| | 0 Sync | make-pr `sync` | Red = behind origin/master: rebase (never onto local `master`), re-run. A listed PR-set file you did not edit = the rebase went wrong; a conflict on a file also changed upstream keeps origin/master's. Squash only AFTER the rebase - `git reset --soft master` on a stale `master` bakes other PRs in; already pushed: rebase + `git push --force-with-lease`. Re-read any `skills/*.md` / `REVIEW*.md` the rebase changed | | 0 Untracked | preflight `untracked` gate | Empty at PR time - commit, delete, or ignore each (`.gitignore`; `.git/info/exclude` for box-local) | -| 0a0 Comment harvest | the diff's ADDED comments, per touched module root | Working comments are welcome while building the PR; this row is where they settle. When the diff adds comments beyond the hygiene skill's kept set, spawn ONE `harvester` per touched module root, scoped to the comments the diff adds (a full-file harvest is the on-first-touch sweep, not a PR gate). YOU rule on its ledger - RENAME first (the strongest resolution), RULE/FACT proposals land in the folder's REVIEW.md / ARCHITECTURE*.md, KEEP one-liners are `//!` contract comments, TODO signals are possibly unfinished PR work, lint candidates surface per CLAUDE.md's lint-opportunities rule. Sits before the audits because landings re-enter the diff | +| 0a0 Comment harvest | the diff's ADDED comments, per touched module root | Working comments are welcome while building the PR; this row is where they settle. When the diff adds comments beyond the hygiene skill's kept set, spawn ONE `harvester` per touched module root, scoped to the comments the diff adds (a full-file harvest is the on-first-touch sweep, not a PR gate). YOU rule on its ledger - RENAME first (the strongest resolution), RULE proposals land in the folder's REVIEW.md, a FACT lands as three things - the section, its `{#anchor}`, and the `[arch]` citation from the function the comment came from (the harvest duty in REVIEW_COMMON.md; the ledger names the citer), KEEP one-liners are `//!` contract comments, TODO signals are possibly unfinished PR work, lint candidates surface per CLAUDE.md's lint-opportunities rule. Sits before the audits because landings re-enter the diff | | 0a REVIEW audit | make-pr `review-md` | Red = a discovered `REVIEW.das` gate failed - fail-fix, no agents until green. Then one `review-md-auditor` per checklist. Discovered rules bind on top of this file; checklist defects fixed in the same batch | | 0a TDD audit | one `tdd-auditor` over the whole diff, REVIEW.md folders or not (`skills/tdd_audit.md`) | UNTESTED branch -> test in the same change, never a follow-up promise. UNPROVEN -> run its named settling gate or state the claim in the PR body. RETUNED/WEAKENED test edit -> restore the expectation/instrument or state the reason | | 0a2 Style hygiene | `style-hygiene-auditor` (`skills/comment_style_hygiene.md`) | Mandatory run, non-blocking findings: fix each or consciously decline it | diff --git a/skills/mcp_tools.md b/skills/mcp_tools.md index b04bdb1246..9e2447385d 100644 --- a/skills/mcp_tools.md +++ b/skills/mcp_tools.md @@ -66,7 +66,7 @@ The daslang MCP server (`utils/mcp/main.das`) exposes compiler diagnostics, prog **`with_cpp_source` redirect.** `find_symbol` and `goto_definition` accept an optional `with_cpp_source` boolean. When `true`, results that have a C++ implementation (builtin functions, handled types via `addExtern`/`MAKE_TYPE_FACTORY`) get a resolved C++ source location appended via the lazily-built cpp index. First call costs ~2s (one full scan); subsequent calls cost ~150ms (a git-state staleness signature: `rev-parse HEAD` + filtered `git status` + per-file mtimes + `cpp_search_config.das` mtime). The index rebuilds automatically when relevant `.cpp/.cc/.h/.hpp` files change, when HEAD moves, or when the search config is edited. Default off - opt in when the question is "where is X *actually* implemented", not when just enumerating symbols. -**`[arch]` tools.** `arch_of` (code -> document) and `arch_sites` (document -> code) resolve citations exactly as LINT026 does: the path in `[arch(at=".md#")]` resolves against the CITING file's folder, an anchor is a `{#name}` heading suffix, a match with `//` earlier on its line is not a citation, and a failure reports lint's own reason (`no such file` / `no such anchor` / `anchor appears N times` / `malformed citation`). Both read source rather than the AST, so they answer for a file this environment cannot compile. `arch_of` returns the cited section - its heading line through the last line before the next heading of the same or a higher level. `arch_sites` searches the subtree of the folder that owns the document, skipping `_`- and `.`-prefixed names the way lint's reverse pass does. +**`[arch]` tools.** `arch_of` (code -> document) and `arch_sites` (document -> code) resolve citations exactly as LINT026 does: the path in `[arch(at=".md#")]` resolves against the CITING file's folder, an anchor is a `{#name}` heading suffix, a match with `//` earlier on its line is not a citation, the document must sit in the citing file's own folder tree (its folder is the file's folder or an ancestor - restate a far mechanism in prose in your own tree's document and cite that), and a failure reports lint's own reason (`no such file` / `no such anchor` / `anchor appears N times` / `malformed citation` / `outside this file's folder tree`). Both read source rather than the AST, so they answer for a file this environment cannot compile. `arch_of` returns the cited section - its heading line through the last line before the next heading of the same or a higher level. `arch_sites` searches the subtree of the folder that owns the document, skipping `_`- and `.`-prefixed names the way lint's reverse pass does. **Live tools.** `live_*` interact with a running `daslang-live` instance via its REST API. `live_launch` starts one if not already running (sets working directory to the script's folder). All live tools accept an optional `port` parameter (default 9090). When a compilation error is active, `live_command` and `live_pause` return HTTP 503 with the error - use `live_reload` to fix. Hitting any unknown endpoint returns JSON help with all endpoints + curl examples. diff --git a/tests/spirv/_spirv_common.das b/tests/spirv/_spirv_common.das index 62b4ea1a45..1639bdad87 100644 --- a/tests/spirv/_spirv_common.das +++ b/tests/spirv/_spirv_common.das @@ -801,7 +801,7 @@ def tern { let i = gl_GlobalInvocationID.x let a = fdata[i] let b = fdata[i + 1u] - let s = a > b ? a : b // scalar OpSelect // nolint:PERF015 + let s = a > b ? a : b // nolint:PERF015 -- the scalar OpSelect fixture: the ternary IS the shape under test let v = a > b ? float4(a, a, a, a) : float4(b, b, b, b) // vector OpSelect (splatted condition) fdata[i] = s + v.x } @@ -2056,6 +2056,48 @@ def public q8blk_words : array { return clone_to_move(q8blk_spv) } +// ===== the 8/16-bit STORE half: narrowing casts (OpSConvert/OpUConvert) and stores through +// 8/16-bit access chains — the requant-writer shapes (a quantizer writing int8 quants and an +// f16 scale directly instead of hand-packing words). Zero NEW opcodes. ===== +var @ssbo @binding = 0 st_blocks : array +var @ssbo @binding = 1 st_i8 : array +var @ssbo @binding = 2 st_i16 : array +var @ssbo @binding = 3 st_u16 : array +var @ssbo @binding = 4 st_src : array + +[compute_shader(local_size_x=64, name="q8store_spv"), marker(no_coverage)] +def q8store_kernel { + let gi = gl_GlobalInvocationID.x + let v = st_src[gi] + st_i8[gi] = int8(v) + st_i16[gi] = int16(v) + st_u16[gi] = uint16(uint(v)) + st_blocks[gi].qs[int(gi & 31u)] = int8(v) + st_blocks[gi].d = float16(0.5) +} + +def public q8store_words : array { + return clone_to_move(q8store_spv) +} + +// ===== unpack8 on 16-bit lanes: an int16/uint16 SSBO element bitcast to byte2/ubyte2 and a runtime +// lane select (the cm2 decode callbacks' 16-bit quant read). Zero NEW opcodes. ===== +var @ssbo @binding = 0 up16_i16 : array +var @ssbo @binding = 1 up16_u16 : array +var @ssbo @binding = 2 up16_i8 : array +var @ssbo @binding = 3 up16_u8 : array + +[compute_shader(local_size_x=64, name="unpack16_spv"), marker(no_coverage)] +def unpack16_kernel { + let gi = gl_GlobalInvocationID.x + up16_i8[gi] = unpack8(up16_i16[gi >> 1u])[int(gi & 1u)] + up16_u8[gi] = unpack8(up16_u16[gi >> 1u])[int(gi & 1u)] +} + +def public unpack16_words : array { + return clone_to_move(unpack16_spv) +} + // ===== cooperative matrix (SPV_KHR_cooperative_matrix): an f16 x f16 -> f32 GEMM tile. Loads a 16x16 A // (row-major) and B (column-major) from f16 SSBOs into subgroup tiles, one MulAdd into a zero-initialized // (OpConstantNull) f32 accumulator, stores the 16x16 result column-major. Exercises @@ -2405,7 +2447,8 @@ def public coopmat2fa_emitter_opcodes : table { for (op in [ SpvOp.CooperativeMatrixReduceNV, SpvOp.CooperativeMatrixPerElementOpNV, - SpvOp.MatrixTimesScalar + SpvOp.MatrixTimesScalar, + SpvOp.VectorExtractDynamic // a runtime lane select on unpack8's byte vector (the cm2 decode callbacks' quant read) ]) { s |> insert(uint(op)) } diff --git a/tests/spirv/test_census.das b/tests/spirv/test_census.das index ccd10eb043..d832372117 100644 --- a/tests/spirv/test_census.das +++ b/tests/spirv/test_census.das @@ -103,6 +103,8 @@ def test_opcode_census(t : T?) { // nolint:STYLE038 — a flat one-add_set-per-f add_set(present, coopmat_sm_words()) add_set(present, coopmat2_words()) add_set(present, coopmat2fa_words()) + add_set(present, q8store_words()) + add_set(present, unpack16_words()) var declared <- coopmat2fa_emitter_opcodes() for (op in keys(present)) { t |> success(key_exists(declared, op), "emitted {op_name(op)} is in the declared set") diff --git a/tests/spirv/test_coopmat2.das b/tests/spirv/test_coopmat2.das index 4aa579d372..e952e9ff9f 100644 --- a/tests/spirv/test_coopmat2.das +++ b/tests/spirv/test_coopmat2.das @@ -66,6 +66,8 @@ def test_coopmat2(t : T?) { "one OpCooperativeMatrixLengthKHR") t |> success(count_op(words, SpvOp.Phi) == 1, "one OpPhi (the coopmatClamp element loop)") + t |> success(count_op_operand_at(words, SpvOp.LoopMerge, 2, uint(SpvLoopControl.Unroll)) == 1, + "the coopmatClamp element loop's OpLoopMerge carries Unroll (left rolled, the accumulator leaves tensor registers)") // the decode function: a second OpFunction with exactly 3 OpFunctionParameters, its // PhysicalStorageBuffer pointer decorated Aliased, and Aligned loads inside t |> success(count_op(words, SpvOp.Function) == 2, diff --git a/tests/spirv/test_storage_8_16.das b/tests/spirv/test_storage_8_16.das index 9748c0988d..cf8d5e04a8 100644 --- a/tests/spirv/test_storage_8_16.das +++ b/tests/spirv/test_storage_8_16.das @@ -39,4 +39,32 @@ def test_storage_8_16(t : T?) { } delete words } + t |> run("small-int STORE half: narrowing casts + 8/16-bit stores; spirv-val clean") <| @(t : T?) { + var words <- q8store_words() + t |> success(count_op(words, SpvOp.SConvert) >= 3, "int8()/int16() narrow via OpSConvert - 3 in q8store_kernel") + t |> success(has_op(words, SpvOp.UConvert), "uint16(uint) narrows via OpUConvert") + t |> success(op_has_operand_at(words, SpvOp.Capability, 0, uint(SpvCapability.StorageBuffer8BitAccess)), "StorageBuffer8BitAccess") + t |> success(op_has_operand_at(words, SpvOp.Capability, 0, uint(SpvCapability.StorageBuffer16BitAccess)), "StorageBuffer16BitAccess") + let r = validate_spirv(words) + if (r.ran) { + t |> success(r.ok, "spirv-val: {r.msg}") + } else { + feint("spirv-val not found locally; skipping (CI enforces)\n") + } + delete words + } + t |> run("unpack8 on 16-bit lanes: int16 -> byte2 and uint16 -> ubyte2 bitcasts with a runtime lane select; spirv-val clean") <| @(t : T?) { + var words <- unpack16_words() + t |> success(count_op(words, SpvOp.Bitcast) >= 2, "the two 16-bit -> 8-bit x2 bitcasts are present") + t |> success(has_op(words, SpvOp.VectorExtractDynamic), "the runtime lane select is a dynamic extract") + t |> success(op_has_operand_at(words, SpvOp.Capability, 0, uint(SpvCapability.Int16)), "Int16") + t |> success(op_has_operand_at(words, SpvOp.Capability, 0, uint(SpvCapability.Int8)), "Int8") + let r = validate_spirv(words) + if (r.ran) { + t |> success(r.ok, "spirv-val: {r.msg}") + } else { + feint("spirv-val not found locally; skipping (CI enforces)\n") + } + delete words + } } diff --git a/utils/REVIEW.md b/utils/REVIEW.md index c2f6157f50..e5c6398e93 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -5,8 +5,10 @@ doc: `CLAUDE.md` (repo root). **A file under `utils/` that belongs to a tool other than the one owning the directory it sits in is reviewed with that tool's own `REVIEW.md`, where one exists, as well as with this -checklist - not with the checklist of the directory it sits in.** A tool's file OUTSIDE -`utils/` answers to the checklist of the folder that contains it. +checklist - not with the checklist of the directory it sits in. A file in a `utils/` library +directory (`common/`) is reviewed with this checklist and with the checklist of every tool +that requires it.** A tool's file OUTSIDE `utils/` answers to the checklist of the folder that +contains it. **A diff that changes the consent wording in `watchdog/watchdog.py` answers to `modules/dasLLAMA/performance/REVIEW.md` (repo root) too.** diff --git a/utils/common/arch_citations.das b/utils/common/arch_citations.das index b6a8a4c074..ed8d644009 100644 --- a/utils/common/arch_citations.das +++ b/utils/common/arch_citations.das @@ -51,6 +51,7 @@ struct public MdAnchor { struct public ArchDocs { counts : table known : table + anchors : table> } @@ -188,16 +189,28 @@ def private arch_doc_exists(var docs : ArchDocs; doc : string) : bool { let ok = st.is_valid && st.is_reg docs.known[doc] = ok if (ok) { - for (a in collect_md_anchors(doc)) { + var found <- collect_md_anchors(doc) + for (a in found) { let key = "{doc}#{a.name}" docs.counts[key] = (docs.counts ?[key] ?? 0) + 1 } + docs.anchors[doc] <- found } } return docs.known[doc] } +// Every anchor `doc` declares, through the same one-parse index the forward check uses; nothing +// when the document is missing. +def public each_arch_anchor(var docs : ArchDocs; doc : string; blk : block<(a : MdAnchor) : void>) { + return if (!arch_doc_exists(docs, doc)) + for (a in docs.anchors[doc]) { + invoke(blk, a) + } +} + + // Why a citation does not resolve, in the words LINT026 reports; "" when it resolves. A citation // names a document section that must be there, exactly once: an anchor declared twice is worse // than a missing one - the citation reads as precise and points at whichever section the reader @@ -210,3 +223,24 @@ def public citation_failure_reason(var docs : ArchDocs; c : ArchCitation) : stri return "anchor appears {n} times" if (n > 1) return "" } + + +// True when the cited document does not sit in the citing file's own folder tree - the document's +// folder must be the file's folder or one of its ancestors. A mechanism another folder's document +// states is restated in prose in a document of the citer's own tree and cited there. +def public citation_outside_subtree(file : string; c : ArchCitation) : bool { + let fdir = arch_path(dir_name(file)) + let ddir = dir_name(c.doc) + return !(fdir == ddir || fdir |> starts_with("{ddir}/")) +} + + +// `citation_failure_reason` plus the scope check: a citation that resolves but reaches outside +// the citing file's folder tree is reported after the resolution reasons, so a broken far +// citation is named for what is broken first. +def public citation_failure_reason(var docs : ArchDocs; file : string; c : ArchCitation) : string { + let reason = citation_failure_reason(docs, c) + return reason if (!empty(reason)) + return "cites a document outside this file's folder tree - restate the mechanism in an architecture doc of this folder's tree and cite that" if (citation_outside_subtree(file, c)) + return "" +} diff --git a/utils/lint/README.md b/utils/lint/README.md index 02a9b7f50a..fbd70e94f1 100644 --- a/utils/lint/README.md +++ b/utils/lint/README.md @@ -14,11 +14,11 @@ Three rules are the runner's own, because they are about folders rather than cod once per invocation, over a walk of the directory roots the run was given. A directory whose `.lint_config` carries `[docs] rule_docs_only = true` may hold only rule documents (`REVIEW*.md`, `ARCHITECTURE*.md`, `LAWS.md`); any other `.md` beside the sources is -**LINT025**. `[docs] enforce_arch = true` adds **LINT026**'s reverse direction: every -`{#anchor}` in the folder's `.md` must be cited by an `[arch]` in its `.das`. Either tag arms -**LINT027**, which caps each `REVIEW*.md` / `ARCHITECTURE*.md` there at 300 lines. The keys -are folder properties - they never cascade, unlike `[format]`. Fixtures: `tests/lint025_*`, -`tests/lint026_*` and `tests/lint027_*`, each driving the CLI over a planted tree. +**LINT025**. **LINT026**'s reverse direction needs no tag: every `{#anchor}` in any `.md` under +the run's roots must be cited by an `[arch]` in a `.das` there. **LINT027** caps each +`REVIEW*.md` / `ARCHITECTURE*.md` at 300 lines in every folder that holds one. The +`rule_docs_only` key is a folder property - it never cascades, unlike `[format]`. Fixtures: +`tests/lint025_*`, `tests/lint026_*` and `tests/lint027_*`, each driving the CLI over a planted tree. Design: the runner stays thin - rules live in the daslib modules (authoring rails: `skills/internal/perf_lint_authoring.md`, `skills/internal/style_lint_authoring.md`); diff --git a/utils/lint/REVIEW.md b/utils/lint/REVIEW.md index 6e9c256414..607e6dd276 100644 --- a/utils/lint/REVIEW.md +++ b/utils/lint/REVIEW.md @@ -3,8 +3,8 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `README.md`. -**A diff that removes `lint` from `DAS_UTILS_SHIPPED_EXES` in `utils/CMakeLists.txt` is a -defect.** +**A diff that removes `lint` from `DAS_UTILS_SHIPPED_EXES` in `utils/CMakeLists.txt` (repo +root) is a defect.** **A diff that shrinks the set of rule ids `REVIEW.das` (beside this file) scans is a defect** - whether by editing the gate or by deleting an id's last scannable spelling. The diff --git a/utils/lint/main.das b/utils/lint/main.das index 1029fae708..ee3e58f8fa 100644 --- a/utils/lint/main.das +++ b/utils/lint/main.das @@ -140,6 +140,12 @@ def is_skip_dir(name : string) : bool { return name |> starts_with(".") } +// A CMake build tree is not source, whatever it is named: FetchContent checkouts under it +// carry .das and .md the tree never wrote +def is_build_tree(path : string) : bool { + return stat("{path}/CMakeCache.txt").is_valid +} + def scan_das_files(path : string; var files : array; var cache : table; var skipped : int&) { let st = stat(path) return if (!st.is_valid) @@ -164,6 +170,7 @@ def scan_das_files(path : string; var files : array; var cache : table ends_with(".das") && !cache |> key_exists(full)) { if (is_skip_file(name)) { @@ -192,51 +199,51 @@ def rule_is_on(code : string; disabled_codes, enabled_codes : table) : b return !key_exists(disabled_codes, code) && (empty(enabled_codes) || key_exists(enabled_codes, code)) } -// Directories under `path` whose OWN .lint_config arms a [docs] flag: rule_docs_only feeds -// LINT025, enforce_arch feeds LINT026's reverse direction, and either one feeds LINT027's -// line gate. Folder property, no cascade - each directory answers for itself. Skip rules -// mirror scan_das_files so all passes see one tree. -def scan_docs_folders(path : string; want_rule_docs, want_arch, want_doc_lines : bool; - var folders, arch_folders, line_folders : array; var seen : table) { +// Directories under `path` the two tagged-or-present folder rules answer for: a folder whose +// OWN .lint_config sets rule_docs_only feeds LINT025, and any folder holding a rule document +// feeds LINT027's line gate. Folder property, no cascade - each directory answers for itself. +// Skip rules mirror scan_das_files so all passes see one tree. +def scan_docs_folders(path : string; want_rule_docs, want_doc_lines : bool; + var folders, line_folders : array; var seen : table) { let st = stat(path) return if (!st.is_valid || !st.is_dir || seen |> key_exists(path)) seen |> insert(path, null) - let cfg = "{path}/.lint_config" - let armed_rule_docs = (want_rule_docs || want_doc_lines) && rule_docs_only_at(cfg) - let armed_arch = (want_arch || want_doc_lines) && enforce_arch_at(cfg) - if (want_rule_docs && armed_rule_docs) { + if (want_rule_docs && rule_docs_only_at("{path}/.lint_config")) { folders |> push(path) } - if (want_arch && armed_arch) { - arch_folders |> push(path) - } - if (want_doc_lines && (armed_rule_docs || armed_arch)) { + if (want_doc_lines && folder_holds_rule_doc(path)) { line_folders |> push(path) } fio::dir(path) $(name) { return if (name == "." || name == ".." || name |> starts_with("_") || is_skip_dir(name)) let full = "{path}/{name}" let fst = stat(full) - if (fst.is_valid && fst.is_dir) { - scan_docs_folders(full, want_rule_docs, want_arch, want_doc_lines, folders, arch_folders, line_folders, seen) + if (fst.is_valid && fst.is_dir && !is_build_tree(full)) { + scan_docs_folders(full, want_rule_docs, want_doc_lines, folders, line_folders, seen) } } } -// One walk of the run's scan roots for all three folder rules - the tree is walked once +// One walk of the run's scan roots for the two folder-keyed rules - the tree is walked once // however many of them are on, and not at all when none is. -def collect_docs_folders(roots : array; want_rule_docs, want_arch, want_doc_lines : bool; - var folders, arch_folders, line_folders : array&) { - return if (!want_rule_docs && !want_arch && !want_doc_lines) +def collect_docs_folders(roots : array; want_rule_docs, want_doc_lines : bool; + var folders, line_folders : array&) { + return if (!want_rule_docs && !want_doc_lines) var seen : table for (root in roots) { - scan_docs_folders(root, want_rule_docs, want_arch, want_doc_lines, folders, arch_folders, line_folders, seen) + scan_docs_folders(root, want_rule_docs, want_doc_lines, folders, line_folders, seen) } folders |> sort - arch_folders |> sort line_folders |> sort } +// A folder holds a rule document when any of its own (non-recursive) .md files is one +def folder_holds_rule_doc(path : string) : bool { + var docs : array + collect_folder_mds(path, true, docs) + return !empty(docs) +} + // The .md files in `folder` (non-recursive) on one side of `is_rule_doc_name`: the strays // LINT025 reports, or the rule documents LINT027 sizes. LAWS.md is append-only ruling // provenance, never read end to end, so it stays out of the rule-document side. @@ -306,22 +313,25 @@ def run_doc_lines_pass(folders : array) : int { // Forward direction: one finding per citation the shared scanner // (`utils/common/arch_citations.das`) cannot resolve, in that scanner's words. def private report_arch_citation(file : string; c : ArchCitation; var docs : ArchDocs) : int { - let reason = citation_failure_reason(docs, c) + let reason = citation_failure_reason(docs, file, c) return 0 if (empty(reason)) print("{file}:{c.line}: LINT026: [arch] citation does not resolve - {reason} (\"{c.raw}\")\n") return 1 } -// Both halves of the reverse check in one walk of an armed folder: the .md files that declare -// anchors and the .das files that may cite them. Same pruning as scan_das_files. -def private scan_arch_folder(path : string; var mds, dases : array) { +// Both halves of the citation check in one walk: the .md files that declare anchors and the +// .das files that may cite them. Same pruning as scan_das_files; `seen` folds nested roots. +def private scan_arch_folder(path : string; var mds, dases : array; var seen : table) { + return if (seen |> key_exists(path)) + seen |> insert(path, null) fio::dir(path) $(name) { return if (name == "." || name == ".." || name |> starts_with("_") || is_skip_dir(name)) let full = "{path}/{name}" let fst = stat(full) return if (!fst.is_valid) if (fst.is_dir) { - scan_arch_folder(full, mds, dases) + return if (is_build_tree(full)) + scan_arch_folder(full, mds, dases, seen) } elif (fst.is_reg && name |> ends_with(".md")) { mds |> push(full) } elif (fst.is_reg && name |> ends_with(".das") && !is_skip_file(name)) { @@ -330,29 +340,37 @@ def private scan_arch_folder(path : string; var mds, dases : array) { } } -// Reverse direction, for a folder whose .lint_config sets [docs] enforce_arch = true: an anchor -// is a promise that some code answers for the section, so an uncited one is either code that -// forgot to say so or a section that was never anyone's contract. -def private run_arch_reverse(folder : string) : int { +// [arch] citation pass, both directions off one read of each file: forward reports every +// citation that does not resolve, reverse every anchor no citation names - an anchor is a +// promise that some code answers for the section, so an uncited one is either code that forgot +// to say so or a section that was never anyone's contract. The roots are read as ONE set (a +// daslib citer satisfies a module's anchor), each .md parsed once through the shared index. +// A folder pass like LINT025 - once per invocation, not once per file. +def run_arch_pass(roots : array) : int { var mds : array var dases : array - scan_arch_folder(folder, mds, dases) + var seen : table + for (root in roots) { + scan_arch_folder(root, mds, dases, seen) + } mds |> sort dases |> sort + var found = 0 + var docs : ArchDocs var cited : table for (f in dases) { let text = fread(f) - continue if (empty(text)) + continue if (empty(text) || find(text, "arch") < 0) for (c in collect_arch_citations(f, text)) { + found += report_arch_citation(f, c, docs) continue if (empty(c.anchor)) cited |> insert("{c.doc}#{c.anchor}") } } - var found = 0 for (md in mds) { let doc = arch_path(md) - for (a in collect_md_anchors(md)) { - continue if (cited |> key_exists("{doc}#{a.name}")) + each_arch_anchor(docs, doc) $(a) { + return if (cited |> key_exists("{doc}#{a.name}")) print("{md}:{a.line}: LINT026: anchor \{#{a.name}\} has no [arch] citation - cite it from the owning code, or strip the anchor to demote the section to narrative\n") found ++ } @@ -360,31 +378,6 @@ def private run_arch_reverse(folder : string) : int { return found } -// [arch] citation pass: forward over every .das under the run's scan roots, reverse over the -// folders that armed it. A folder pass like LINT025 - once per invocation, not once per file. -def run_arch_pass(roots, arch_folders : array) : int { - var files : array - var cache : table - var skipped = 0 - for (root in roots) { - scan_das_files(root, files, cache, skipped) - } - files |> sort - var found = 0 - var docs : ArchDocs - for (f in files) { - let text = fread(f) - continue if (empty(text)) - for (c in collect_arch_citations(f, text)) { - found += report_arch_citation(f, c, docs) - } - } - for (folder in arch_folders) { - found += run_arch_reverse(folder) - } - return found -} - // Validates a rule code: ^(LINT|PERF|STYLE)\d{3}$. Returns true if shape matches. def is_valid_rule_code(code : string) : bool { let n = length(code) @@ -861,16 +854,14 @@ def run_folder_passes(docs_roots : array; disabled_codes, enabled_codes let want_arch = rule_is_on("LINT026", disabled_codes, enabled_codes) let want_doc_lines = rule_is_on("LINT027", disabled_codes, enabled_codes) var rule_docs_folders : array - var arch_folders : array var line_folders : array - collect_docs_folders(docs_roots, want_rule_docs, want_arch, want_doc_lines, - rule_docs_folders, arch_folders, line_folders) + collect_docs_folders(docs_roots, want_rule_docs, want_doc_lines, rule_docs_folders, line_folders) var found = 0 if (want_rule_docs) { found += run_rule_docs_pass(rule_docs_folders) } if (want_arch) { - found += run_arch_pass(docs_roots, arch_folders) + found += run_arch_pass(docs_roots) } if (want_doc_lines) { found += run_doc_lines_pass(line_folders) diff --git a/utils/lint/tests/lint026_arch_citations.das b/utils/lint/tests/lint026_arch_citations.das index afd3abfaf0..a3baa60c6f 100644 --- a/utils/lint/tests/lint026_arch_citations.das +++ b/utils/lint/tests/lint026_arch_citations.das @@ -144,13 +144,12 @@ def private build_forward(name : string) : string { } -// An armed folder whose document declares one cited anchor and one orphan, beside an unarmed -// folder whose anchors nobody owes a citation for. +// A folder whose document declares one cited anchor and one orphan, beside a sibling folder +// whose NOTES.md carries a loose anchor - every anchor is owed a citation, no tag arms it. def private build_reverse(name : string) : string { let root = path_join(tmp_root(), name) let armed = path_join(root, "armed") mkdir_rec(armed) - plant(armed, ".lint_config", "[docs]\nenforce_arch = true\n") plant_doc(armed, "ARCHITECTURE.md", ["cited", "orphan"]) plant_code(armed, "ok.das", [Cite(doc = "ARCHITECTURE.md", anchor = "cited", fn = "one"), Cite(doc = "ARCHITECTURE.md", anchor = "cited", fn = "two")]) @@ -202,18 +201,19 @@ def test_arch_forward_control(t : T?) { plant_code(dir, "ok.das", [Cite(doc = "ARCHITECTURE.md", anchor = "nope", fn = "only")]) var out : string run_lint(dir, out) - t |> equal(count_of(out, "LINT026"), 1, "the dangling anchor fires\n{out}") + t |> equal(count_of(out, "does not resolve"), 1, "the dangling citation fires\n{out}") + t |> equal(count_of(out, "has no [arch] citation"), 1, "and the anchor it missed is uncited\n{out}") // mutate: the same citation, pointed at a section the document declares plant_code(dir, "ok.das", [Cite(doc = "ARCHITECTURE.md", anchor = "alpha", fn = "only")]) var fixed : string let rc = run_lint(dir, fixed) - t |> equal(count_of(fixed, "LINT026"), 0, "a resolving citation is silent\n{fixed}") + t |> equal(count_of(fixed, "LINT026"), 0, "a resolving citation is silent both ways\n{fixed}") t |> equal(rc, 0, "and the run is clean\n{fixed}") // restore plant_code(dir, "ok.das", [Cite(doc = "ARCHITECTURE.md", anchor = "nope", fn = "only")]) var again : string run_lint(dir, again) - t |> equal(count_of(again, "LINT026"), 1, "restoring the citation brings the finding back\n{again}") + t |> equal(count_of(again, "LINT026"), 2, "restoring the citation brings both findings back\n{again}") } t |> run("a commented-out citation is not a citation") @(t : T?) { let dir = path_join(tmp_root(), "commented") @@ -226,8 +226,9 @@ def test_arch_forward_control(t : T?) { }) var out : string let rc = run_lint(dir, out) - t |> equal(count_of(out, "LINT026"), 0, "a `//` line is prose, not a citation\n{out}") - t |> equal(rc, 0, "and the run is clean\n{out}") + t |> equal(count_of(out, "does not resolve"), 0, "a `//` line is prose, not a citation\n{out}") + t |> equal(count_of(out, "has no [arch] citation"), 1, "so the document's anchor has no citer\n{out}") + t |> equal(rc, 2, "the uncited anchor sets the issue exit code\n{out}") } t |> run("a structure's citation is shape-checked by lint - the compiler never sees it") @(t : T?) { let dir = path_join(tmp_root(), "malformed") @@ -242,24 +243,83 @@ def test_arch_forward_control(t : T?) { }) var out : string let rc = run_lint(dir, out) - t |> equal(count_of(out, "LINT026"), 1, "the anchor-less citation fires\n{out}") + t |> equal(count_of(out, "does not resolve"), 1, "the anchor-less citation fires\n{out}") t |> success(find(out, "ok.das:3: LINT026: [arch] citation does not resolve - malformed citation") >= 0, "reported at the annotation, as malformed\n{out}") t |> equal(rc, 2, "a finding, not a compile error\n{out}") } + t |> run("a citation stays in the document's folder tree - an ancestor's document resolves, a sibling's does not") @(t : T?) { + let root = path_join(tmp_root(), "scope") + let a = path_join(root, "a") + let b = path_join(root, "b") + mkdir_rec(a) + mkdir_rec(b) + plant_doc(root, "ARCHITECTURE.md", ["alpha"]) + plant_doc(a, "ARCHITECTURE.md", ["beta"]) + plant_code(b, "ok.das", [Cite(doc = "../ARCHITECTURE.md", anchor = "alpha", fn = "near"), + Cite(doc = "../a/ARCHITECTURE.md", anchor = "beta", fn = "far")]) + var out : string + let rc = run_lint(root, out) + t |> equal(count_of(out, "outside this file's folder tree"), 1, "the sibling-folder citation is the finding\n{out}") + t |> equal(count_of(out, "LINT026"), 1, "the ancestor citation resolves, and the far citation still counts as beta's citer\n{out}") + t |> equal(rc, 2, "a finding sets the issue exit code\n{out}") + } +} + + +[test] +def test_arch_roots(t : T?) { + t |> run("a CMake build tree under a root is not source - its documents and files are skipped") @(t : T?) { + let root = path_join(tmp_root(), "buildtree") + let gen = path_join(root, "gen") + mkdir_rec(gen) + plant(gen, "CMakeCache.txt", "CMAKE_HOME_DIRECTORY:INTERNAL=x\n") + plant_doc(gen, "vendored.md", ["build_include"]) + plant_code(gen, "ok.das", [Cite(doc = "missing.md", anchor = "nope", fn = "only")]) + plant_code(root, "ok.das", []) + var out : string + let rc = run_lint(root, out) + t |> equal(count_of(out, "LINT026"), 0, "nothing under the build tree is reported\n{out}") + t |> equal(rc, 0, "and the run is clean\n{out}") + } + t |> run("a .md positional arms its folder with no .das compiled - the changed-document PR shape") @(t : T?) { + let root = path_join(tmp_root(), "mdpos") + mkdir_rec(root) + plant_doc(root, "ARCHITECTURE.md", ["alpha", "orphan"]) + plant_code(root, "ok.das", [Cite(doc = "ARCHITECTURE.md", anchor = "alpha", fn = "only")]) + var out : string + let rc = run_lint(path_join(root, "ARCHITECTURE.md"), out) + t |> success(find(out, "0 files") >= 0, "no .das is compiled for a document positional\n{out}") + t |> equal(count_of(out, "has no [arch] citation"), 1, "the folder's orphan anchor is found through the document\n{out}") + t |> equal(rc, 2, "a finding sets the issue exit code\n{out}") + } + t |> run("nested roots fold into one walk - a subfolder's citer satisfies the parent's anchor, and nothing reports twice") @(t : T?) { + let root = path_join(tmp_root(), "nested") + let sub = path_join(root, "sub") + mkdir_rec(sub) + plant_doc(root, "ARCHITECTURE.md", ["alpha", "orphan"]) + plant_code(sub, "ok.das", [Cite(doc = "../ARCHITECTURE.md", anchor = "alpha", fn = "only")]) + plant_doc(sub, "NOTES.md", ["loose"]) + var out : string + let rc = run_lint_argv([sub], root, out) + t |> equal(count_of(out, "\{#orphan\}"), 1, "the parent's orphan is found once\n{out}") + t |> equal(count_of(out, "\{#loose\}"), 1, "the subfolder given twice reports its loose anchor once\n{out}") + t |> equal(count_of(out, "\{#alpha\}"), 0, "the subfolder's citer satisfies the parent's anchor\n{out}") + t |> equal(rc, 2, "the two findings set the issue exit code\n{out}") + } } [test] def test_arch_reverse(t : T?) { - t |> run("an uncited anchor fires, and only in an armed folder") @(t : T?) { + t |> run("an uncited anchor fires in every folder - no tag arms the reverse pass") @(t : T?) { let root = build_reverse("rev") var out : string let rc = run_lint(root, out) - t |> equal(count_of(out, "LINT026"), 1, "only the orphan is a finding\n{out}") + t |> equal(count_of(out, "LINT026"), 2, "the orphan and the sibling's loose anchor are the findings\n{out}") t |> success(find(out, "armed/ARCHITECTURE.md:6: LINT026: anchor \{#orphan\} has no [arch] citation") >= 0, "the finding names the anchor at its heading line\n{out}") - t |> success(find(out, "NOTES.md") < 0, "an unarmed folder's anchors stay narrative\n{out}") + t |> success(find(out, "NOTES.md") >= 0, "a sibling folder's loose anchor is owed too\n{out}") t |> success(find(out, "\{#cited\}") < 0, "two citations satisfy one anchor\n{out}") t |> equal(rc, 2, "a finding sets the issue exit code\n{out}") } @@ -268,7 +328,7 @@ def test_arch_reverse(t : T?) { let armed = path_join(root, "armed") var before : string run_lint(root, before) - t |> equal(count_of(before, "LINT026"), 1, "only the orphan\n{before}") + t |> equal(count_of(before, "LINT026"), 2, "the orphan and the loose anchor\n{before}") // mutate: the code stops citing `cited`, so both anchors are owed plant_code(armed, "ok.das", [Cite(doc = "ARCHITECTURE.md", anchor = "orphan", fn = "one")]) var swapped : string @@ -298,7 +358,7 @@ def test_arch_modes(t : T?) { let root = build_reverse("revdis") var out : string let rc = run_lint_argv(["--disable", "LINT026"], root, out) - t |> equal(count_of(out, "LINT026"), 0, "the armed folder reports nothing\n{out}") + t |> equal(count_of(out, "LINT026"), 0, "the reverse pass reports nothing\n{out}") t |> equal(rc, 0, "and the run is clean\n{out}") } t |> run("--enable whitelist mode without LINT026 silences it") @(t : T?) { diff --git a/utils/lint/tests/lint027_rule_doc_lines.das b/utils/lint/tests/lint027_rule_doc_lines.das index 0d1198642b..0b7c6611f4 100644 --- a/utils/lint/tests/lint027_rule_doc_lines.das +++ b/utils/lint/tests/lint027_rule_doc_lines.das @@ -10,9 +10,9 @@ require daslib/fio // LINT027 is a FOLDER rule the runner enforces with its own tree walk, not a compile pass, so // like LINT025 it is driven through the CLI over a planted tree rather than an `expect` header. // -// The tree carries four folders: `tagged` (rule_docs_only = true), `arch` (enforce_arch = true, -// the other tag that arms the gate), `off` (both tags false) and `plain` (no config). Two -// findings may come back - one per oversized doc in an armed folder. +// The tree carries four folders: `tagged` (rule_docs_only = true), `arch` (no config, a routed +// checklist), `off` (rule_docs_only = false) and `plain` (no config). Four findings come back - +// one per oversized rule document, in whatever folder holds it: no tag arms the gate. var private _tmp_dir : string var private _tmp_dir_inited = false @@ -82,9 +82,8 @@ def private build_tree() : string { // 301 fires, 300 is at the cap, and LAWS.md is provenance rather than a checklist plant_folder(root, "tagged", "[docs]\nrule_docs_only = true\n", ["ARCHITECTURE.md", "REVIEW.md", "LAWS.md"], [301, 300, 400]) - plant_folder(root, "arch", "[docs]\nenforce_arch = true\n", ["REVIEW_GPU.md"], [301]) - plant_folder(root, "off", "[docs]\nrule_docs_only = false\nenforce_arch = false\n", - ["ARCHITECTURE.md"], [400]) + plant_folder(root, "arch", "", ["REVIEW_GPU.md"], [301]) + plant_folder(root, "off", "[docs]\nrule_docs_only = false\n", ["ARCHITECTURE.md"], [400]) plant_folder(root, "plain", "", ["ARCHITECTURE.md"], [400]) return root } @@ -130,22 +129,22 @@ def private count_of(text, sub : string) : int { [test] def test_rule_doc_line_gate(t : T?) { - t |> run("an oversized rule document fires, once per armed folder") @(t : T?) { + t |> run("an oversized rule document fires, once per document, in any folder") @(t : T?) { let root = build_tree() var out : string let rc = run_lint(root, out) - t |> equal(count_of(out, "LINT027"), 2, "one per oversized doc in an armed folder\n{out}") + t |> equal(count_of(out, "LINT027"), 4, "one per oversized rule document, in any folder\n{out}") t |> success(find(out, "tagged/ARCHITECTURE.md: LINT027: rule document exceeds 300 lines (301)") >= 0, "the finding names the document and its size\n{out}") t |> success(find(out, "arch/REVIEW_GPU.md: LINT027") >= 0, - "enforce_arch arms the gate too\n{out}") + "a folder holding a rule document is gated with no tag\n{out}") t |> equal(rc, 2, "a finding sets the issue exit code\n{out}") } t |> run("--enable LINT027 alone still arms the folder walk") @(t : T?) { let root = build_tree() var out : string let rc = run_lint_argv(["--enable", "LINT027"], root, out) - t |> equal(count_of(out, "LINT027"), 2, "the line gate runs with LINT025/026 whitelisted off\n{out}") + t |> equal(count_of(out, "LINT027"), 4, "the line gate runs with LINT025/026 whitelisted off\n{out}") t |> equal(rc, 2, "a finding sets the issue exit code\n{out}") } t |> run("the cap itself is silent, and so is LAWS.md at any length") @(t : T?) { @@ -155,21 +154,21 @@ def test_rule_doc_line_gate(t : T?) { t |> success(find(out, "REVIEW.md: LINT027") < 0, "300 lines exactly is at the cap\n{out}") t |> success(find(out, "LAWS.md: LINT027") < 0, "the rulings sidecar carries no line gate\n{out}") } - t |> run("an untagged folder is silent however long its documents are") @(t : T?) { + t |> run("an untagged folder is gated like a tagged one") @(t : T?) { let root = build_tree() var out : string run_lint(root, out) - t |> success(find(out, "off/ARCHITECTURE.md") < 0, "both tags false reports nothing\n{out}") - t |> success(find(out, "plain/ARCHITECTURE.md") < 0, "no .lint_config reports nothing\n{out}") + t |> success(find(out, "off/ARCHITECTURE.md") >= 0, "rule_docs_only = false does not disarm the line gate\n{out}") + t |> success(find(out, "plain/ARCHITECTURE.md") >= 0, "no .lint_config does not disarm it either\n{out}") } - t |> run("the tag is a folder property - a subfolder does not inherit it") @(t : T?) { + t |> run("a nested folder holding a rule document is gated on its own") @(t : T?) { let root = build_tree() let nested = path_join(path_join(root, "tagged"), "nested") mkdir_rec(nested) plant(nested, "ARCHITECTURE.md", md_of_lines(400)) var out : string run_lint(root, out) - t |> success(find(out, "nested/ARCHITECTURE.md") < 0, "no cascade into an untagged subfolder\n{out}") + t |> success(find(out, "nested/ARCHITECTURE.md") >= 0, "the subfolder's own rule document is gated\n{out}") } t |> run("--disable LINT027 silences the pass") @(t : T?) { let root = build_tree() diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 50f6bf0c8f..4e67bfac41 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -7,3 +7,8 @@ interpreted through `.mcp.json` instead.** Development runs the server through the python keep-alive supervisor, so an exe form would never be used in development before it ships. + +**A diff that adds a top-level `.das` under `utils/mcp/` that `main.das` reaches also adds it to +the `install(FILES ...)` block in `CMakeLists.txt` (repo root), in the same change.** `tools/` +and `subtools/` are globbed; a top-level file left out of the list dies in the shipped SDK on +`error[20605] missing prerequisite` while the in-tree server keeps working. diff --git a/utils/mcp/tools/arch_of.das b/utils/mcp/tools/arch_of.das index ff36b383be..d20720368d 100644 --- a/utils/mcp/tools/arch_of.das +++ b/utils/mcp/tools/arch_of.das @@ -22,12 +22,12 @@ def private write_section(var w : StringBuilderWriter; body : string) { } def private write_citation(var w : StringBuilderWriter; var docs : ArchDocs; - rel : string; c : ArchCitation; symbol : string) { + file, rel : string; c : ArchCitation; symbol : string) { w |> write("{rel}:{c.line}: arch(at=\"{c.raw}\")\n") if (!empty(symbol)) { w |> write(" symbol: {symbol}\n") } - let reason = citation_failure_reason(docs, c) + let reason = citation_failure_reason(docs, file, c) if (!empty(reason)) { w |> write(" does not resolve - {reason}\n") if (!empty(c.doc)) { @@ -59,7 +59,7 @@ def do_arch_of(file : string; symbol : string = "") : string { let owner = annotated_symbol(lines, c.line) continue if (!empty(symbol) && owner != symbol) shown ++ - write_citation(w, docs, rel, c, owner) + write_citation(w, docs, arch_path(resolved), rel, c, owner) } } if (shown == 0) {