Refactor inference layer

AGENTS.md

@@ -8,18 +8,20 @@ Orb is an agentic roleplay and writing application with a Python/FastAPI backend
 
 ## Backend layout
 
-The backend is split into layers. Dependencies point downward only:
+The backend is split into layers with explicit allowed dependency edges:
 
 1. `api/` — HTTP routes and schemas
 2. `pipeline/` — conversation turn orchestration
-3. `features/` — self-contained application features
-4. `workflows/` — pluggable secondary workflows
-5. `inference/` and `analysis/` — model access, prompt assembly, and pure text analysis
+3. `features/` and `workflows/` — sibling product composition inputs
+4. `prompting/` — deterministic, provider-independent model-facing construction
+5. `inference/` and `analysis/` — model execution and pure text analysis
 6. `database/` — schema, migrations, queries, and row models
 7. `core/` — dependency-free shared utilities and types
 
 Lower layers must not import higher layers or peer feature slices. Use dependency inversion when a lower layer needs higher-layer behavior. Keep pure logic separate from integration code and persistence.
 
+See `docs/architecture/prompting.md` for the exact allowed-edge matrix, the prompting ownership test, and prompt/tool ordering contracts. `scripts/check_backend_layers.py` is the executable source of truth for dependency edges.
+
 Feature slices should expose a small facade, keep local contracts near their logic, and persist through the database layer rather than reaching into unrelated features.
 
 Before changing prompt assembly, pass ordering, tool schemas, or streaming behavior, read the relevant document in `docs/architecture/`.
@@ -42,6 +44,13 @@ Before changing prompt assembly, pass ordering, tool schemas, or streaming behav
 - Use registered actions and `data-*` attributes for UI events. Do not add globals or inline event handlers.
 - Keep frontend layer checks passing.
 
+Backend workflow plug-ins under `backend/workflows/<id>/` follow the same rule:
+import only their own package and `backend.workflows.toolkit`. Root modules
+directly under `backend/workflows/` are host adapters and may bridge to lower
+application layers. Import named toolkit exports explicitly; wildcard,
+module-object, and non-`__all__` toolkit imports are not part of the plug-in API.
+The backend layer checker enforces this distinction.
+
 ## Validation
 
 Use the repository scripts from the project root:

backend/api/routes/conversations.py

@@ -76,16 +76,20 @@
     AbortToken,
     agent_lane_from_settings,
     client_from_settings,
-    group_context,
-    macro_identity,
-    prompt_builder,
 )
 from ...pipeline import (
     agent_enabled,
     conversation_macro_seed,
     persona_macros,
     resolve_card_and_persona,
 )
+from ...prompting import (
+    compute_style_injection_block,
+    group_context,
+    group_speaker_label,
+    macro_identity,
+    resolve_mood_fragment_randoms,
+)
 from ..deps import (
     _active_aborts,
     _CleanupStreamingResponse,
@@ -729,16 +733,16 @@ async def api_get_context_size(cid: str, conv: ConversationRow = Depends(require
         for message in messages:
             if message.get("role") != "assistant":
                 continue
-            label = prompt_builder.group_speaker_label(names, message.get("speaker_member_id"))
+            label = group_speaker_label(names, message.get("speaker_member_id"))
             msg_chars += len(f"{label}: ")
 
     # Director injection — fragment {{random}} resolves against a throwaway
     # copy of the stored choice map so the estimate matches the prompt bytes a
     # real turn would inject, without recording new picks.
     active_moods = director.get("active_moods", []) if director else []
     est_choices = dict(director.get("macro_choices", {}) if director else {})
-    est_mood_frags = prompt_builder.resolve_mood_fragment_randoms(mood_frags, active_moods, est_choices)
-    inj_block = prompt_builder.compute_style_injection_block(
+    est_mood_frags = resolve_mood_fragment_randoms(mood_frags, active_moods, est_choices)
+    inj_block = compute_style_injection_block(
         active_moods,
         active_moods,
         est_mood_frags,

backend/api/routes/messages.py

@@ -32,6 +32,7 @@
     update_message_content,
 )
 from ...database.models import ConversationRow
+from ...features import autocomplete
 from ...features.prose_rewriter import (
     ProseRewriteConfig,
     resolve_config,
@@ -496,7 +497,7 @@ async def api_autocomplete(
         for m in messages[-4:]
     ]
     summary = macros.resolve_prompt(summary_source)
-    prompt = local_ml.build_prompt(char_name, user_name, summary, recent, macros.resolve_prompt(data.draft))
+    prompt = autocomplete.build_prompt(char_name, user_name, summary, recent, macros.resolve_prompt(data.draft))
 
-    completion = await local_ml.complete(prompt)
+    completion = await autocomplete.complete(prompt)
     return {"completion": completion}

backend/api/routes/settings.py

@@ -5,7 +5,7 @@
 from fastapi import APIRouter, HTTPException
 
 from ...database import get_settings, reset_to_defaults, update_settings
-from ...inference import TOOLS
+from ...prompting.tool_catalog import has_tool
 from ..schemas import ResetConfirm, SettingsUpdate
 
 router = APIRouter()
@@ -22,7 +22,7 @@ async def api_update_settings(data: SettingsUpdate):
     # enabled_tools holds only model-callable tools. Drop any key that is not a
     # registered tool so non-tool feature flags can never be persisted into it.
     if isinstance(payload.get("enabled_tools"), dict):
-        payload["enabled_tools"] = {k: v for k, v in payload["enabled_tools"].items() if k in TOOLS}
+        payload["enabled_tools"] = {k: v for k, v in payload["enabled_tools"].items() if has_tool(k)}
     return await update_settings(payload)
 
 

backend/core/__init__.py

@@ -32,6 +32,7 @@
     split_sentences,
 )
 from .utils import (
+    agent_lane_max_tokens,
     build_multimodal_content,
     estimate_tokens,
     extract_hyperparams,
@@ -71,6 +72,7 @@
     "remove_quoted_spans",
     "split_sentences",
     # utils — token/log/multimodal helpers
+    "agent_lane_max_tokens",
     "build_multimodal_content",
     "estimate_tokens",
     "extract_hyperparams",

backend/core/domain_types.py

@@ -10,7 +10,7 @@
 
 # Which character information every group generation carries. Stored on
 # ``conversations.group_context_mode``; the projection each value selects lives
-# in ``inference/group_context.py`` and nowhere else.
+# in ``prompting/group_context.py`` and nowhere else.
 GroupContextMode: TypeAlias = Literal["private", "shared", "swap"]
 
 

backend/core/utils.py

@@ -5,6 +5,7 @@
 from collections.abc import Mapping, Sequence
 from typing import Any
 
+from .domain_types import AgentLane
 from .llm_types import ContentPart
 
 #: Heuristic characters-per-token ratio used for rough context-size estimates.
@@ -31,27 +32,75 @@ def scrub_log(value: object) -> str:
     return str(value).replace("\r", "").replace("\n", "")
 
 
-def extract_hyperparams(settings: Mapping[str, Any], *, defaults: Mapping[str, Any] | None = None) -> dict:
-    """Extract LLM hyperparameters from a settings dict.
+#: The sampler/budget fields a settings row carries for a lane, in the order the
+#: endpoint editor shows them.
+_HYPERPARAM_KEYS = (
+    "temperature",
+    "max_tokens",
+    "top_p",
+    "min_p",
+    "top_k",
+    "repetition_penalty",
+)
 
-    Optionally fills in *defaults* for any keys not present in settings.
+
+def extract_hyperparams(
+    settings: Mapping[str, Any],
+    *,
+    lane: AgentLane = "writer",
+    token_floor: int | None = None,
+    defaults: Mapping[str, Any] | None = None,
+) -> dict:
+    """Extract LLM hyperparameters from a settings dict for the lane making the call.
+
+    The agent lane reads each key's ``agent_`` twin. ``get_settings`` overlays those
+    from the agent endpoint's own model config, and only when a separate lane
+    resolves, so single-model mode falls through to the writer's values -- which is
+    the same endpoint it is calling. Passing the writer's lane to an agent call is
+    not a harmless default: it sends one endpoint's preset to another. The fallback
+    is per key rather than whole-row only as a guard for partial mappings; the six
+    columns behind these keys are all NOT NULL, so a resolved agent lane carries
+    every twin and an unresolved one carries none.
+
+    ``token_floor`` is what the call needs to answer in full. The configured budget
+    may only *raise* it: a budget is a reply-length preference, and a short-reply
+    preset is a normal setting, while the floor is what the call needs to answer at
+    all -- so honoring a smaller one would truncate the reply mid-answer and turn a
+    sampling preference into a silent failure. Every call whose whole answer has to
+    fit in one reply (a forced tool call, a constrained-decoding call) passes one;
+    passes that stream prose take the setting as-is and leave it unset.
+
+    Optionally fills in *defaults* for any keys not present in settings. Note that
+    both ``settings`` and ``model_configs`` declare all six columns NOT NULL, so
+    *defaults* only ever fires for a partial mapping, never for a real row -- it is
+    not a way to spell a minimum, which is what ``token_floor`` is for.
     """
-    keys = [
-        "temperature",
-        "max_tokens",
-        "top_p",
-        "min_p",
-        "top_k",
-        "repetition_penalty",
-    ]
-    params = {k: v for k in keys if (v := settings.get(k)) is not None}
+    prefix = "agent_" if lane == "agent" else ""
+    params: dict[str, Any] = {}
+    for key in _HYPERPARAM_KEYS:
+        value = settings.get(f"{prefix}{key}") if prefix else None
+        if value is None:
+            value = settings.get(key)
+        if value is not None:
+            params[key] = value
     if defaults:
         for k, v in defaults.items():
             if k not in params:
                 params[k] = v
+    if token_floor is not None:
+        params["max_tokens"] = max(token_floor, int(params.get("max_tokens") or 0))
     return params
 
 
+def agent_lane_max_tokens(settings: Mapping[str, Any], *, floor: int) -> int:
+    """The agent lane's reply budget for a call that needs at least *floor* tokens.
+
+    The lane cascade and the floor rule are ``extract_hyperparams``'; this is the
+    spelling for a caller that sets its own samplers and wants only the budget.
+    """
+    return int(extract_hyperparams(settings, lane="agent", token_floor=floor)["max_tokens"])
+
+
 def build_multimodal_content(text: str, attachments: Sequence[Mapping[str, Any]] | None = None) -> str | list[ContentPart]:
     """Wrap *text* (and optional image attachments) into a multimodal content list.
 

backend/database/queries/worlds.py

@@ -270,7 +270,7 @@ async def get_lorebook_entries(world_id: str) -> list[LorebookEntryRow]:
     """Every row in the World, both layers, archived included.
 
     The drawer labels the layers and shows archived overlay rows, and the
-    projection (``inference.lorebook.select_effective_entries``) filters from
+    projection (``prompting.lorebook.select_effective_entries``) filters from
     this same superset -- so this reader stays deliberately unfiltered.
     """
     async with get_db() as db:
@@ -395,7 +395,7 @@ async def get_active_lorebook_entries() -> list[ActiveLorebookEntryRow]:
 
     This is the raw overlay pool, not the effective lore: an authored entry
     hidden by a replacement is still in here, and so is the suppression marker
-    that hides it. Resolving that is ``inference.lorebook`` -- the projection
+    that hides it. Resolving that is ``prompting.lorebook`` -- the projection
     lives in the lore layer, which ``database/`` sits below and may not import.
     """
     async with get_db() as db:

backend/features/autocomplete/__init__.py

@@ -0,0 +1,5 @@
+"""Autocomplete prompt construction and completion adaptation."""
+
+from .service import build_prompt, complete
+
+__all__ = ["build_prompt", "complete"]

backend/features/autocomplete/service.py

@@ -0,0 +1,55 @@
+"""Feature-specific prompt and output handling for local autocomplete."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+
+from ...inference import local_ml
+
+
+async def complete(
+    prompt: str,
+    n_predict: int = 12,
+    stop: Sequence[str] = ("\n",),
+    temperature: float = 0.25,
+) -> str:
+    """Autocomplete continuation over the generic local-model runtime.
+
+    The typeahead model produces garbage for a whitespace-ending prompt, so
+    trim that tail before inference. If whitespace was removed, also remove the
+    model's re-emitted leading separator because the frontend appends the result
+    to the original, untrimmed draft.
+    """
+    trimmed = prompt.rstrip()
+    completion = await local_ml.acomplete("autocomplete", trimmed, n_predict, stop, temperature)
+    return completion.lstrip() if trimmed != prompt else completion
+
+
+def build_prompt(
+    char_name: str,
+    user_name: str,
+    char_summary: str,
+    recent: Sequence[Mapping[str, str]],
+    draft: str,
+    *,
+    max_msg_chars: int = 500,
+    max_summary_chars: int = 400,
+) -> str:
+    """Assemble a short raw-continuation prompt ending at the user's draft.
+
+    *recent* is oldest-to-newest and may carry a ``name`` that labels a group
+    speaker instead of using *char_name*. The lightweight prompt deliberately
+    excludes Director and pipeline injection content.
+    """
+    lines: list[str] = []
+    summary = (char_summary or "").strip()
+    if summary:
+        lines.append(summary[:max_summary_chars])
+        lines.append("***Roleplay chat below***")
+    for message in recent:
+        name = (message.get("name") or "").strip() or (user_name if message.get("role") == "user" else char_name)
+        content = (message.get("content") or "").strip()[-max_msg_chars:]
+        if content:
+            lines.append(f"{name}: {content}")
+    lines.append(f"{user_name}: {draft}")
+    return "\n".join(lines)

backend/features/cards/public_profile.py

@@ -15,7 +15,7 @@
 # output contract below has something to be the enforcement of.
 #
 # Braces are on the list because a profile is macro-resolved at turn time
-# (``inference/group_context._render_public_cast``): a generated ``{{user}}``
+# (``prompting/group_context._render_public_cast``): a generated ``{{user}}``
 # would quietly substitute months later, in a string the user already reviewed
 # and approved.
 PROFILE_FLOOR = (
@@ -43,10 +43,8 @@
     "a character's card fields or display name. Call the requested tool."
 )
 
-# Deliberately not registered in ``inference.tool_registry.TOOLS``: that module
-# asserts ``PRE_WRITER_TOOLS | POST_WRITER_TOOLS == BUILTIN_TOOL_NAMES`` at
-# import, so registering here would force a turn-phase partition onto a tool that
-# has nothing to do with a turn.
+# Deliberately not registered in ``prompting.tool_catalog``: this card-only
+# schema is a one-shot contract, not part of the stable pipeline tool blob.
 DRAFT_PROFILE_TOOL = {
     "type": "function",
     "function": {

backend/features/cards/sheet_update.py

@@ -27,10 +27,9 @@
     "character's sheet or display name. Call the requested tool, reporting no change when there is none."
 )
 
-# Deliberately not registered in ``inference.tool_registry.TOOLS``, for the same
-# reason ``DRAFT_PROFILE_TOOL`` is not: that module partitions its tools by turn
-# phase, and this call is bookkeeping about a finished exchange rather than a phase
-# of one.
+# Deliberately not registered in ``prompting.tool_catalog``, for the same reason
+# ``DRAFT_PROFILE_TOOL`` is not: this call is bookkeeping about a finished
+# exchange, not part of the stable pipeline tool blob.
 UPDATE_SHEET_TOOL = {
     "type": "function",
     "function": {

backend/features/documents/audit.py

@@ -23,7 +23,8 @@
     sentence_boundary_ends,
 )
 from ...core import ChatMessage, extract_hyperparams
-from ...inference import TOOLS, LLMClient, parse_tool_calls, reasoning_cfg
+from ...inference import LLMClient, parse_tool_calls, reasoning_cfg
+from ...prompting.tool_catalog import require_tool
 from .continuation import _MACRO_RE, build_generation_messages
 
 if TYPE_CHECKING:
@@ -265,8 +266,13 @@ async def patch_document(
     # generation prompt is what keeps the KV prefix warm); only the scanners
     # see the cleaned/capped ctx above.
     report_text = format_numbered_report(targets)
-    params = extract_hyperparams(settings, defaults={"temperature": 0.25, "max_tokens": 8192})
-    schema = TOOLS["editor_apply_patch"]["schema"]
+    # Writer lane on purpose (the route serves this call from the writer endpoint,
+    # for byte parity with the prompt that generated the draft), but floored like
+    # the agent-lane forced calls: the whole patch set has to fit in one reply, and
+    # a document preset kept short for brief continuations would truncate it.
+    params = extract_hyperparams(settings, token_floor=8192, defaults={"temperature": 0.25})
+    editor_patch = require_tool("editor_apply_patch")
+    schema = editor_patch["schema"]
     if client.completion_mode == "text":
         # Both text shapes byte-extend the generation prompt as a raw
         # continuation: verbatim document (raw) or the re-run /apply-template
@@ -291,7 +297,7 @@ async def patch_document(
             messages,
             model,
             tools=[schema],
-            tool_choice=TOOLS["editor_apply_patch"]["choice"],
+            tool_choice=editor_patch["choice"],
             tools_in_prompt=False,
             **params,
             **reasoning_cfg(False),