From e7da5909107cb76691f0d761eca605c8e33f956f Mon Sep 17 00:00:00 2001 From: parashardhapola Date: Tue, 4 Aug 2026 21:47:39 +0200 Subject: [PATCH 1/5] Add optional scarf.agent Phase 0 plumbing for grounded local LLM decisions. Introduce scarf[agent], decide/check_runtime/load_env, and isolation tests so later ingest and reanalysis stages can call a validated structured-choice primitive. --- .gitignore | 3 + pyproject.toml | 3 + scarf/agent/.env.example | 15 ++ scarf/agent/__init__.py | 23 +++ scarf/agent/_deps.py | 25 +++ scarf/agent/decide.py | 120 ++++++++++++ scarf/agent/runtime.py | 116 ++++++++++++ scarf/agent/types.py | 44 +++++ tests/test_agent_decide.py | 123 ++++++++++++ tests/test_agent_deps.py | 93 +++++++++ tests/test_agent_runtime.py | 147 ++++++++++++++ uv.lock | 368 +++++++++++++++++++++++++++++++++++- 12 files changed, 1079 insertions(+), 1 deletion(-) create mode 100644 scarf/agent/.env.example create mode 100644 scarf/agent/__init__.py create mode 100644 scarf/agent/_deps.py create mode 100644 scarf/agent/decide.py create mode 100644 scarf/agent/runtime.py create mode 100644 scarf/agent/types.py create mode 100644 tests/test_agent_decide.py create mode 100644 tests/test_agent_deps.py create mode 100644 tests/test_agent_runtime.py diff --git a/.gitignore b/.gitignore index d871bad3..146a4e91 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,9 @@ benchmarks/.env.* profiling/.env profiling/.env.* !profiling/.env.example +scarf/agent/.env +scarf/agent/.env.* +!scarf/agent/.env.example /profiling/config.toml /profiling/config.*.toml !/profiling/config.example.toml diff --git a/pyproject.toml b/pyproject.toml index 6bbc748e..c917eeb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ dependencies = [ ] [project.optional-dependencies] +agent = [ + "pydantic-ai-slim[openai]", +] extra = [ "anndata>=0.12", "kneed>=0.8", diff --git a/scarf/agent/.env.example b/scarf/agent/.env.example new file mode 100644 index 00000000..9ead698e --- /dev/null +++ b/scarf/agent/.env.example @@ -0,0 +1,15 @@ +# Copy to .env and fill in values: +# cp scarf/agent/.env.example scarf/agent/.env +# +# Local Ollama (no API key needed): +# OLLAMA_BASE_URL=http://localhost:11434/v1 +# OLLAMA_MODEL=qwen3.5:4b +# +# Ollama Cloud: +# OLLAMA_BASE_URL=https://ollama.com/v1 +# OLLAMA_API_KEY=your-key-here +# OLLAMA_MODEL=qwen3.5:4b + +OLLAMA_BASE_URL=http://localhost:11434/v1 +OLLAMA_MODEL=qwen3.5:4b +# OLLAMA_API_KEY= diff --git a/scarf/agent/__init__.py b/scarf/agent/__init__.py new file mode 100644 index 00000000..9d0c4ff5 --- /dev/null +++ b/scarf/agent/__init__.py @@ -0,0 +1,23 @@ +"""Optional grounded decision helpers for Scarf workflows.""" + +from .decide import DecisionValidationError, decide +from .runtime import check_runtime, load_env +from .types import ( + Decision, + EvidenceItem, + NeedsInput, + StageResult, + StageStatus, +) + +__all__ = [ + "Decision", + "DecisionValidationError", + "EvidenceItem", + "NeedsInput", + "StageResult", + "StageStatus", + "check_runtime", + "decide", + "load_env", +] diff --git a/scarf/agent/_deps.py b/scarf/agent/_deps.py new file mode 100644 index 00000000..b248e2dc --- /dev/null +++ b/scarf/agent/_deps.py @@ -0,0 +1,25 @@ +"""Lazy loaders for optional agent dependencies.""" + +from typing import Any + +AGENT_INSTALL_HINT = ( + "Scarf agent support requires the agent extra. " + "Install with: pip install 'scarf[agent]' " + "or: uv sync --extra agent" +) + + +def require_pydantic() -> tuple[Any, Any]: + try: + from pydantic import BaseModel, Field + except ImportError as exc: + raise ImportError(AGENT_INSTALL_HINT) from exc + return BaseModel, Field + + +def require_pydantic_ai() -> Any: + try: + import pydantic_ai + except ImportError as exc: + raise ImportError(AGENT_INSTALL_HINT) from exc + return pydantic_ai diff --git a/scarf/agent/decide.py b/scarf/agent/decide.py new file mode 100644 index 00000000..f5c6af67 --- /dev/null +++ b/scarf/agent/decide.py @@ -0,0 +1,120 @@ +"""Grounded structured decisions over evidence IDs.""" + +from collections.abc import Sequence +from typing import Any + +from ._deps import require_pydantic_ai +from .types import Decision, EvidenceItem + +_SYSTEM_PROMPT = ( + "Choose exactly one evidence id that answers the question. " + "selectedId must be copied exactly from the id= values. " + "Do not put labels or summaries in selectedId. " + "evidenceIds must include selectedId and only provided ids. " + "Keep the rationale to one short sentence." +) + + +class DecisionValidationError(ValueError): + """Raised when a model decision cites unknown or invalid evidence.""" + + +def _coerce_selected_id(selected_id: str, allowed: set[str]) -> str: + if selected_id in allowed: + return selected_id + matches = [evidence_id for evidence_id in allowed if evidence_id in selected_id] + if len(matches) == 1: + return matches[0] + return selected_id + + +def validate_decision( + decision: Decision, + evidence: Sequence[EvidenceItem], +) -> Decision: + allowed = {item.id for item in evidence} + if not allowed: + raise DecisionValidationError("evidence must contain at least one item") + selected_id = _coerce_selected_id(decision.selectedId, allowed) + evidence_ids = list(decision.evidenceIds) + if selected_id != decision.selectedId or ( + selected_id not in evidence_ids and selected_id in allowed + ): + if selected_id not in evidence_ids: + evidence_ids = [selected_id, *evidence_ids] + decision = Decision( + selectedId=selected_id, + rationale=decision.rationale, + evidenceIds=evidence_ids, + ) + if decision.selectedId not in allowed: + raise DecisionValidationError( + f"selectedId {decision.selectedId!r} is not in evidence ids {sorted(allowed)}" + ) + unknown = [ + evidence_id + for evidence_id in decision.evidenceIds + if evidence_id not in allowed + ] + if unknown: + raise DecisionValidationError( + f"evidenceIds cite unknown ids {unknown}; allowed {sorted(allowed)}" + ) + if decision.selectedId not in decision.evidenceIds: + raise DecisionValidationError( + f"evidenceIds must include selectedId {decision.selectedId!r}" + ) + return decision + + +def _format_user_prompt(question: str, evidence: Sequence[EvidenceItem]) -> str: + lines = [ + question.strip(), + "", + "Choose selectedId from these exact id values:", + ] + for item in evidence: + lines.append(f"- id={item.id} | label={item.label} | summary={item.summary}") + lines.append("") + lines.append( + f"Allowed selectedId values: {', '.join(item.id for item in evidence)}" + ) + return "\n".join(lines) + + +def decide( + *, + model: Any, + question: str, + evidence: Sequence[EvidenceItem], + systemPrompt: str = _SYSTEM_PROMPT, +) -> Decision: + """Ask the model to choose among evidence IDs and validate citations.""" + if not question.strip(): + raise ValueError("question must be non-empty") + if not evidence: + raise ValueError("evidence must contain at least one item") + seen: set[str] = set() + duplicates: set[str] = set() + for item in evidence: + if item.id in seen: + duplicates.add(item.id) + else: + seen.add(item.id) + if duplicates: + raise ValueError( + f"evidence ids must be unique; duplicates: {sorted(duplicates)}" + ) + + require_pydantic_ai() + from pydantic_ai import Agent + from pydantic_ai.settings import ModelSettings + + agent = Agent( + model, + output_type=Decision, + system_prompt=systemPrompt, + model_settings=ModelSettings(thinking=False, extra_body={"think": False}), + ) + result = agent.run_sync(_format_user_prompt(question, evidence)) + return validate_decision(result.output, evidence) diff --git a/scarf/agent/runtime.py b/scarf/agent/runtime.py new file mode 100644 index 00000000..0f5d8616 --- /dev/null +++ b/scarf/agent/runtime.py @@ -0,0 +1,116 @@ +"""Runtime checks and local env loading for LLM endpoints.""" + +import json +import os +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +_ENV_PATH = Path(__file__).resolve().parent / ".env" + + +def load_env(path: Path | None = None) -> Path | None: + """Load scarf/agent/.env into os.environ without overriding existing vars. + + Returns the path loaded, or None if no file was found. + """ + env_path = path or _ENV_PATH + if not env_path.is_file(): + return None + for line in env_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + if key and key not in os.environ: + os.environ[key] = value + return env_path + + +def _unreachable_message(baseUrl: str, model: str) -> str: + return ( + f"OpenAI-compatible endpoint unreachable at {baseUrl!r} " + f"(requested model {model!r})." + ) + + +def _missing_model_message(baseUrl: str, model: str, available: set[str]) -> str: + listed = ", ".join(sorted(available)) if available else "(none)" + return f"Model {model!r} not found at {baseUrl!r}. Available models: {listed}." + + +def _missing_config_message() -> str: + example = _ENV_PATH.with_name(".env.example") + return ( + "baseUrl and model are required. Pass them as arguments or set " + "OLLAMA_BASE_URL and OLLAMA_MODEL in the environment or in " + f"{_ENV_PATH} (see {example})." + ) + + +def _request_json( + url: str, + *, + timeout: float, + api_key: str | None, +) -> Any: + headers = {"Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + request = urllib.request.Request(url, headers=headers, method="GET") + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def _listed_model_ids(payload: Any) -> set[str]: + ids: set[str] = set() + if not isinstance(payload, dict): + return ids + data = payload.get("data") + if isinstance(data, list): + for item in data: + if isinstance(item, dict) and isinstance(item.get("id"), str): + ids.add(item["id"]) + models = payload.get("models") + if isinstance(models, list): + for item in models: + if not isinstance(item, dict): + continue + name = item.get("name") or item.get("model") + if isinstance(name, str): + ids.add(name) + return ids + + +def check_runtime( + *, + baseUrl: str | None = None, + model: str | None = None, + timeout: float = 5.0, +) -> None: + """Fail fast if the OpenAI-compatible endpoint or model is unavailable. + + Loads `scarf/agent/.env` first (without overriding existing env vars). + Uses `OLLAMA_BASE_URL`, `OLLAMA_MODEL`, and `OLLAMA_API_KEY` when args are omitted. + """ + load_env() + resolved_base = baseUrl or os.environ.get("OLLAMA_BASE_URL") + resolved_model = model or os.environ.get("OLLAMA_MODEL") + if not resolved_base or not resolved_model: + raise ValueError(_missing_config_message()) + + api_key = os.environ.get("OLLAMA_API_KEY") or None + models_url = f"{resolved_base.rstrip('/')}/models" + try: + payload = _request_json(models_url, timeout=timeout, api_key=api_key) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + raise RuntimeError(_unreachable_message(resolved_base, resolved_model)) from exc + + available = _listed_model_ids(payload) + if resolved_model not in available: + raise RuntimeError( + _missing_model_message(resolved_base, resolved_model, available) + ) diff --git a/scarf/agent/types.py b/scarf/agent/types.py new file mode 100644 index 00000000..074ec7b2 --- /dev/null +++ b/scarf/agent/types.py @@ -0,0 +1,44 @@ +"""Shared result types for scarf.agent stages and decide().""" + +from typing import Literal + +from ._deps import AGENT_INSTALL_HINT + +try: + from pydantic import BaseModel, Field +except ImportError as exc: + raise ImportError(AGENT_INSTALL_HINT) from exc + + +type StageStatus = Literal["done", "needsInput", "failed"] + + +class EvidenceItem(BaseModel): + id: str + label: str + summary: str + + +class Decision(BaseModel): + selectedId: str = Field( + description="Exact evidence id string from the provided list, nothing else" + ) + rationale: str = Field(description="Short reason for the choice") + evidenceIds: list[str] = Field( + default_factory=list, + description="Evidence ids used; must include selectedId and only provided ids", + ) + + +class NeedsInput(BaseModel): + question: str + options: list[str] = Field(default_factory=list) + evidenceIds: list[str] = Field(default_factory=list) + + +class StageResult(BaseModel): + status: StageStatus + decision: Decision | None = None + needsInput: NeedsInput | None = None + actions: list[str] = Field(default_factory=list) + notes: list[str] = Field(default_factory=list) diff --git a/tests/test_agent_decide.py b/tests/test_agent_decide.py new file mode 100644 index 00000000..e7f0fab6 --- /dev/null +++ b/tests/test_agent_decide.py @@ -0,0 +1,123 @@ +"""Unit tests for grounded decide().""" + +from collections.abc import Sequence + +import pytest +from pydantic_ai.messages import ModelMessage, ModelResponse, ToolCallPart +from pydantic_ai.models.function import FunctionModel +from pydantic_ai.models.function import AgentInfo +from pydantic_ai.models.test import TestModel + +from scarf.agent import DecisionValidationError, EvidenceItem, decide +from scarf.agent.decide import validate_decision +from scarf.agent.types import Decision + + +def _evidence() -> list[EvidenceItem]: + return [ + EvidenceItem(id="matrix:X", label="X", summary="float, mostly non-integer"), + EvidenceItem(id="matrix:raw/X", label="raw/X", summary="integer-like"), + ] + + +def _function_model(decision: Decision) -> FunctionModel: + def reply(_messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + tool = info.output_tools[0] + return ModelResponse( + parts=[ + ToolCallPart( + tool_name=tool.name, + args=decision.model_dump(), + ) + ] + ) + + return FunctionModel(reply) + + +def test_decide_with_function_model_returns_valid_decision() -> None: + expected = Decision( + selectedId="matrix:raw/X", + rationale="integer-like counts", + evidenceIds=["matrix:raw/X"], + ) + result = decide( + model=_function_model(expected), + question="Which matrix looks like raw counts?", + evidence=_evidence(), + ) + assert result == expected + + +def test_decide_with_test_model_returns_schema_valid_decision() -> None: + result = decide( + model=TestModel( + custom_output_args={ + "selectedId": "matrix:raw/X", + "rationale": "integer-like", + "evidenceIds": ["matrix:raw/X"], + } + ), + question="Which matrix looks like raw counts?", + evidence=_evidence(), + ) + assert result.selectedId == "matrix:raw/X" + assert result.evidenceIds == ["matrix:raw/X"] + validate_decision(result, _evidence()) + + +def test_validate_decision_coerces_selected_id_embedded_in_line() -> None: + decision = Decision( + selectedId="id=matrix:raw/X | label=raw/X | summary=integer-like", + rationale="integer-like", + evidenceIds=[], + ) + result = validate_decision(decision, _evidence()) + assert result.selectedId == "matrix:raw/X" + assert result.evidenceIds == ["matrix:raw/X"] + + +def test_decide_rejects_unknown_selected_id() -> None: + bad = Decision( + selectedId="matrix:missing", + rationale="guess", + evidenceIds=["matrix:missing"], + ) + with pytest.raises(DecisionValidationError, match="selectedId"): + decide( + model=_function_model(bad), + question="Which matrix looks like raw counts?", + evidence=_evidence(), + ) + + +def test_validate_decision_rejects_unknown_evidence_ids() -> None: + decision = Decision( + selectedId="matrix:X", + rationale="ok", + evidenceIds=["matrix:X", "matrix:ghost"], + ) + with pytest.raises(DecisionValidationError, match="unknown ids"): + validate_decision(decision, _evidence()) + + +def test_decide_rejects_empty_evidence() -> None: + with pytest.raises(ValueError, match="evidence"): + decide( + model=TestModel(), + question="Which matrix looks like raw counts?", + evidence=[], + ) + + +def test_decide_rejects_duplicate_evidence_ids() -> None: + evidence: Sequence[EvidenceItem] = [ + EvidenceItem(id="matrix:X", label="X", summary="a"), + EvidenceItem(id="matrix:X", label="X2", summary="b"), + ] + with pytest.raises(ValueError, match="unique"): + decide( + model=TestModel(), + question="Which matrix looks like raw counts?", + evidence=evidence, + ) diff --git a/tests/test_agent_deps.py b/tests/test_agent_deps.py new file mode 100644 index 00000000..67a4b299 --- /dev/null +++ b/tests/test_agent_deps.py @@ -0,0 +1,93 @@ +"""Isolation and install-hint tests for scarf.agent.""" + +import ast +import subprocess +import sys +from pathlib import Path + +import pytest + +_SCARF_ROOT = Path(__file__).resolve().parents[1] / "scarf" +_CORE_PACKAGES = ( + "assay", + "clustering", + "datastore", + "embeddings", + "features", + "graph", + "mapping", + "matrix", + "merge", + "metadata", + "metrics", + "neighbors", + "plotting", + "quality_control", + "readers", + "storage", + "trajectory", + "utils", + "writers", +) + + +def test_require_pydantic_ai_missing_extra_hint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import builtins + + from scarf.agent import _deps + + real_import = builtins.__import__ + + def fake_import(name: str, *args: object, **kwargs: object): + if name == "pydantic_ai" or name.startswith("pydantic_ai."): + raise ImportError("simulated missing pydantic_ai") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + with pytest.raises(ImportError, match=r"scarf\[agent\]") as exc_info: + _deps.require_pydantic_ai() + assert "uv sync --extra agent" in str(exc_info.value) + + +def test_import_scarf_does_not_load_pydantic_ai() -> None: + script = """ +import sys +import scarf +assert "pydantic_ai" not in sys.modules +assert not any(name.startswith("pydantic_ai.") for name in sys.modules) +assert "scarf.agent" not in sys.modules +print("ok") +""" + completed = subprocess.run( + [sys.executable, "-c", script], + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stderr + assert completed.stdout.strip() == "ok" + + +def test_core_packages_do_not_import_scarf_agent() -> None: + violations: list[str] = [] + for package_name in _CORE_PACKAGES: + package_root = _SCARF_ROOT / package_name + if not package_root.exists(): + continue + for path in package_root.rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "scarf.agent" or alias.name.startswith( + "scarf.agent." + ): + violations.append(path.as_posix()) + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module == "scarf.agent" or node.module.startswith( + "scarf.agent." + ): + violations.append(path.as_posix()) + assert violations == [] diff --git a/tests/test_agent_runtime.py b/tests/test_agent_runtime.py new file mode 100644 index 00000000..3dc4e534 --- /dev/null +++ b/tests/test_agent_runtime.py @@ -0,0 +1,147 @@ +"""Runtime reachability tests for scarf.agent.check_runtime.""" + +import json +import os +from typing import Any + +import pytest + +from scarf.agent import check_runtime, load_env +from scarf.agent import runtime as runtime_module + + +class _FakeResponse: + def __init__(self, payload: dict[str, Any]) -> None: + self._payload = payload + + def read(self) -> bytes: + return json.dumps(self._payload).encode("utf-8") + + def __enter__(self) -> "_FakeResponse": + return self + + def __exit__(self, *args: object) -> None: + return None + + +def test_load_env_reads_file_without_overriding( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + env_file = tmp_path / ".env" + env_file.write_text( + "OLLAMA_BASE_URL=https://from-file.example/v1\n" + "OLLAMA_MODEL=from-file\n" + "OLLAMA_API_KEY=from-file-key\n", + encoding="utf-8", + ) + monkeypatch.delenv("OLLAMA_BASE_URL", raising=False) + monkeypatch.delenv("OLLAMA_MODEL", raising=False) + monkeypatch.setenv("OLLAMA_API_KEY", "already-set") + + loaded = load_env(env_file) + assert loaded == env_file + assert os.environ["OLLAMA_BASE_URL"] == "https://from-file.example/v1" + assert os.environ["OLLAMA_MODEL"] == "from-file" + assert os.environ["OLLAMA_API_KEY"] == "already-set" + + +def test_check_runtime_ok_when_model_listed(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_urlopen(request: object, timeout: float = 0.0): + return _FakeResponse({"data": [{"id": "test-model"}]}) + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + check_runtime(baseUrl="http://example.test/v1", model="test-model") + + +def test_check_runtime_sends_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: + seen: dict[str, str] = {} + + def fake_urlopen(request: Any, timeout: float = 0.0): + seen["authorization"] = request.get_header("Authorization") + return _FakeResponse({"data": [{"id": "test-model"}]}) + + monkeypatch.setenv("OLLAMA_API_KEY", "secret-key") + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + check_runtime(baseUrl="http://example.test/v1", model="test-model") + assert seen["authorization"] == "Bearer secret-key" + + +def test_check_runtime_uses_env_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_urlopen(request: object, timeout: float = 0.0): + return _FakeResponse({"data": [{"id": "env-model"}]}) + + monkeypatch.setattr(runtime_module, "load_env", lambda path=None: None) + monkeypatch.setenv("OLLAMA_BASE_URL", "http://example.test/v1") + monkeypatch.setenv("OLLAMA_MODEL", "env-model") + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + check_runtime() + + +def test_check_runtime_fails_when_endpoint_down( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import urllib.error + + def fake_urlopen(request: object, timeout: float = 0.0): + raise urllib.error.URLError("connection refused") + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + with pytest.raises(RuntimeError, match="endpoint unreachable"): + check_runtime(baseUrl="http://example.test/v1", model="test-model") + + +def test_check_runtime_fails_when_model_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_urlopen(request: object, timeout: float = 0.0): + return _FakeResponse({"data": [{"id": "other-model"}]}) + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + with pytest.raises(RuntimeError, match="Model 'test-model' not found"): + check_runtime(baseUrl="http://example.test/v1", model="test-model") + + +@pytest.mark.integration +def test_check_runtime_live_ollama_smoke() -> None: + import urllib.error + import urllib.request + + load_env() + base_url = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434/v1") + model = os.environ.get("OLLAMA_MODEL", "qwen3.5:4b") + try: + urllib.request.urlopen(f"{base_url.rstrip('/')}/models", timeout=2.0) + except (urllib.error.URLError, TimeoutError): + pytest.skip("OpenAI-compatible endpoint is not reachable") + + try: + check_runtime(baseUrl=base_url, model=model) + except RuntimeError as exc: + pytest.skip(str(exc)) + + from pydantic_ai.models.ollama import OllamaModel + from pydantic_ai.providers.ollama import OllamaProvider + + from scarf.agent import EvidenceItem, decide + + decision = decide( + model=OllamaModel( + model, + provider=OllamaProvider(base_url=base_url), + ), + question="Which matrix looks like raw counts?", + evidence=[ + EvidenceItem( + id="matrix:X", + label="X", + summary="float, mostly non-integer", + ), + EvidenceItem( + id="matrix:raw/X", + label="raw/X", + summary="integer-like", + ), + ], + ) + assert decision.selectedId in {"matrix:X", "matrix:raw/X"} + assert decision.selectedId in decision.evidenceIds diff --git a/uv.lock b/uv.lock index 7f2b366d..37abd81e 100644 --- a/uv.lock +++ b/uv.lock @@ -680,6 +680,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "docutils" version = "0.22.4" @@ -876,6 +885,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, ] +[[package]] +name = "genai-prices" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx2" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/9b/85e646305a90a2da18f1edf055498668391e71f9849d3e1754d66559a311/genai_prices-0.1.1.tar.gz", hash = "sha256:54a2237691e0aaefb057d10a0c3c20160accc9fc09521c64c03fcdb7a4a69f68", size = 91182, upload-time = "2026-08-01T09:02:49.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/5e/cfe36dff790ffad6aeff8a069b6f36743987ac17053579035ee0a67635dd/genai_prices-0.1.1-py3-none-any.whl", hash = "sha256:de2e3d8ea3ca1d0d292025995c598da447a74e94f22cd3342df46941aeb5416b", size = 95300, upload-time = "2026-08-01T09:02:48.308Z" }, +] + [[package]] name = "google-crc32c" version = "1.8.0" @@ -954,6 +976,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/0d/ca7d15afbdc397e3401134c9e1800d51d12b829661786187a4ad08fe484f/greenlet-3.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:b7068bd09f761f3f5b4d214c2bed063186b2a86148c740b3873e3f56d79bac31", size = 242586, upload-time = "2026-06-26T18:23:37.93Z" }, ] +[[package]] +name = "griffelib" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/e4/8d187ea29c2e30b3a09505c567513077d6117861bde1fbd997a167f262ec/griffelib-2.1.0.tar.gz", hash = "sha256:762a186d2c6fd6794d4ea20d428d597ffb857cb56b66421651cbba15bdd5e813", size = 216234, upload-time = "2026-06-19T12:05:42.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, +] + [[package]] name = "grpclib" version = "0.4.9" @@ -1087,6 +1118,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/a8/20ed1ed79cbc2ecdf5301c0968ab7c85547212e2a7bd126ddd2d986e206e/httpcore2-2.9.1.tar.gz", hash = "sha256:4d8acbf8b306f48c9d6046591fd5ba4037d1b1b1000d140fc2c3eab1e9a0c0e2", size = 67089, upload-time = "2026-07-24T09:21:03.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/fb/46c52b781975c335a2bcf1072c7bbc007cbdc8d674217f5ee1daba2c848b/httpcore2-2.9.1-py3-none-any.whl", hash = "sha256:6182472379e855fe4221246a2bb7ecede403bc61c6798062ae1787d051ccde26", size = 82809, upload-time = "2026-07-24T09:21:01.178Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -1102,6 +1146,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/14/38128fbafd7e0ed41d874df6c9a653d47c2d111cfe59e2b4ac95161b4abd/httpx2-2.9.1.tar.gz", hash = "sha256:1932a768737e3666291582833da748cc4e563c337cf96706fccc04fa6e58764a", size = 95458, upload-time = "2026-07-24T09:21:04.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/b8/cfd91c4ab9134d386d48f0b6ac662ff3d4be6efdee59ee1c67ebc3c0487c/httpx2-2.9.1-py3-none-any.whl", hash = "sha256:1820fe14a9ab1107bfeff39259987429450b070ec0ff38cc87eb0d8c97fdc71a", size = 91191, upload-time = "2026-07-24T09:21:02.6Z" }, +] + [[package]] name = "huggingface-hub" version = "1.24.0" @@ -1300,6 +1360,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "joblib" version = "1.5.3" @@ -1606,6 +1734,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/79/d3bbab197e86e0ff4f9c07122895b66a3e0d024247fcff7f12c473cb36d9/llvmlite-0.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6842cf6f707ec4be3d985a385ad03f72b2d724439e118fcbe99b2929964f0453", size = 39153839, upload-time = "2026-03-31T18:29:51.004Z" }, ] +[[package]] +name = "logfire-api" +version = "4.39.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/f4/41f8647f6091fb9b9aac5a4b6d164bddb11d55b8369bf38de154ef91b8f4/logfire_api-4.39.0.tar.gz", hash = "sha256:1e885f95c37d58cdb927bbc6baea4f4a7c13066f6b3019758627d4dc442643d0", size = 90619, upload-time = "2026-07-24T18:31:36.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/d4/87747d12eaf2d852676fd6535df76df945ef62681f1eb5391b63d1fc05e2/logfire_api-4.39.0-py3-none-any.whl", hash = "sha256:20057bbd2898dec2eed02e2559bd73f4e10bc4b108987821df55e9c762da3ba8", size = 140413, upload-time = "2026-07-24T18:31:32.818Z" }, +] + [[package]] name = "loguru" version = "0.7.3" @@ -2223,6 +2360,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/9a/d6127f5422b78e0222b0a9eadcfd7a5aa8d873a9498da7d4a77d4ac8ce2e/obstore-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:676d1154f6f08721110f9b7d14ee3a3c0293abaf9da135bb90f54e276dca1cac", size = 5314113, upload-time = "2026-06-25T18:29:21.209Z" }, ] +[[package]] +name = "openai" +version = "2.53.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -2626,6 +2794,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] +[[package]] +name = "pydantic-ai-slim" +version = "2.23.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "genai-prices" }, + { name = "griffelib" }, + { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pydantic-graph" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/9f/53b19efefa041c1080f7c4ad41679a9293cce64f1265168a98cbe06a0ab7/pydantic_ai_slim-2.23.0.tar.gz", hash = "sha256:d16dcbfb2bfea0ee162bf0f499442fab5a4d69b41e4c54f3b694c2e90b983768", size = 965485, upload-time = "2026-08-04T01:58:20.668Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/6f/539a255524178a8421d582271a8d7f8667b036f02b4ddc4f20abcc63888b/pydantic_ai_slim-2.23.0-py3-none-any.whl", hash = "sha256:a2fa3e56408bbf1b83900e3dd4ad9b137297742f450863c2f0f9a03a547d0e33", size = 1157486, upload-time = "2026-08-04T01:58:12.356Z" }, +] + +[package.optional-dependencies] +openai = [ + { name = "openai" }, + { name = "tiktoken" }, +] + [[package]] name = "pydantic-core" version = "2.46.4" @@ -2701,6 +2894,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, ] +[[package]] +name = "pydantic-graph" +version = "2.23.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "logfire-api" }, + { name = "pydantic" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/fc/273bac7d14fb62c060e0c20c51a9cc60e9e90a96d992fd20e77abf4b6ac1/pydantic_graph-2.23.0.tar.gz", hash = "sha256:54c9939f47fd8a268c96320d7d90e7cef037cbfd2625a675dc4028c1377f70ab", size = 45179, upload-time = "2026-08-04T01:58:23.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/c4/875cf853d205dc55422bd44ff0fbfac82e6e34ff7693df016fc3a4088d32/pydantic_graph-2.23.0-py3-none-any.whl", hash = "sha256:b0f12b4f72adb2a5522b5962c95e1a7b140cb3f631a628036f935e291a9e50ba", size = 52662, upload-time = "2026-08-04T01:58:15.858Z" }, +] + [[package]] name = "pydata-sphinx-theme" version = "0.16.1" @@ -2935,6 +3144,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -3136,6 +3433,9 @@ dependencies = [ ] [package.optional-dependencies] +agent = [ + { name = "pydantic-ai-slim", extra = ["openai"] }, +] docs = [ { name = "jedi" }, { name = "jinja2" }, @@ -3211,6 +3511,7 @@ requires-dist = [ { name = "obstore", specifier = ">=0.5.1" }, { name = "packaging" }, { name = "pandas", specifier = ">=2.2.2" }, + { name = "pydantic-ai-slim", extras = ["openai"], marker = "extra == 'agent'" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=7.0" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4" }, { name = "pytest-xdist", marker = "extra == 'test'", specifier = ">=3" }, @@ -3238,7 +3539,7 @@ requires-dist = [ { name = "zarr", specifier = ">=3.2.0" }, { name = "zstandard", specifier = ">=0.25.0" }, ] -provides-extras = ["extra", "test", "docs"] +provides-extras = ["agent", "extra", "test", "docs"] [package.metadata.requires-dev] dev = [ @@ -3439,6 +3740,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "snowballstemmer" version = "3.1.1" @@ -3770,6 +4080,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] +[[package]] +name = "tiktoken" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, + { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, + { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, + { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, + { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, + { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, + { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, + { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/8c/93/0dd6adca026a616c3a92974566b43381eea4b475ce1f36c062b8271a9ac5/tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471", size = 1034977, upload-time = "2026-05-15T04:51:00.957Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5ec6e6bc5b30bed6d93f7f2162d8f6b32437b3ba27cb527cfe004f6109c9/tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd", size = 983635, upload-time = "2026-05-15T04:51:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c8ae9aff00d625c50659b4513e707a0462c4bf5d4d6cc1b802103225c02e/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881", size = 1116036, upload-time = "2026-05-15T04:51:04.082Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/6a5dddd1d0a6018ecb389bd0353e6b4a515eb4d2286611bd0ace1937b9e1/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24", size = 1135544, upload-time = "2026-05-15T04:51:05.229Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b8/585032b4384b2f7dcdaddcb52865c83a701a420d09e3c2b4a2be1c450c57/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273", size = 1182217, upload-time = "2026-05-15T04:51:06.517Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b6/993ff1ded3958215fd341a847b8e5ffeb5de473f435296870d314fc91ac4/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51", size = 1239404, upload-time = "2026-05-15T04:51:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3d/fef7e06e3b33e7538db0ced734cf9fe23b6832d2ac4990c119c377aec55e/tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58", size = 918686, upload-time = "2026-05-15T04:51:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/c1/82/a7fc44582bc32ab00de988a2299bf77c077f59068b233109e34b7d6ca7e6/tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b", size = 1034454, upload-time = "2026-05-15T04:51:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/37/d0/24d8a890c14f432a05cea669c17bebeaa99f96a7c79523b590f564246411/tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448", size = 982976, upload-time = "2026-05-15T04:51:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/49/b7/2ab43f62788a9266187a9bfc1d3af99ad83e5eaa25fbef168a69cd5ad14f/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a", size = 1115526, upload-time = "2026-05-15T04:51:12.608Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/1494321ed323ce7a14d88e3cd6cb9058625977df1c6961ddc492bd10a9f3/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad", size = 1136466, upload-time = "2026-05-15T04:51:13.926Z" }, + { url = "https://files.pythonhosted.org/packages/96/d9/dfd086aa2d918c563a140720e0ce296cada1634efd2783d5cf51e05f984e/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e", size = 1181863, upload-time = "2026-05-15T04:51:15.025Z" }, + { url = "https://files.pythonhosted.org/packages/2f/68/a18b4f307086954fdae32714cb4f85562e34f9d34ab206e61f1816aa6018/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424", size = 1239218, upload-time = "2026-05-15T04:51:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, +] + [[package]] name = "toml" version = "0.10.2" @@ -3835,6 +4192,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/8d/1080ee4c231f361b6ce4470d556c8c435b67c7e0753aaa641497ee92f88b/traitlets-5.15.1-py3-none-any.whl", hash = "sha256:770a53705f84b81ac107e83a1b3328ff2dae16094d8fc3cfc004e4b22dfd8e92", size = 85858, upload-time = "2026-06-03T12:26:04.395Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "types-certifi" version = "2021.10.8.3" From b5fde18462f3e33042cb09bc07442d475e588fd4 Mon Sep 17 00:00:00 2001 From: parashardhapola Date: Tue, 4 Aug 2026 23:05:42 +0200 Subject: [PATCH 2/5] Add scarf.agent Phase 1 ingest for format detect, convert, and open. Keep imports deterministic: block prenorm-only H5AD unless matrixKey is forced, and require an explicit HTO vs ADT choice when Antibody Capture names look like hashtags. --- scarf/agent/__init__.py | 4 + scarf/agent/ingest/__init__.py | 113 ++++++++++++++ scarf/agent/ingest/cellranger.py | 67 +++++++++ scarf/agent/ingest/common.py | 152 +++++++++++++++++++ scarf/agent/ingest/detect.py | 42 ++++++ scarf/agent/ingest/h5ad.py | 136 +++++++++++++++++ scarf/agent/ingest/loom.py | 46 ++++++ scarf/agent/ingest/mtx.py | 48 ++++++ scarf/agent/ingest/result.py | 68 +++++++++ scarf/agent/ingest/seurat.py | 37 +++++ scarf/agent/ingest/zarr_store.py | 22 +++ scarf/readers/_h5ad_inspect.py | 24 ++- tests/test_agent_ingest.py | 251 +++++++++++++++++++++++++++++++ 13 files changed, 1008 insertions(+), 2 deletions(-) create mode 100644 scarf/agent/ingest/__init__.py create mode 100644 scarf/agent/ingest/cellranger.py create mode 100644 scarf/agent/ingest/common.py create mode 100644 scarf/agent/ingest/detect.py create mode 100644 scarf/agent/ingest/h5ad.py create mode 100644 scarf/agent/ingest/loom.py create mode 100644 scarf/agent/ingest/mtx.py create mode 100644 scarf/agent/ingest/result.py create mode 100644 scarf/agent/ingest/seurat.py create mode 100644 scarf/agent/ingest/zarr_store.py create mode 100644 tests/test_agent_ingest.py diff --git a/scarf/agent/__init__.py b/scarf/agent/__init__.py index 9d0c4ff5..3155f489 100644 --- a/scarf/agent/__init__.py +++ b/scarf/agent/__init__.py @@ -1,6 +1,7 @@ """Optional grounded decision helpers for Scarf workflows.""" from .decide import DecisionValidationError, decide +from .ingest import IngestResult, detect_format, ingest from .runtime import check_runtime, load_env from .types import ( Decision, @@ -14,10 +15,13 @@ "Decision", "DecisionValidationError", "EvidenceItem", + "IngestResult", "NeedsInput", "StageResult", "StageStatus", "check_runtime", "decide", + "detect_format", + "ingest", "load_env", ] diff --git a/scarf/agent/ingest/__init__.py b/scarf/agent/ingest/__init__.py new file mode 100644 index 00000000..3470e15b --- /dev/null +++ b/scarf/agent/ingest/__init__.py @@ -0,0 +1,113 @@ +"""Ingest Scarf-supported inputs into a typed Zarr store.""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .cellranger import ingest_cellranger +from .detect import detect_format +from .h5ad import ingest_h5ad +from .loom import ingest_loom +from .mtx import ingest_mtx +from .result import IngestResult, needs_input +from .seurat import ingest_seurat +from .zarr_store import ingest_zarr + +__all__ = [ + "IngestResult", + "detect_format", + "ingest", +] + + +def ingest( + *, + path: str | Path, + zarrPath: str | Path | None = None, + model: Any | None = None, + directions: Mapping[str, Any] | None = None, +) -> IngestResult: + """Detect, inspect, convert, and open a Scarf store, or return NeedsInput.""" + source = Path(path) + if not source.exists(): + return IngestResult( + status="failed", + notes=[f"Input path does not exist: {source}"], + ) + + direction_map = dict(directions or {}) + notes: list[str] = [] + format_name = str(direction_map.get("format") or detect_format(source)) + notes.append(f"Detected format: {format_name}") + + if format_name == "zarr": + return ingest_zarr( + source, + notes, + default_assay=direction_map.get("defaultAssay"), + ) + if format_name == "h5ad": + return ingest_h5ad( + source, + zarrPath=zarrPath, + model=model, + directions=direction_map, + notes=notes, + ) + if format_name == "10x_h5": + return ingest_cellranger( + source, + format_name=format_name, + reader_class_name="CrH5Reader", + zarrPath=zarrPath, + model=model, + directions=direction_map, + notes=notes, + ) + if format_name == "10x_dir": + return ingest_cellranger( + source, + format_name=format_name, + reader_class_name="CrDirReader", + zarrPath=zarrPath, + model=model, + directions=direction_map, + notes=notes, + ) + if format_name == "mtx": + return ingest_mtx( + source, + zarrPath=zarrPath, + directions=direction_map, + notes=notes, + ) + if format_name == "loom": + return ingest_loom( + source, + zarrPath=zarrPath, + directions=direction_map, + notes=notes, + ) + if format_name == "seurat": + return ingest_seurat( + source, + zarrPath=zarrPath, + directions=direction_map, + notes=notes, + ) + if format_name == "csv": + return needs_input( + format_name="csv", + question=( + "CSV/TSV import needs explicit parameters " + "(assay name, cell/feature id columns). Provide directions to continue." + ), + options=[], + evidence_ids=[], + notes=notes, + ) + return IngestResult( + status="failed", + format=format_name, + notes=[*notes, f"Unsupported input format for path {source}"], + ) diff --git a/scarf/agent/ingest/cellranger.py b/scarf/agent/ingest/cellranger.py new file mode 100644 index 00000000..95dcf5b2 --- /dev/null +++ b/scarf/agent/ingest/cellranger.py @@ -0,0 +1,67 @@ +"""Cell Ranger H5 and directory ingest handlers.""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .common import finish, require_zarr_path, resolve_modality_choice +from .result import IngestResult + + +def ingest_cellranger( + path: Path, + *, + format_name: str, + reader_class_name: str, + zarrPath: str | Path | None, + model: Any | None, + directions: Mapping[str, Any], + notes: list[str], +) -> IngestResult: + from ...readers import CrDirReader, CrH5Reader + from ...writers import CrToZarr + + reader_cls = CrH5Reader if reader_class_name == "CrH5Reader" else CrDirReader + reader = reader_cls(str(path)) + decision = None + rename_assays: dict[str, str] = dict(directions.get("renameAssays") or {}) + + assay_columns = list(reader.assayFeats.columns) + if "ADT" in assay_columns and "HTO" not in assay_columns: + adt_names = [str(name) for name in reader.feature_names("ADT")] + choice, decision, blocked = resolve_modality_choice( + model=model, + directions=directions, + feature_names=adt_names, + format_name=format_name, + ) + if blocked is not None: + blocked.notes = [*notes, *blocked.notes] + return blocked + if choice == "HTO": + rename_assays.setdefault("ADT", "HTO") + notes.append("Renamed ADT assay to HTO") + + if rename_assays: + reader.rename_assays(rename_assays) + + zarr_path = require_zarr_path(zarrPath, format_name=format_name) + writer = CrToZarr(reader, zarr_loc=zarr_path) + writer.dump() + return finish( + format_name=format_name, + zarr_path=zarr_path, + notes=notes, + convert_actions=[ + { + "op": "CrToZarr", + "path": str(path), + "zarrPath": zarr_path, + "readerClass": reader_class_name, + "renameAssays": rename_assays or None, + } + ], + action_labels=["convert_cellranger", "open_datastore"], + default_assay=directions.get("defaultAssay"), + decision=decision, + ) diff --git a/scarf/agent/ingest/common.py b/scarf/agent/ingest/common.py new file mode 100644 index 00000000..e4e75464 --- /dev/null +++ b/scarf/agent/ingest/common.py @@ -0,0 +1,152 @@ +"""Shared helpers for format-specific ingest handlers.""" + +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from ..decide import decide +from ..types import Decision, EvidenceItem +from .result import IngestResult, done, needs_input + +# `hto(?![a-z])` matches HTO, HTO1, HTO-1; avoids requiring a word boundary +# after HTO (digits are word chars, so `hto\b` misses HTO1). +_HTO_NAME_RE = re.compile( + r"(hashtag|hto(?![a-z])|totalseq[^a-z0-9]*hash|hash[^a-z0-9]*tag)", + re.IGNORECASE, +) + + +def require_zarr_path(zarrPath: str | Path | None, *, format_name: str) -> str: + if zarrPath is None: + raise ValueError(f"zarrPath is required when converting {format_name} inputs") + return str(zarrPath) + + +def open_summary( + zarr_path: str, + *, + default_assay: str | None = None, +) -> tuple[list[str], str | None, dict[str, Any]]: + import zarr + + from ...datastore.datastore import DataStore + from ...storage.stores import load_zarr + + resolved_default = default_assay + if resolved_default is None: + root = load_zarr(zarr_loc=zarr_path, mode="r") + assay_names = [ + name + for name in sorted(dict.fromkeys(root.group_keys())) + if isinstance(root[name], zarr.Group) and "is_assay" in root[name].attrs + ] + if "RNA" in assay_names: + resolved_default = "RNA" + elif assay_names: + resolved_default = assay_names[0] + + ds = DataStore(zarr_path, default_assay=resolved_default) + summary = ds.summary().to_dict() + return list(ds.assay_names), resolved_default, summary + + +def datastore_action(zarr_path: str, default_assay: str | None) -> dict[str, Any]: + action: dict[str, Any] = {"op": "DataStore", "zarrPath": zarr_path} + if default_assay is not None: + action["defaultAssay"] = default_assay + return action + + +def finish( + *, + format_name: str, + zarr_path: str, + notes: list[str], + convert_actions: list[dict[str, Any]], + action_labels: list[str], + default_assay: str | None = None, + decision: Decision | None = None, +) -> IngestResult: + """Open the store, append DataStore replay action, and return a done result.""" + assay_names, resolved_default, summary = open_summary( + zarr_path, + default_assay=default_assay, + ) + return done( + format_name=format_name, + zarr_path=zarr_path, + assay_names=assay_names, + summary=summary, + accepted_actions=[ + *convert_actions, + datastore_action(zarr_path, resolved_default), + ], + action_labels=action_labels, + notes=notes, + decision=decision, + ) + + +def antibody_names_look_like_hto(names: Sequence[str]) -> bool: + if not names: + return False + hits = sum(1 for name in names if _HTO_NAME_RE.search(str(name))) + return hits >= max(1, (len(names) + 1) // 2) + + +def resolve_modality_choice( + *, + model: Any | None, + directions: Mapping[str, Any], + feature_names: Sequence[str], + format_name: str, +) -> tuple[str | None, Decision | None, IngestResult | None]: + forced = directions.get("modalityChoice") + if forced in {"ADT", "HTO"}: + return str(forced), None, None + if not antibody_names_look_like_hto(feature_names): + return "ADT", None, None + + evidence = [ + EvidenceItem( + id="modality:ADT", + label="ADT", + summary="Treat Antibody Capture features as surface protein ADT assay", + ), + EvidenceItem( + id="modality:HTO", + label="HTO", + summary=( + "Treat Antibody Capture features as multiplexing HTO assay; " + f"names include {[str(name) for name in feature_names[:8]]}" + ), + ), + ] + if model is None: + return ( + None, + None, + needs_input( + format_name=format_name, + question=( + "Antibody Capture features look like hashtags. " + "Should they be imported as ADT or HTO?" + ), + options=["ADT", "HTO"], + evidence_ids=[item.id for item in evidence], + notes=[ + "Hashtag-like Antibody Capture features require an explicit choice" + ], + ), + ) + decision = decide( + model=model, + question=( + "Antibody Capture features look like hashtags. " + "Choose modality:ADT or modality:HTO." + ), + evidence=evidence, + ) + choice = "HTO" if decision.selectedId.endswith("HTO") else "ADT" + return choice, decision, None diff --git a/scarf/agent/ingest/detect.py b/scarf/agent/ingest/detect.py new file mode 100644 index 00000000..4e42d705 --- /dev/null +++ b/scarf/agent/ingest/detect.py @@ -0,0 +1,42 @@ +"""Input format detection for ingest.""" + +from pathlib import Path + + +def detect_format(path: str | Path) -> str: + """Detect a Scarf-supported input family from path layout.""" + source = Path(path) + suffix = "".join(source.suffixes).lower() if source.suffixes else "" + name = source.name.lower() + + if name.endswith(".h5ad") or suffix.endswith(".h5ad"): + return "h5ad" + if name.endswith(".loom") or suffix.endswith(".loom"): + return "loom" + if name.endswith(".rds") or name.endswith(".h5seurat") or suffix.endswith(".rds"): + return "seurat" + if name.endswith(".csv") or name.endswith(".tsv") or name.endswith(".txt"): + return "csv" + if name.endswith(".zarr") or (source.is_dir() and _looks_like_zarr(source)): + return "zarr" + if source.is_file() and name.endswith((".h5", ".hdf5")): + return "10x_h5" + if source.is_dir() and _looks_like_10x_dir(source): + return "10x_dir" + if source.is_dir() or name.endswith((".mtx", ".mtx.gz")): + return "mtx" + return "unknown" + + +def _looks_like_zarr(path: Path) -> bool: + return (path / "zarr.json").exists() or (path / "cellData").exists() + + +def _looks_like_10x_dir(path: Path) -> bool: + names = {child.name.lower() for child in path.iterdir()} if path.exists() else set() + has_matrix = any("matrix.mtx" in name for name in names) + has_barcodes = any("barcode" in name for name in names) + has_features = any( + name.startswith("features") or name.startswith("genes") for name in names + ) + return has_matrix and has_barcodes and has_features diff --git a/scarf/agent/ingest/h5ad.py b/scarf/agent/ingest/h5ad.py new file mode 100644 index 00000000..c60ec7a6 --- /dev/null +++ b/scarf/agent/ingest/h5ad.py @@ -0,0 +1,136 @@ +"""H5AD ingest handler.""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .common import finish, require_zarr_path, resolve_modality_choice +from .result import IngestResult, needs_input + + +def _antibody_names(inspection: Any) -> list[str]: + if "ADT" not in inspection.suggestedAssays: + return [] + import h5py + + from ...readers._h5ad_inspect import _as_text, _read_column + + with h5py.File(inspection.h5adFn, mode="r") as h5: + feature_node = h5.get(inspection.featureAttrsKey) + if feature_node is None: + return [] + types = _read_column(feature_node, inspection.assaySplitKey) + names = _read_column(feature_node, inspection.featureNameKey) + if types is None or names is None: + return [] + return [ + _as_text(name) + for feature_type, name in zip(types, names, strict=True) + if _as_text(feature_type) == "Antibody Capture" + ] + + +def ingest_h5ad( + path: Path, + *, + zarrPath: str | Path | None, + model: Any | None, + directions: Mapping[str, Any], + notes: list[str], +) -> IngestResult: + from ...readers import H5adReader, inspect_h5ad + from ...writers import H5adToZarr + + forced_matrix = directions.get("matrixKey") + if forced_matrix is not None: + forced_matrix = str(forced_matrix) + try: + inspection = inspect_h5ad(str(path), matrix_key=forced_matrix) + except ValueError as exc: + return IngestResult( + status="failed", + format="h5ad", + notes=[*notes, str(exc)], + ) + notes.append( + f"Forced matrix {inspection.matrixKey} via directions " + f"(integerLike={inspection.integerLike})" + ) + else: + inspection = inspect_h5ad(str(path)) + notes.append( + f"Selected matrix {inspection.matrixKey} " + f"(integerLike={inspection.integerLike})" + ) + if not inspection.integerLike: + return needs_input( + format_name="h5ad", + question=( + "No integer-like count matrix matched this H5AD layout. " + "Provide a raw counts matrix, or retry with " + 'directions={"matrixKey": ""} to force a candidate.' + ), + options=list(inspection.matrixCandidates), + evidence_ids=[f"matrix:{key}" for key in inspection.matrixCandidates], + notes=[ + *notes, + "Prenormalized-only H5AD inputs are not imported silently", + ], + ) + + assay_name_map = dict(directions.get("assayNameMap") or {}) + decision = None + if inspection.assaySplitKey and "ADT" in inspection.suggestedAssays: + choice, decision, blocked = resolve_modality_choice( + model=model, + directions=directions, + feature_names=_antibody_names(inspection), + format_name="h5ad", + ) + if blocked is not None: + blocked.notes = [*notes, *blocked.notes] + return blocked + if choice == "HTO": + assay_name_map.setdefault("Antibody Capture", "HTO") + notes.append("Mapped Antibody Capture to HTO") + + zarr_path = require_zarr_path(zarrPath, format_name="h5ad") + reader = H5adReader.from_inspect(inspection) + try: + writer_kwargs: dict[str, Any] = {"zarr_loc": zarr_path} + if inspection.assaySplitKey is not None: + writer_kwargs["assay_split_key"] = inspection.assaySplitKey + if assay_name_map: + writer_kwargs["assay_name_map"] = assay_name_map + else: + writer_kwargs["assay_name"] = directions.get("assayName") or "RNA" + writer = H5adToZarr(reader, **writer_kwargs) + writer.dump() + finally: + reader.h5.close() + + return finish( + format_name="h5ad", + zarr_path=zarr_path, + notes=notes, + convert_actions=[ + { + "op": "inspect_h5ad", + "path": str(path), + "matrixKey": inspection.matrixKey, + }, + { + "op": "H5adToZarr", + "path": str(path), + "zarrPath": zarr_path, + "assaySplitKey": inspection.assaySplitKey, + "assayNameMap": assay_name_map or None, + "assayName": None + if inspection.assaySplitKey is not None + else writer_kwargs.get("assay_name"), + }, + ], + action_labels=["inspect_h5ad", "convert_h5ad", "open_datastore"], + default_assay=directions.get("defaultAssay"), + decision=decision, + ) diff --git a/scarf/agent/ingest/loom.py b/scarf/agent/ingest/loom.py new file mode 100644 index 00000000..7495b721 --- /dev/null +++ b/scarf/agent/ingest/loom.py @@ -0,0 +1,46 @@ +"""Loom ingest handler.""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .common import finish, require_zarr_path +from .result import IngestResult + + +def ingest_loom( + path: Path, + *, + zarrPath: str | Path | None, + directions: Mapping[str, Any], + notes: list[str], +) -> IngestResult: + from ...readers import LoomReader + from ...writers import LoomToZarr + + zarr_path = require_zarr_path(zarrPath, format_name="loom") + reader_kwargs = {} + if directions.get("cellNamesKey"): + reader_kwargs["cell_names_key"] = directions["cellNamesKey"] + if directions.get("featureNamesKey"): + reader_kwargs["feature_names_key"] = directions["featureNamesKey"] + reader = LoomReader(str(path), **reader_kwargs) + try: + writer = LoomToZarr( + reader, + zarr_loc=zarr_path, + assay_name=directions.get("assayName") or "RNA", + ) + writer.dump() + finally: + reader.h5.close() + return finish( + format_name="loom", + zarr_path=zarr_path, + notes=notes, + convert_actions=[ + {"op": "LoomToZarr", "path": str(path), "zarrPath": zarr_path} + ], + action_labels=["convert_loom", "open_datastore"], + default_assay=directions.get("defaultAssay"), + ) diff --git a/scarf/agent/ingest/mtx.py b/scarf/agent/ingest/mtx.py new file mode 100644 index 00000000..6294580e --- /dev/null +++ b/scarf/agent/ingest/mtx.py @@ -0,0 +1,48 @@ +"""Matrix Market ingest handler.""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .common import finish, require_zarr_path +from .result import IngestResult, needs_input + + +def ingest_mtx( + path: Path, + *, + zarrPath: str | Path | None, + directions: Mapping[str, Any], + notes: list[str], +) -> IngestResult: + from ...readers import MtxReader, inspect_mtx + from ...writers import MtxToZarr + + candidates = inspect_mtx(path) + if not candidates: + return IngestResult( + status="failed", + format="mtx", + notes=[*notes, f"No MTX matrix candidates found under {path}"], + ) + if len(candidates) > 1 and directions.get("mtxIndex") is None: + return needs_input( + format_name="mtx", + question="Multiple MTX layouts found. Which candidate index should be used?", + options=[str(index) for index in range(len(candidates))], + evidence_ids=[f"mtx:{index}" for index in range(len(candidates))], + notes=[*notes, f"Found {len(candidates)} MTX candidates"], + ) + index = int(directions.get("mtxIndex") or 0) + reader = MtxReader(candidates[index]) + zarr_path = require_zarr_path(zarrPath, format_name="mtx") + writer = MtxToZarr(reader, zarr_loc=zarr_path) + writer.dump() + return finish( + format_name="mtx", + zarr_path=zarr_path, + notes=notes, + convert_actions=[{"op": "MtxToZarr", "path": str(path), "zarrPath": zarr_path}], + action_labels=["convert_mtx", "open_datastore"], + default_assay=directions.get("defaultAssay"), + ) diff --git a/scarf/agent/ingest/result.py b/scarf/agent/ingest/result.py new file mode 100644 index 00000000..8e0e732d --- /dev/null +++ b/scarf/agent/ingest/result.py @@ -0,0 +1,68 @@ +"""Ingest result types and stage helpers.""" + +from typing import Any + +from .._deps import AGENT_INSTALL_HINT +from ..types import Decision, NeedsInput, StageStatus + +try: + from pydantic import BaseModel, Field +except ImportError as exc: + raise ImportError(AGENT_INSTALL_HINT) from exc + + +class IngestResult(BaseModel): + status: StageStatus + format: str | None = None + zarrPath: str | None = None + assayNames: list[str] = Field(default_factory=list) + summary: dict[str, Any] | None = None + decision: Decision | None = None + needsInput: NeedsInput | None = None + actions: list[str] = Field(default_factory=list) + acceptedActions: list[dict[str, Any]] = Field(default_factory=list) + notes: list[str] = Field(default_factory=list) + + +def done( + *, + format_name: str, + zarr_path: str, + assay_names: list[str], + summary: dict[str, Any], + accepted_actions: list[dict[str, Any]], + action_labels: list[str], + notes: list[str], + decision: Decision | None = None, +) -> IngestResult: + return IngestResult( + status="done", + format=format_name, + zarrPath=zarr_path, + assayNames=assay_names, + summary=summary, + decision=decision, + actions=action_labels, + acceptedActions=accepted_actions, + notes=notes, + ) + + +def needs_input( + *, + format_name: str, + question: str, + options: list[str], + evidence_ids: list[str], + notes: list[str] | None = None, +) -> IngestResult: + return IngestResult( + status="needsInput", + format=format_name, + needsInput=NeedsInput( + question=question, + options=options, + evidenceIds=evidence_ids, + ), + notes=notes or [], + ) diff --git a/scarf/agent/ingest/seurat.py b/scarf/agent/ingest/seurat.py new file mode 100644 index 00000000..1897e253 --- /dev/null +++ b/scarf/agent/ingest/seurat.py @@ -0,0 +1,37 @@ +"""Seurat ingest handler.""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .common import finish, require_zarr_path +from .result import IngestResult + + +def ingest_seurat( + path: Path, + *, + zarrPath: str | Path | None, + directions: Mapping[str, Any], + notes: list[str], +) -> IngestResult: + from ...readers import SeuratReader + from ...writers import SeuratToZarr + + zarr_path = require_zarr_path(zarrPath, format_name="seurat") + reader = SeuratReader(str(path)) + try: + writer = SeuratToZarr(reader, zarr_loc=zarr_path) + writer.dump() + finally: + reader.close() + return finish( + format_name="seurat", + zarr_path=zarr_path, + notes=notes, + convert_actions=[ + {"op": "SeuratToZarr", "path": str(path), "zarrPath": zarr_path} + ], + action_labels=["convert_seurat", "open_datastore"], + default_assay=directions.get("defaultAssay"), + ) diff --git a/scarf/agent/ingest/zarr_store.py b/scarf/agent/ingest/zarr_store.py new file mode 100644 index 00000000..f1a06a53 --- /dev/null +++ b/scarf/agent/ingest/zarr_store.py @@ -0,0 +1,22 @@ +"""Open an existing Scarf Zarr store.""" + +from pathlib import Path + +from .common import finish +from .result import IngestResult + + +def ingest_zarr( + path: Path, + notes: list[str], + *, + default_assay: str | None, +) -> IngestResult: + return finish( + format_name="zarr", + zarr_path=str(path), + notes=notes, + convert_actions=[], + action_labels=["open_datastore"], + default_assay=default_assay, + ) diff --git a/scarf/readers/_h5ad_inspect.py b/scarf/readers/_h5ad_inspect.py index d82cb086..89385935 100644 --- a/scarf/readers/_h5ad_inspect.py +++ b/scarf/readers/_h5ad_inspect.py @@ -55,6 +55,7 @@ class H5adInspectResult: matrixKey: str matrixCandidates: tuple[str, ...] matrixEncoding: str + integerLike: bool cellAttrsKey: str cellIdsKey: str featureAttrsKey: str @@ -545,11 +546,18 @@ def _select_matrix( raise ValueError("No matrix candidate matches the obs and var dimensions") -def inspect_h5ad(h5ad_fn: str) -> H5adInspectResult: +def inspect_h5ad( + h5ad_fn: str, + *, + matrix_key: str | None = None, +) -> H5adInspectResult: """Report the matrix and metadata layout of an H5AD file. Args: h5ad_fn: Path to the H5AD file. + matrix_key: Optional matrix path to force (for example ``X`` or + ``raw/X``). When set, that candidate must exist and match obs/var + dimensions. Returns: Keys, shape, and column names needed to configure @@ -560,7 +568,18 @@ def inspect_h5ad(h5ad_fn: str) -> H5adInspectResult: if not candidates: raise ValueError("No sparse or numeric 2D matrix found in the H5AD file") - matrix, feature_attrs_key = _select_matrix(h5, candidates) + if matrix_key is not None: + forced = [ + candidate for candidate in candidates if candidate.key == matrix_key + ] + if not forced: + available = ", ".join(candidate.key for candidate in candidates) + raise ValueError( + f"matrix_key {matrix_key!r} not found. Available: {available}" + ) + matrix, feature_attrs_key = _select_matrix(h5, forced) + else: + matrix, feature_attrs_key = _select_matrix(h5, candidates) n_cells, n_features = matrix.shape cell_node = h5.get("obs") feature_node = h5.get(feature_attrs_key) @@ -624,6 +643,7 @@ def inspect_h5ad(h5ad_fn: str) -> H5adInspectResult: matrixKey=matrix.key, matrixCandidates=tuple(candidate.key for candidate in candidates), matrixEncoding=matrix.encoding, + integerLike=matrix.integerLike, cellAttrsKey="obs", cellIdsKey=cell_ids_key, featureAttrsKey=feature_attrs_key, diff --git a/tests/test_agent_ingest.py b/tests/test_agent_ingest.py new file mode 100644 index 00000000..e298f448 --- /dev/null +++ b/tests/test_agent_ingest.py @@ -0,0 +1,251 @@ +"""Tests for scarf.agent.ingest.""" + +from pathlib import Path + +import h5py +import numpy as np +import pytest +from scipy.sparse import csr_matrix + +from scarf.agent import detect_format, ingest +from scarf.agent.types import Decision +from scarf.readers import inspect_h5ad + + +def _write_sparse_group( + h5: h5py.File | h5py.Group, key: str, values: np.ndarray +) -> None: + matrix = csr_matrix(values) + group = h5.create_group(key) + group.attrs["encoding-type"] = "csr_matrix" + group.attrs["shape"] = values.shape + group.create_dataset("data", data=matrix.data) + group.create_dataset("indices", data=matrix.indices) + group.create_dataset("indptr", data=matrix.indptr) + + +def _write_h5ad( + path: Path, + values: np.ndarray, + *, + feature_types: list[bytes] | None = None, + feature_names: list[bytes] | None = None, + raw_values: np.ndarray | None = None, +) -> None: + n_cells, n_feats = values.shape + with h5py.File(path, mode="w") as h5: + _write_sparse_group(h5, "X", values) + if raw_values is not None: + _write_sparse_group(h5, "raw/X", raw_values) + raw_var = h5.create_group("raw/var") + raw_n = raw_values.shape[1] + raw_var.create_dataset( + "_index", + data=np.array([f"rf{i}".encode() for i in range(raw_n)]), + ) + raw_var.create_dataset( + "feature_name", + data=np.array( + feature_names + if feature_names is not None + else [f"g{i}".encode() for i in range(raw_n)] + ), + ) + if feature_types is not None: + raw_var.create_dataset("feature_types", data=np.array(feature_types)) + + obs = h5.create_group("obs") + obs.create_dataset( + "_index", + data=np.array([f"c{i}".encode() for i in range(n_cells)]), + ) + var = h5.create_group("var") + var.create_dataset( + "_index", + data=np.array([f"f{i}".encode() for i in range(n_feats)]), + ) + var.create_dataset( + "feature_name", + data=np.array( + feature_names + if feature_names is not None and raw_values is None + else [f"g{i}".encode() for i in range(n_feats)] + ), + ) + if feature_types is not None and raw_values is None: + var.create_dataset("feature_types", data=np.array(feature_types)) + + +def test_detect_format_by_suffix(tmp_path: Path) -> None: + assert detect_format(tmp_path / "a.h5ad") == "h5ad" + assert detect_format(tmp_path / "a.loom") == "loom" + assert detect_format(tmp_path / "a.rds") == "seurat" + assert detect_format(tmp_path / "a.csv") == "csv" + assert detect_format(tmp_path / "a.zarr") == "zarr" + + +def test_ingest_h5ad_prefers_raw_integer_matrix(tmp_path: Path) -> None: + path = tmp_path / "counts.h5ad" + _write_h5ad( + path, + np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32), + raw_values=np.array([[1, 0, 3], [0, 2, 4]], dtype=np.uint16), + feature_types=[b"Gene Expression", b"Gene Expression", b"Gene Expression"], + feature_names=[b"g1", b"g2", b"g3"], + ) + inspection = inspect_h5ad(str(path)) + assert inspection.matrixKey == "raw/X" + assert inspection.integerLike is True + + result = ingest(path=path, zarrPath=tmp_path / "out.zarr") + assert result.status == "done" + assert result.format == "h5ad" + assert result.zarrPath is not None + assert "RNA" in result.assayNames + assert result.summary is not None + assert result.acceptedActions + assert result.acceptedActions[-1]["op"] == "DataStore" + + +def test_ingest_h5ad_stops_on_prenormalized_only(tmp_path: Path) -> None: + path = tmp_path / "prenorm.h5ad" + _write_h5ad( + path, + np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32), + ) + result = ingest(path=path, zarrPath=tmp_path / "out.zarr") + assert result.status == "needsInput" + assert result.needsInput is not None + assert ( + "integer-like" in result.needsInput.question.lower() + or "raw" in result.needsInput.question.lower() + ) + assert "matrixKey" in result.needsInput.question + + +def test_ingest_h5ad_force_matrix_key_allows_prenorm(tmp_path: Path) -> None: + path = tmp_path / "prenorm.h5ad" + _write_h5ad( + path, + np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32), + ) + result = ingest( + path=path, + zarrPath=tmp_path / "forced.zarr", + directions={"matrixKey": "X"}, + ) + assert result.status == "done" + assert result.format == "h5ad" + assert "RNA" in result.assayNames + assert any("Forced matrix X" in note for note in result.notes) + + +def test_ingest_h5ad_hto_digit_names_need_input(tmp_path: Path) -> None: + path = tmp_path / "hto_digits.h5ad" + _write_h5ad( + path, + np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint16), + feature_types=[b"Gene Expression", b"Antibody Capture", b"Antibody Capture"], + feature_names=[b"GENE1", b"HTO1", b"HTO2"], + ) + result = ingest(path=path, zarrPath=tmp_path / "out.zarr") + assert result.status == "needsInput" + assert result.needsInput is not None + assert set(result.needsInput.options) == {"ADT", "HTO"} + + +def test_antibody_names_look_like_hto_digit_suffix() -> None: + from scarf.agent.ingest.common import antibody_names_look_like_hto + + assert antibody_names_look_like_hto(["HTO1", "HTO2"]) + assert antibody_names_look_like_hto(["Hashtag1"]) + assert not antibody_names_look_like_hto(["CD3", "CD19"]) + + +def test_ingest_h5ad_ambiguous_hto_without_model_needs_input(tmp_path: Path) -> None: + path = tmp_path / "hto.h5ad" + _write_h5ad( + path, + np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint16), + feature_types=[b"Gene Expression", b"Antibody Capture", b"Antibody Capture"], + feature_names=[b"GENE1", b"Hashtag1", b"TotalSeq-Hashtag2"], + ) + result = ingest(path=path, zarrPath=tmp_path / "out.zarr") + assert result.status == "needsInput" + assert result.needsInput is not None + assert set(result.needsInput.options) == {"ADT", "HTO"} + + +def test_ingest_h5ad_modality_choice_via_directions(tmp_path: Path) -> None: + path = tmp_path / "hto.h5ad" + _write_h5ad( + path, + np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint16), + feature_types=[b"Gene Expression", b"Antibody Capture", b"Antibody Capture"], + feature_names=[b"GENE1", b"Hashtag1", b"TotalSeq-Hashtag2"], + ) + result = ingest( + path=path, + zarrPath=tmp_path / "out.zarr", + directions={"modalityChoice": "HTO"}, + ) + assert result.status == "done" + assert "HTO" in result.assayNames + assert "ADT" not in result.assayNames + + +def test_ingest_h5ad_modality_choice_via_function_model(tmp_path: Path) -> None: + from pydantic_ai.messages import ModelMessage, ModelResponse, ToolCallPart + from pydantic_ai.models.function import AgentInfo, FunctionModel + + path = tmp_path / "hto.h5ad" + _write_h5ad( + path, + np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint16), + feature_types=[b"Gene Expression", b"Antibody Capture", b"Antibody Capture"], + feature_names=[b"GENE1", b"Hashtag1", b"TotalSeq-Hashtag2"], + ) + + expected = Decision( + selectedId="modality:HTO", + rationale="hashtag names", + evidenceIds=["modality:HTO"], + ) + + def reply(_messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + tool = info.output_tools[0] + return ModelResponse( + parts=[ToolCallPart(tool_name=tool.name, args=expected.model_dump())] + ) + + result = ingest( + path=path, + zarrPath=tmp_path / "out.zarr", + model=FunctionModel(reply), + ) + assert result.status == "done" + assert result.decision is not None + assert result.decision.selectedId == "modality:HTO" + assert "HTO" in result.assayNames + + +def test_ingest_csv_needs_input(tmp_path: Path) -> None: + path = tmp_path / "table.csv" + path.write_text("a,b\n1,2\n", encoding="utf-8") + result = ingest(path=path, zarrPath=tmp_path / "out.zarr") + assert result.status == "needsInput" + assert result.format == "csv" + + +def test_ingest_10x_h5(tmp_path: Path) -> None: + from tests import full_path + + fixture = Path(full_path("1K_pbmc_citeseq.h5")) + if not fixture.is_file(): + pytest.skip("10x H5 fixture not downloaded") + result = ingest(path=fixture, zarrPath=tmp_path / "pbmc.zarr") + assert result.status == "done" + assert result.format == "10x_h5" + assert "RNA" in result.assayNames + assert result.acceptedActions + assert result.acceptedActions[-1]["op"] == "DataStore" From 7c967a8e18c540c6d83fc3e66ba4e7485410bc58 Mon Sep 17 00:00:00 2001 From: parashardhapola Date: Wed, 5 Aug 2026 13:03:56 +0200 Subject: [PATCH 3/5] Add scarf.agent Phase 2a covariate characterization and design confounding. Label cell metadata domains and coefficients, collapse same-domain aliases, and report design-table association with an intercept-aware estimability check that skips technicals varying within the observation unit. --- scarf/agent/__init__.py | 6 + scarf/agent/characterize_covariates.py | 832 ++++++++++++++++++++ scarf/metrics/__init__.py | 18 + scarf/metrics/association.py | 429 ++++++++++ tests/test_agent_characterize_covariates.py | 333 ++++++++ tests/test_metrics_association.py | 337 ++++++++ 6 files changed, 1955 insertions(+) create mode 100644 scarf/agent/characterize_covariates.py create mode 100644 scarf/metrics/association.py create mode 100644 tests/test_agent_characterize_covariates.py create mode 100644 tests/test_metrics_association.py diff --git a/scarf/agent/__init__.py b/scarf/agent/__init__.py index 3155f489..a0ba1c03 100644 --- a/scarf/agent/__init__.py +++ b/scarf/agent/__init__.py @@ -1,5 +1,9 @@ """Optional grounded decision helpers for Scarf workflows.""" +from .characterize_covariates import ( + CovariateCharacterization, + characterize_covariates, +) from .decide import DecisionValidationError, decide from .ingest import IngestResult, detect_format, ingest from .runtime import check_runtime, load_env @@ -12,6 +16,7 @@ ) __all__ = [ + "CovariateCharacterization", "Decision", "DecisionValidationError", "EvidenceItem", @@ -19,6 +24,7 @@ "NeedsInput", "StageResult", "StageStatus", + "characterize_covariates", "check_runtime", "decide", "detect_format", diff --git a/scarf/agent/characterize_covariates.py b/scarf/agent/characterize_covariates.py new file mode 100644 index 00000000..594f4afb --- /dev/null +++ b/scarf/agent/characterize_covariates.py @@ -0,0 +1,832 @@ +"""Characterize cell covariates and study-design confounding.""" + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Literal, cast + +import numpy as np +import pandas as pd + +from ..metrics.association import report_confounding, report_technical_nesting +from ..storage.types import as_zarr_array, as_zarr_group +from ._deps import AGENT_INSTALL_HINT +from .decide import decide +from .types import Decision, EvidenceItem, StageStatus + +try: + from pydantic import BaseModel, Field +except ImportError as exc: + raise ImportError(AGENT_INSTALL_HINT) from exc + +__all__ = [ + "CovariateCharacterization", + "characterize_covariates", +] + +Domain = Literal["biological", "technical", "design", "ignore", "unknown"] +ColumnKind = Literal["categorical", "continuous"] + +_DOMAINS = frozenset({"biological", "technical", "design", "ignore", "unknown"}) +# Only these domains reach the design table, so only they are worth collapsing. +_ANALYSED = frozenset({"biological", "technical", "design"}) +_KINDS = frozenset({"categorical", "continuous"}) +_RESERVED_COLUMNS = frozenset({"I", "ids", "names"}) +_EMBEDDING_TOKENS = ( + "umap", + "pca", + "tsne", + "scvi", + "latent", + "phate", + "forceatlas", + "diffmap", + "diffusionmap", + "diffusion", +) +_SHORT_EMBEDDING_PARTS = frozenset({"fa", "dm", "pc"}) +_INDEXED_NAME = re.compile(r"(?P.+?)[-_]?(?P\d+)") +_ONTOLOGY_SUFFIX = "_ontology_term_id" +_CATEGORICAL_MAX_LEVELS = 50 +_SAMPLE_LEVELS = 8 +_CONTEXT_LIMIT = 1200 +_ASSOCIATION_FLOOR = 0.1 +_DROP_REASONS = { + "dropAssayStat": "Scarf assay statistic column", + "dropProvenance": "analysis-linked column", + "dropEmbedding": "embedding-style column", +} + +_DOMAIN_EVIDENCE = [ + EvidenceItem( + id="domain:biological", + label="biological", + summary="Biology of interest such as disease, sex, genotype, treatment", + ), + EvidenceItem( + id="domain:technical", + label="technical", + summary="Technical handling such as batch, chemistry, sequencing run", + ), + EvidenceItem( + id="domain:design", + label="design", + summary="Sampling design such as donor, sample, replicate, subject", + ), + EvidenceItem( + id="domain:ignore", + label="ignore", + summary="Identifiers, QC metrics, clusters, or other non-design labels", + ), + EvidenceItem( + id="domain:unknown", + label="unknown", + summary="Cannot classify from available evidence", + ), +] +_COEFFICIENT_EVIDENCE = [ + EvidenceItem( + id="coefficient:yes", + label="yes", + summary="Treat this biological column as a coefficient of interest", + ), + EvidenceItem( + id="coefficient:no", + label="no", + summary="Do not treat this biological column as a coefficient of interest", + ), +] + + +class CovariateCharacterization(BaseModel): + status: StageStatus + auditLog: list[dict[str, Any]] = Field(default_factory=list) + actions: list[str] = Field(default_factory=list) + notes: list[str] = Field(default_factory=list) + decisions: list[dict[str, Any]] = Field(default_factory=list) + columns: list[dict[str, Any]] = Field(default_factory=list) + coefficients: list[dict[str, Any]] = Field(default_factory=list) + technicalNesting: list[dict[str, Any]] = Field(default_factory=list) + confounding: list[dict[str, Any]] = Field(default_factory=list) + + +@dataclass +class _Run: + """Mutable state shared by the stage steps.""" + + frame: pd.DataFrame + context: str + model: Any | None + kinds: dict[str, ColumnKind] = field(default_factory=dict) + domains: dict[str, Domain] = field(default_factory=dict) + audit: list[dict[str, Any]] = field(default_factory=list) + actions: list[str] = field(default_factory=list) + decisions: list[dict[str, Any]] = field(default_factory=list) + + def note(self, *, kind: str, detail: str, **fields: Any) -> None: + self.audit.append({"kind": kind, "detail": detail, **fields}) + + def summary(self, name: str) -> str: + return _summarize(self.frame[name].to_numpy(), self.kinds[name]) + + def ask( + self, + *, + task: str, + question: str, + evidence: Sequence[EvidenceItem], + column: str | None = None, + ) -> Decision | None: + """Run one grounded decision, or return None when it cannot be asked.""" + if self.model is None or len(evidence) < 2: + return None + decision = decide(model=self.model, question=question, evidence=evidence) + record: dict[str, Any] = { + "task": task, + "selectedId": decision.selectedId, + "rationale": decision.rationale, + "evidenceIds": list(decision.evidenceIds), + } + if column is not None: + record["column"] = column + self.decisions.append(record) + return decision + + +def _is_embedding_column(name: str) -> bool: + match = _INDEXED_NAME.fullmatch(name) + if match is None: + return False + parts = [part for part in re.split(r"[-_]+", match.group("stem").lower()) if part] + compact = "".join(parts) + if any(token in compact for token in _EMBEDDING_TOKENS): + return True + return any(part in _SHORT_EMBEDDING_PARTS for part in parts) + + +def _has_source_artifact(store: Any, column: str) -> bool: + try: + cell_data = as_zarr_group(store.zw["cellData"], name="cellData") + if column not in cell_data: + return False + attrs = as_zarr_array(cell_data[column], name=column).attrs + except (KeyError, TypeError, ValueError): + return False + return isinstance(attrs.get("source_artifact"), dict) + + +def _infer_kind(values: np.ndarray) -> ColumnKind: + if ( + values.dtype == object + or np.issubdtype(values.dtype, np.str_) + or np.issubdtype(values.dtype, np.bool_) + ): + return "categorical" + try: + numeric = np.asarray(values, dtype=float) + except (TypeError, ValueError): + return "categorical" + finite = numeric[np.isfinite(numeric)] + if finite.size == 0 or not bool(np.all(np.mod(finite, 1) == 0)): + return "continuous" + limit = min(_CATEGORICAL_MAX_LEVELS, max(2, len(values) // 20)) + return "categorical" if int(np.unique(finite).size) <= limit else "continuous" + + +def _summarize(values: np.ndarray, kind: ColumnKind) -> str: + series = pd.Series(values) + missing = int(series.isna().sum()) + if kind == "categorical": + levels = series.dropna().astype(str).value_counts() + top = ", ".join( + f"{level}={int(count)}" + for level, count in levels.head(_SAMPLE_LEVELS).items() + ) + return f"categorical levels={levels.shape[0]} missing={missing} top=[{top}]" + numeric = pd.to_numeric(series, errors="coerce").to_numpy(dtype=float, copy=False) + finite = numeric[np.isfinite(numeric)] + if finite.size == 0: + return f"continuous missing={missing} finite=0" + return ( + f"continuous missing={missing} min={float(finite.min()):.4g} " + f"max={float(finite.max()):.4g} mean={float(finite.mean()):.4g}" + ) + + +def _partition_signature(values: np.ndarray) -> tuple[int, ...]: + codes, _ = pd.factorize(pd.Series(values), use_na_sentinel=True) + return tuple(int(code) for code in codes.tolist()) + + +def _triage_columns( + store: Any, + *, + cell_key: str, + exclude: set[str], +) -> tuple[list[str], list[tuple[str, str]]]: + """Split cell columns into model candidates and deterministic drops.""" + assay_prefixes = tuple(f"{name}_" for name in store.assay_names) + candidates: list[str] = [] + dropped: list[tuple[str, str]] = [] + for name in store.cells.columns: + if name in _RESERVED_COLUMNS or name == cell_key or name in exclude: + continue + if name.startswith(assay_prefixes): + dropped.append((name, "dropAssayStat")) + elif _has_source_artifact(store, name): + dropped.append((name, "dropProvenance")) + elif _is_embedding_column(name): + dropped.append((name, "dropEmbedding")) + else: + candidates.append(name) + return candidates, dropped + + +def _collapse_ontology_aliases( + columns: Sequence[str], + frame: pd.DataFrame, +) -> tuple[list[str], dict[str, list[str]], list[dict[str, Any]]]: + """Collapse ``x`` with ``x_ontology_term_id`` when partitions match. + + Arbitrary identical partitions are kept apart: perfect confounding between + biology and batch is a finding to report, not an alias to drop. + """ + present = set(columns) + aliases: dict[str, list[str]] = {} + dropped: set[str] = set() + notes: list[dict[str, Any]] = [] + for name in columns: + if not name.endswith(_ONTOLOGY_SUFFIX): + continue + base = name[: -len(_ONTOLOGY_SUFFIX)] + if base not in present or {name, base} & dropped: + continue + if _partition_signature(frame[name].to_numpy()) != _partition_signature( + frame[base].to_numpy() + ): + continue + aliases.setdefault(base, []).append(name) + dropped.add(name) + notes.append( + { + "kind": "ontologyAlias", + "detail": f"Collapsed ontology alias {name} onto {base}", + "representative": base, + "aliases": [name], + } + ) + return [name for name in columns if name not in dropped], aliases, notes + + +def _bounded_context(study_context: str | None) -> str: + text = (study_context or "").strip() + return text if len(text) <= _CONTEXT_LIMIT else text[: _CONTEXT_LIMIT - 3] + "..." + + +def _validate_directions( + directions: Mapping[str, Any], + available: set[str], +) -> list[str]: + errors: list[str] = [] + + def check_names(key: str, names: Any) -> list[str] | None: + if not isinstance(names, Sequence) or isinstance(names, str | bytes): + errors.append(f"{key} must be a sequence of column names") + return None + unknown = sorted(set(names) - available) + if unknown: + errors.append(f"{key} cites unknown columns: {unknown}") + return list(names) + + for key, allowed in (("columnKinds", _KINDS), ("columnDomains", _DOMAINS)): + mapping = directions.get(key) + if mapping is None: + continue + if not isinstance(mapping, Mapping): + errors.append(f"{key} must be a mapping") + continue + check_names(key, list(mapping)) + invalid = sorted({str(value) for value in mapping.values()} - allowed) + if invalid: + errors.append(f"{key} has unsupported values: {invalid}") + + for key in ("coefficientsOfInterest", "excludeColumns"): + names = directions.get(key) + if names is not None: + check_names(key, names) + + units = directions.get("unitsOfInference") + if units is None: + return errors + if not isinstance(units, Mapping): + errors.append("unitsOfInference must be a mapping") + return errors + for coefficient, unit_map in units.items(): + if coefficient not in available: + errors.append(f"unitsOfInference cites unknown coefficient {coefficient!r}") + continue + if not isinstance(unit_map, Mapping): + errors.append(f"unitsOfInference[{coefficient!r}] must be a mapping") + continue + for unit_key in ("observationUnit", "independentUnit"): + unit_name = unit_map.get(unit_key) + if unit_name is not None and unit_name not in available: + errors.append( + f"unitsOfInference[{coefficient!r}].{unit_key} " + f"cites unknown column {unit_name!r}" + ) + return errors + + +def _level_correspondence(frame: pd.DataFrame, names: Sequence[str]) -> str: + """Matched label tuples for columns known to share one partition.""" + rows = frame.loc[:, list(names)].drop_duplicates() + shown = "; ".join( + " = ".join(str(value) for value in row) + for row in rows.head(_SAMPLE_LEVELS).itertuples(index=False) + ) + return shown if len(rows) <= _SAMPLE_LEVELS else f"{shown}; ..." + + +def _choose_representative( + run: _Run, + members: Sequence[str], + correspondence: str, +) -> str | None: + decision = run.ask( + task="equivalentColumns", + question=( + f"Columns {', '.join(members)} assign every cell to the same groups. " + f"Their labels line up as {correspondence}. Decide whether they record " + "one variable under different labels, and if so whose labels to keep." + ), + evidence=[ + EvidenceItem( + id="equivalent:distinct", + label="distinct variables", + summary="Different variables that happen to coincide in this dataset", + ), + *( + EvidenceItem( + id=f"equivalent:{name}", + label=name, + summary=run.summary(name), + ) + for name in members + ), + ], + ) + selected = ( + None if decision is None else decision.selectedId.removeprefix("equivalent:") + ) + if selected in set(members): + run.actions.append(f"equivalentColumns:{selected}") + return selected + run.note( + kind="equivalentKeptApart", + detail=( + f"{', '.join(members)} share one partition but were not collapsed " + + ("(no model decision)" if decision is None else "(judged distinct)") + ), + columns=list(members), + levels=correspondence, + ) + return None + + +def _collapse_equivalent_columns( + run: _Run, + candidates: Sequence[str], + aliases: dict[str, list[str]], +) -> list[str]: + """Collapse categorical columns that cut the cells into identical groups. + + An identical partition is also what perfect confounding looks like, so a + class is only eligible when every member carries the same analysis domain, + and the representative is a model judgement rather than a name rule. + Extends ``aliases`` in place. + """ + classes: dict[tuple[int, ...], list[str]] = {} + for name in candidates: + if run.kinds[name] != "categorical" or run.domains[name] not in _ANALYSED: + continue + signature = _partition_signature(run.frame[name].to_numpy()) + classes.setdefault(signature, []).append(name) + + # Store column order is not stable, and members of a class are + # interchangeable, so order them here to keep prompts and notes reproducible. + dropped: set[str] = set() + for members in sorted(sorted(group) for group in classes.values()): + if len(members) < 2: + continue + domains = sorted({run.domains[name] for name in members}) + correspondence = _level_correspondence(run.frame, members) + if len(domains) > 1: + run.note( + kind="equivalentAcrossDomains", + detail=( + f"{', '.join(members)} share one partition across domains " + f"{domains}; kept apart as perfect confounding" + ), + columns=list(members), + domains=domains, + levels=correspondence, + ) + continue + representative = _choose_representative(run, members, correspondence) + if representative is None: + continue + others = [name for name in members if name != representative] + aliases.setdefault(representative, []).extend(others) + dropped.update(others) + run.note( + kind="equivalentColumns", + detail=f"Collapsed {', '.join(others)} onto {representative}", + representative=representative, + aliases=others, + levels=correspondence, + ) + return [name for name in candidates if name not in dropped] + + +def _assign_domain(run: _Run, name: str, directed: Mapping[str, Domain]) -> Domain: + if name in directed: + domain = directed[name] + run.actions.append(f"domain:{name}->{domain} (directions)") + return domain + decision = run.ask( + task="columnDomain", + column=name, + question=( + f"Assign a domain for cell metadata column {name}. " + f"Column summary: {run.summary(name)}. " + f"Study context: {run.context or 'none provided'}." + ), + evidence=_DOMAIN_EVIDENCE, + ) + if decision is None: + run.note( + kind="domainUnknown", + detail=f"Left {name} as unknown domain", + column=name, + ) + return "unknown" + selected = decision.selectedId.removeprefix("domain:") + if selected not in _DOMAINS: + run.note( + kind="domainUnknown", + detail=f"Unsupported domain {selected!r} returned for {name}", + column=name, + ) + return "unknown" + run.actions.append(f"domain:{name}->{selected}") + return cast(Domain, selected) + + +def _select_coefficients( + run: _Run, + *, + biological: Sequence[str], + directed: set[str], +) -> list[str]: + selected: list[str] = [] + for name in biological: + if name in directed: + selected.append(name) + run.actions.append(f"coefficient:{name} (directions)") + continue + decision = run.ask( + task="coefficientOfInterest", + column=name, + question=( + f"Should biological column {name} be a coefficient of interest? " + f"Column summary: {run.summary(name)}. " + f"Study context: {run.context or 'none provided'}." + ), + evidence=_COEFFICIENT_EVIDENCE, + ) + if decision is None: + run.note( + kind="coefficientSkipped", + detail=f"No model or direction for biological column {name}", + column=name, + ) + elif decision.selectedId == "coefficient:yes": + selected.append(name) + run.actions.append(f"coefficient:{name}") + return selected + + +def _unit_evidence(run: _Run, names: Sequence[str], prefix: str) -> list[EvidenceItem]: + return [ + EvidenceItem( + id=f"{prefix}:{name}", + label=name, + summary=(f"domain={run.domains.get(name, 'unknown')}; {run.summary(name)}"), + ) + for name in names + ] + + +def _resolve_units( + run: _Run, + coefficient: str, + *, + directed: Mapping[str, Any], + design_columns: Sequence[str], + unit_candidates: Sequence[str], +) -> tuple[str | None, str | None]: + unit_map = dict(directed.get(coefficient) or {}) + observation = unit_map.get("observationUnit") + independent = unit_map.get("independentUnit") + + if observation is None: + decision = run.ask( + task="observationUnit", + column=coefficient, + question=( + f"Choose the observation unit for coefficient {coefficient}. " + "Each distinct value of this column is one design-table row. " + f"Study context: {run.context or 'none provided'}." + ), + evidence=_unit_evidence( + run, + [name for name in unit_candidates if name != coefficient], + "unit", + ), + ) + if decision is not None: + observation = decision.selectedId.removeprefix("unit:") + run.actions.append(f"observationUnit:{coefficient}->{observation}") + + if observation is not None and independent is None: + decision = run.ask( + task="independentUnit", + column=coefficient, + question=( + f"Optional independent unit for coefficient {coefficient} " + f"with observation unit {observation}." + ), + evidence=[ + EvidenceItem( + id="independentUnit:none", + label="none", + summary="No separate independent unit or subject column", + ), + *_unit_evidence( + run, + [ + name + for name in design_columns + if name not in {coefficient, observation} + ], + "independentUnit", + ), + ], + ) + if decision is not None and decision.selectedId != "independentUnit:none": + independent = decision.selectedId.removeprefix("independentUnit:") + run.actions.append(f"independentUnit:{coefficient}->{independent}") + return observation, independent + + +def _characterize_coefficient( + run: _Run, + coefficient: str, + *, + observation_unit: str | None, + independent_unit: str | None, + technical: Sequence[str], +) -> tuple[dict[str, Any], dict[str, Any] | None]: + record: dict[str, Any] = { + "name": coefficient, + "kind": run.kinds[coefficient], + "observationUnit": observation_unit, + "independentUnit": independent_unit, + "scope": "unresolvedUnit", + } + if observation_unit is None or observation_unit not in run.frame.columns: + run.note( + kind="unresolvedUnit", + detail=f"No usable observation unit for coefficient {coefficient}", + column=coefficient, + ) + return record, None + + grouped = run.frame.groupby(observation_unit, dropna=False)[coefficient] + if not bool(grouped.nunique(dropna=False).le(1).all()): + record["scope"] = "withinUnit" + run.note( + kind="withinUnit", + detail=( + f"{coefficient} varies within {observation_unit}; recorded as " + "composition and skipped for design-table association" + ), + column=coefficient, + observationUnit=observation_unit, + ) + return record, None + + record["scope"] = "betweenUnit" + # Technically only columns constant inside the observation unit have a + # well-defined design-table value; drop_duplicates would otherwise keep an + # arbitrary cell row. + unit_constant: list[str] = [] + for name in technical: + if name not in run.frame.columns: + continue + if bool( + run.frame.groupby(observation_unit, dropna=False)[name] + .nunique(dropna=False) + .le(1) + .all() + ): + unit_constant.append(name) + continue + run.note( + kind="technicalVariesWithinUnit", + detail=( + f"{name} varies within {observation_unit}; " + f"excluded from design-table association for {coefficient}" + ), + column=name, + coefficient=coefficient, + observationUnit=observation_unit, + ) + + group_cols = [observation_unit] + if independent_unit is not None and independent_unit in run.frame.columns: + group_cols.append(independent_unit) + columns = list(dict.fromkeys([*group_cols, coefficient, *unit_constant])) + design = ( + run.frame.loc[:, columns] + .drop_duplicates(subset=group_cols) + .reset_index(drop=True) + ) + record["designRows"] = int(len(design)) + + report = report_confounding( + design, + coefficient=coefficient, + technicalColumns=unit_constant, + columnKinds={ + coefficient: run.kinds[coefficient], + **{name: run.kinds[name] for name in unit_constant}, + }, + associationFloor=_ASSOCIATION_FLOOR, + ) + report["observationUnit"] = observation_unit + report["independentUnit"] = independent_unit + run.actions.append(f"confounding:{coefficient}") + return record, report + + +def _characterize_coefficients( + run: _Run, + coefficients: Sequence[str], + *, + unit_directions: Mapping[str, Any], + design_columns: Sequence[str], + technical: Sequence[str], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + records: list[dict[str, Any]] = [] + reports: list[dict[str, Any]] = [] + for coefficient in coefficients: + observation, independent = _resolve_units( + run, + coefficient, + directed=unit_directions, + design_columns=design_columns, + unit_candidates=[*design_columns, *technical], + ) + record, report = _characterize_coefficient( + run, + coefficient, + observation_unit=observation, + independent_unit=independent, + technical=technical, + ) + records.append(record) + if report is not None: + reports.append(report) + return records, reports + + +def _column_records( + run: _Run, + candidates: Sequence[str], + *, + aliases: Mapping[str, list[str]], + dropped: Sequence[tuple[str, str]], +) -> list[dict[str, Any]]: + records = [ + { + "name": name, + "kind": run.kinds[name], + "domain": run.domains[name], + "summary": run.summary(name), + "aliases": list(aliases.get(name, [])), + } + for name in candidates + ] + records.extend( + { + "name": name, + "kind": "continuous", + "domain": "ignore", + "summary": f"dropped before triage ({_DROP_REASONS[reason]})", + "aliases": [], + } + for name, reason in dropped + ) + return records + + +def characterize_covariates( + store: Any, + *, + studyContext: str | None = None, + model: Any | None = None, + cellKey: str = "I", + directions: Mapping[str, Any] | None = None, +) -> CovariateCharacterization: + """Label cell covariates and record design-level confounding.""" + direction_map = dict(directions or {}) + available = set(store.cells.columns) + if cellKey not in available: + return CovariateCharacterization( + status="failed", + notes=[f"cellKey {cellKey!r} is not present in cell metadata"], + ) + errors = _validate_directions(direction_map, available) + if errors: + return CovariateCharacterization(status="failed", notes=errors) + + candidates, dropped = _triage_columns( + store, + cell_key=cellKey, + exclude=set(direction_map.get("excludeColumns") or []), + ) + frame = store.cells.to_pandas_dataframe([*candidates, cellKey], key=cellKey) + reviewed = len(candidates) + len(dropped) + candidates = [name for name in candidates if name in frame.columns] + candidates, aliases, alias_notes = _collapse_ontology_aliases(candidates, frame) + + run = _Run(frame=frame, context=_bounded_context(studyContext), model=model) + for name, reason in dropped: + run.note( + kind=reason, + detail=f"Dropped {_DROP_REASONS[reason]} {name}", + column=name, + ) + run.audit.extend(alias_notes) + + kind_directions = dict(direction_map.get("columnKinds") or {}) + domain_directions = dict(direction_map.get("columnDomains") or {}) + for name in candidates: + run.kinds[name] = kind_directions.get(name) or _infer_kind( + frame[name].to_numpy() + ) + for name in candidates: + run.domains[name] = _assign_domain(run, name, domain_directions) + candidates = _collapse_equivalent_columns(run, candidates, aliases) + + technical = [name for name in candidates if run.domains[name] == "technical"] + design_columns = [name for name in candidates if run.domains[name] == "design"] + records, reports = _characterize_coefficients( + run, + _select_coefficients( + run, + biological=[ + name for name in candidates if run.domains[name] == "biological" + ], + directed=set(direction_map.get("coefficientsOfInterest") or []), + ), + unit_directions=dict(direction_map.get("unitsOfInference") or {}), + design_columns=design_columns, + technical=technical, + ) + + categorical_technical = { + name: frame[name].to_numpy() + for name in technical + if run.kinds[name] == "categorical" + } + return CovariateCharacterization( + status="done", + auditLog=run.audit, + actions=run.actions, + notes=[ + f"Reviewed {reviewed} columns; {len(candidates)} triaged after " + "deterministic drops and ontology alias collapse" + ], + decisions=run.decisions, + columns=_column_records(run, candidates, aliases=aliases, dropped=dropped), + coefficients=records, + technicalNesting=( + report_technical_nesting(categorical_technical) + if len(categorical_technical) >= 2 + else [] + ), + confounding=reports, + ) diff --git a/scarf/metrics/__init__.py b/scarf/metrics/__init__.py index 7e4c84e3..8d004e29 100644 --- a/scarf/metrics/__init__.py +++ b/scarf/metrics/__init__.py @@ -3,6 +3,16 @@ """ from ._types import MatrixData, NeighborMetric, ZarrArray +from .association import ( + association_pair, + coefficient_estimability, + cramers_v, + directional_mapping, + eta_squared, + report_confounding, + report_technical_nesting, + spearman_rho, +) from .cluster_separability import ( ClusterSeparabilityResult, evaluate_cluster_separability, @@ -29,12 +39,17 @@ "MatrixData", "NeighborMetric", "ZarrArray", + "association_pair", "calculate_knn_cluster_similarity", "calculate_top_k_neighbor_distances", "calculate_weighted_cluster_similarity", "clisi_knn", + "coefficient_estimability", "compute_lisi", "compute_simpson", + "cramers_v", + "directional_mapping", + "eta_squared", "evaluate_cluster_separability", "graph_connectivity", "ilisi_knn", @@ -42,5 +57,8 @@ "label_concordance_score", "lisi_batch_mixing_score", "process_cluster", + "report_confounding", + "report_technical_nesting", "silhouette_scoring", + "spearman_rho", ] diff --git a/scarf/metrics/association.py b/scarf/metrics/association.py new file mode 100644 index 00000000..6f8825cf --- /dev/null +++ b/scarf/metrics/association.py @@ -0,0 +1,429 @@ +"""Association and estimability helpers for covariate characterization.""" + +from collections.abc import Mapping, Sequence +from typing import Any, Literal, cast + +import numpy as np +import pandas as pd +from scipy import stats + +__all__ = [ + "association_pair", + "coefficient_estimability", + "cramers_v", + "directional_mapping", + "eta_squared", + "report_confounding", + "report_technical_nesting", + "spearman_rho", +] + +ColumnKind = Literal["categorical", "continuous"] + + +def _as_1d(values: Any) -> np.ndarray: + array = np.asarray(values) + if array.ndim != 1: + raise ValueError("values must be one-dimensional") + return array + + +def _to_float(values: np.ndarray) -> np.ndarray | None: + """Return values as float, or None when the column is not numeric.""" + try: + return np.asarray(values, dtype=float) + except (TypeError, ValueError): + return None + + +def _pairwise_mask(left: np.ndarray, right: np.ndarray) -> np.ndarray: + if len(left) != len(right): + raise ValueError("paired arrays must have the same length") + left_ok = np.asarray(~pd.isna(left), dtype=bool) + right_ok = np.asarray(~pd.isna(right), dtype=bool) + return cast(np.ndarray, left_ok & right_ok) + + +def _not_computed( + reason: str, + *, + rowsUsed: int = 0, + rowsMissing: int = 0, +) -> dict[str, Any]: + return { + "status": "notComputed", + "reason": reason, + "rowsUsed": int(rowsUsed), + "rowsMissing": int(rowsMissing), + } + + +def directional_mapping( + left: Any, + right: Any, +) -> dict[str, Any]: + """Report exact directional mapping between two categorical columns.""" + left_values = _as_1d(left) + right_values = _as_1d(right) + mask = _pairwise_mask(left_values, right_values) + rows_used = int(mask.sum()) + rows_missing = int((~mask).sum()) + blank = { + "leftMapsToRight": False, + "rightMapsToLeft": False, + "nesting": "none", + "rowsUsed": rows_used, + "rowsMissing": rows_missing, + } + if rows_used == 0: + return blank + frame = pd.DataFrame({"left": left_values[mask], "right": right_values[mask]}) + n_left = int(frame["left"].nunique(dropna=False)) + n_right = int(frame["right"].nunique(dropna=False)) + # A constant column is nested inside everything, which is true but useless. + if n_left < 2 or n_right < 2: + return {**blank, "reason": "constantColumn"} + left_maps = bool(frame.groupby("left", dropna=False)["right"].nunique().le(1).all()) + right_maps = bool( + frame.groupby("right", dropna=False)["left"].nunique().le(1).all() + ) + if left_maps and right_maps: + nesting = "equivalent" + elif left_maps: + nesting = "leftInRight" + elif right_maps: + nesting = "rightInLeft" + else: + nesting = "none" + return { + "leftMapsToRight": left_maps, + "rightMapsToLeft": right_maps, + "nesting": nesting, + "rowsUsed": rows_used, + "rowsMissing": rows_missing, + } + + +def cramers_v(left: Any, right: Any) -> dict[str, Any]: + """Bias-corrected Cramér's V for two categorical columns. + + ``value`` uses the Bergsma correction, which subtracts the expected + chi-square under independence. On the small design tables this module + targets, that correction can shrink even a perfect association to zero, so + ``valueUncorrected`` and ``directionalMapping`` are reported alongside it. + """ + left_values = _as_1d(left) + right_values = _as_1d(right) + mask = _pairwise_mask(left_values, right_values) + rows_used = int(mask.sum()) + rows_missing = int((~mask).sum()) + if rows_used < 2: + return _not_computed( + "insufficientRows", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + contingency = pd.crosstab(left_values[mask], right_values[mask]) + if contingency.shape[0] < 2 or contingency.shape[1] < 2: + return _not_computed( + "constantOrSingleLevel", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + chi2 = float(stats.chi2_contingency(contingency.to_numpy(), correction=False)[0]) + n = float(rows_used) + r, k = contingency.shape + phi2 = chi2 / n + phi2_corr = max(0.0, phi2 - (r - 1) * (k - 1) / (n - 1)) + r_corr = r - (r - 1) ** 2 / (n - 1) + k_corr = k - (k - 1) ** 2 / (n - 1) + denominator = min(r_corr - 1.0, k_corr - 1.0) + mapping = directional_mapping(left_values[mask], right_values[mask]) + common = { + "rowsUsed": rows_used, + "rowsMissing": rows_missing, + "nLevelsLeft": int(r), + "nLevelsRight": int(k), + "valueUncorrected": float(np.sqrt(min(phi2 / (min(r, k) - 1), 1.0))), + "directionalMapping": mapping, + } + # One level per row leaves the correction no residual degrees of freedom. + if denominator <= 0: + return {"status": "notComputed", "reason": "degenerateCorrection", **common} + return { + "status": "ok", + "measure": "cramersV", + "value": float(np.sqrt(phi2_corr / denominator)), + **common, + } + + +def eta_squared(continuous: Any, categorical: Any) -> dict[str, Any]: + """Correlation ratio η² for continuous values grouped by a categorical column.""" + raw = _as_1d(continuous) + groups = _as_1d(categorical) + values = _to_float(raw) + if values is None: + return _not_computed("nonNumeric", rowsMissing=len(raw)) + mask = _pairwise_mask(values, groups) & np.isfinite(values) + rows_used = int(mask.sum()) + rows_missing = int(len(values) - rows_used) + if rows_used < 2: + return _not_computed( + "insufficientRows", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + frame = pd.DataFrame({"y": values[mask], "g": groups[mask]}) + n_levels = int(frame["g"].nunique(dropna=False)) + if n_levels < 2: + return _not_computed( + "constantOrSingleLevel", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + grand_mean = float(frame["y"].mean()) + ss_total = float(((frame["y"] - grand_mean) ** 2).sum()) + if ss_total <= 0: + return _not_computed( + "zeroVariance", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + group_means = frame.groupby("g", dropna=False)["y"].transform("mean") + ss_between = float(((group_means - grand_mean) ** 2).sum()) + return { + "status": "ok", + "measure": "etaSquared", + "value": float(ss_between / ss_total), + "rowsUsed": rows_used, + "rowsMissing": rows_missing, + "nLevels": n_levels, + # One observation per level forces η² to 1 regardless of any real effect. + "saturated": bool(n_levels >= rows_used), + } + + +def spearman_rho(left: Any, right: Any) -> dict[str, Any]: + """Spearman correlation for two continuous columns.""" + raw_left = _as_1d(left) + raw_right = _as_1d(right) + left_values = _to_float(raw_left) + right_values = _to_float(raw_right) + if left_values is None or right_values is None: + return _not_computed("nonNumeric", rowsMissing=len(raw_left)) + mask = ( + _pairwise_mask(left_values, right_values) + & np.isfinite(left_values) + & np.isfinite(right_values) + ) + rows_used = int(mask.sum()) + rows_missing = int(len(left_values) - rows_used) + if rows_used < 3: + return _not_computed( + "insufficientRows", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + x = left_values[mask] + y = right_values[mask] + if np.unique(x).size < 2 or np.unique(y).size < 2: + return _not_computed( + "zeroVariance", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + rho = float(stats.spearmanr(x, y).statistic) + if not np.isfinite(rho): + return _not_computed( + "undefined", + rowsUsed=rows_used, + rowsMissing=rows_missing, + ) + return { + "status": "ok", + "measure": "spearmanRho", + "value": rho, + "rowsUsed": rows_used, + "rowsMissing": rows_missing, + "tiedLeft": bool(np.unique(x).size < rows_used), + "tiedRight": bool(np.unique(y).size < rows_used), + } + + +def association_pair( + left: Any, + right: Any, + *, + leftKind: ColumnKind, + rightKind: ColumnKind, +) -> dict[str, Any]: + """Dispatch the MVP association measure for a typed column pair.""" + if leftKind == "categorical" and rightKind == "categorical": + return cramers_v(left, right) + if leftKind == "continuous" and rightKind == "categorical": + return eta_squared(left, right) + if leftKind == "categorical" and rightKind == "continuous": + return eta_squared(right, left) + return spearman_rho(left, right) + + +def report_technical_nesting( + columns: Mapping[str, Any], +) -> list[dict[str, Any]]: + """Directional nesting among categorical technical columns.""" + names = list(columns) + reports: list[dict[str, Any]] = [] + for index, left_name in enumerate(names): + for right_name in names[index + 1 :]: + mapping = directional_mapping(columns[left_name], columns[right_name]) + if mapping["nesting"] == "none": + continue + reports.append( + { + "left": left_name, + "right": right_name, + "nesting": mapping["nesting"], + "directionalMapping": mapping, + } + ) + return reports + + +def _one_hot(values: np.ndarray) -> np.ndarray: + """Treatment-coded indicators with the first level dropped.""" + dummies = pd.get_dummies(pd.Series(values), dummy_na=False, dtype=float) + if dummies.shape[1] > 1: + dummies = dummies.iloc[:, 1:] + elif dummies.shape[1] == 1: + return np.zeros((len(values), 0), dtype=float) + return cast(np.ndarray, dummies.to_numpy(dtype=float)) + + +def coefficient_estimability( + coefficient: Any, + *, + coefficientKind: ColumnKind, + technicals: Mapping[str, Any], + technicalKinds: Mapping[str, ColumnKind], +) -> dict[str, Any]: + """Minimal model-matrix rank check for a coefficient among technical columns. + + The matrix carries an explicit intercept. Without it, treatment-coded + indicators of two perfectly complementary factors look linearly + independent and a fully aliased coefficient is reported as estimable. + """ + coeff = _as_1d(coefficient) + n = len(coeff) + if n == 0: + return _not_computed("emptyInput") + + mask = np.asarray(~pd.isna(coeff), dtype=bool) + coeff_float: np.ndarray | None = None + if coefficientKind == "continuous": + coeff_float = _to_float(coeff) + if coeff_float is None: + return _not_computed("nonNumeric", rowsMissing=n) + mask &= np.isfinite(coeff_float) + for values in technicals.values(): + mask &= np.asarray(~pd.isna(_as_1d(values)), dtype=bool) + rows_used = int(mask.sum()) + if rows_used < 2: + return _not_computed( + "insufficientRows", + rowsUsed=rows_used, + rowsMissing=n - rows_used, + ) + + blocks: list[np.ndarray] = [np.ones((rows_used, 1), dtype=float)] + for name, values in technicals.items(): + subset = _as_1d(values)[mask] + if technicalKinds[name] == "continuous": + column = _to_float(subset) + if column is None or float(np.nanstd(column)) == 0.0: + continue + blocks.append(column.reshape(-1, 1)) + continue + indicators = _one_hot(subset) + if indicators.shape[1]: + blocks.append(indicators) + technical_matrix = np.concatenate(blocks, axis=1) + + if coefficientKind == "continuous": + assert coeff_float is not None + coeff_block = coeff_float[mask].reshape(-1, 1) + else: + coeff_block = _one_hot(coeff[mask]) + if coeff_block.shape[1] == 0: + return _not_computed("constantCoefficient", rowsUsed=rows_used) + + encoded = int(technical_matrix.shape[1] + coeff_block.shape[1]) + if encoded >= rows_used: + return { + "status": "notComputed", + "reason": "encodedColumnsReachRows", + "rowsUsed": rows_used, + "encodedColumns": encoded, + } + rank_technical = int(np.linalg.matrix_rank(technical_matrix)) + rank_full = int( + np.linalg.matrix_rank(np.concatenate([technical_matrix, coeff_block], axis=1)) + ) + estimable = rank_full > rank_technical + return { + "status": "ok", + "coefficientEstimable": estimable, + "rankDeficient": not estimable, + "rankTechnical": rank_technical, + "rankWithCoefficient": rank_full, + "rowsUsed": rows_used, + "encodedColumns": encoded, + } + + +def report_confounding( + design: pd.DataFrame, + *, + coefficient: str, + technicalColumns: Sequence[str], + columnKinds: Mapping[str, ColumnKind], + associationFloor: float = 0.1, +) -> dict[str, Any]: + """Association of one coefficient with technical columns on a design table.""" + if coefficient not in design.columns: + raise KeyError(f"coefficient column {coefficient!r} missing from design") + missing = [name for name in technicalColumns if name not in design.columns] + if missing: + raise KeyError(f"technical columns missing from design: {missing}") + coeff_kind = columnKinds[coefficient] + pairs: list[dict[str, Any]] = [] + associated: list[str] = [] + for name in technicalColumns: + result = association_pair( + design[coefficient].to_numpy(), + design[name].to_numpy(), + leftKind=coeff_kind, + rightKind=columnKinds[name], + ) + mapping = result.get("directionalMapping") or {} + # Bias-corrected V collapses toward zero on small design tables, so a + # deterministic mapping selects the pair even when the effect size does not. + selected = ( + result.get("status") == "ok" + and abs(float(result["value"])) >= associationFloor + ) or mapping.get("nesting", "none") != "none" + pairs.append({"technical": name, "association": result, "selected": selected}) + if selected: + associated.append(name) + + return { + "coefficient": coefficient, + "nRows": int(len(design)), + "pairs": pairs, + "estimability": coefficient_estimability( + design[coefficient].to_numpy(), + coefficientKind=coeff_kind, + technicals={name: design[name].to_numpy() for name in associated}, + technicalKinds={name: columnKinds[name] for name in associated}, + ), + } diff --git a/tests/test_agent_characterize_covariates.py b/tests/test_agent_characterize_covariates.py new file mode 100644 index 00000000..cae06e09 --- /dev/null +++ b/tests/test_agent_characterize_covariates.py @@ -0,0 +1,333 @@ +"""Tests for characterize_covariates.""" + +from collections.abc import Mapping +from pathlib import Path + +import numpy as np +from pydantic_ai.messages import ModelMessage, ModelResponse, ToolCallPart +from pydantic_ai.models.function import AgentInfo, FunctionModel +from scipy.sparse import csr_matrix + +from scarf.agent import CovariateCharacterization, characterize_covariates +from scarf.agent.characterize_covariates import _is_embedding_column +from scarf.agent.types import Decision +from scarf.datastore.datastore import DataStore +from scarf.writers import SparseToZarr + + +def _function_model(answers: Mapping[str, Decision]) -> FunctionModel: + queue = list(answers.items()) + + def reply(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + text = "" + for message in messages: + for part in getattr(message, "parts", []): + content = getattr(part, "content", None) + if isinstance(content, str): + text += content + selected: Decision | None = None + for key, decision in queue: + if key in text: + selected = decision + break + if selected is None: + selected = next(iter(answers.values())) + tool = info.output_tools[0] + return ModelResponse( + parts=[ + ToolCallPart( + tool_name=tool.name, + args=selected.model_dump(), + ) + ] + ) + + return FunctionModel(reply) + + +def _store_with_design(tmp_path: Path) -> DataStore: + n_cells = 12 + matrix = csr_matrix(np.ones((n_cells, 3), dtype=np.uint16)) + location = tmp_path / "covariates.zarr" + writer = SparseToZarr( + matrix, + str(location), + cell_ids=[f"cell-{i}" for i in range(n_cells)], + feature_ids=["g1", "g2", "g3"], + mem_budget=64 * 1024 * 1024, + nthreads=1, + ) + writer.dump() + store = DataStore( + str(location), + default_assay="RNA", + min_features_per_cell=0, + min_cells_per_feature=0, + nthreads=1, + mem_budget=64 * 1024 * 1024, + ) + # Two donors, two samples each, disease confounded with batch, cell type within sample. + donor = np.array(["d1"] * 6 + ["d2"] * 6) + sample = np.array(["s1"] * 3 + ["s2"] * 3 + ["s3"] * 3 + ["s4"] * 3) + batch = np.array(["b1"] * 6 + ["b2"] * 6) + disease = np.array(["case"] * 6 + ["ctrl"] * 6) + disease_ontology = np.array(["DOID:1"] * 6 + ["DOID:2"] * 6) + cell_type = np.array(["alpha", "beta", "alpha", "beta", "alpha", "beta"] * 2) + # Same partition as cell_type under different labels. + author_cell_type = np.array(["AC", "BC", "AC", "BC", "AC", "BC"] * 2) + # Constant within sample: valid design-table continuous technical. + depth = np.array( + [1000.0] * 3 + [2000.0] * 3 + [3500.5] * 3 + [4800.25] * 3 + ) + # Varies within sample: must be excluded from the design table. + umi_noise = np.linspace(100.0, 500.0, n_cells) + store.cells.insert("donor", donor, overwrite=True) + store.cells.insert("sample", sample, overwrite=True) + store.cells.insert("batch", batch, overwrite=True) + store.cells.insert("disease", disease, overwrite=True) + store.cells.insert("disease_ontology_term_id", disease_ontology, overwrite=True) + store.cells.insert("cell_type", cell_type, overwrite=True) + store.cells.insert("author_cell_type", author_cell_type, overwrite=True) + store.cells.insert("sequencing_depth", depth, overwrite=True) + store.cells.insert("umi_noise", umi_noise, overwrite=True) + store.cells.insert("X_umap1", np.linspace(0, 1, n_cells), overwrite=True) + store.cells.insert("X_umap2", np.linspace(1, 0, n_cells), overwrite=True) + store.cells.insert( + "X_pca-1", np.random.default_rng(0).normal(size=n_cells), overwrite=True + ) + return store + + +def test_embedding_column_name_patterns() -> None: + assert _is_embedding_column("X_umap1") + assert _is_embedding_column("X_umap_1") + assert _is_embedding_column("X_umap-1") + assert _is_embedding_column("X_pca12") + assert _is_embedding_column("scVI_3") + assert _is_embedding_column("PHATE1") + assert _is_embedding_column("FA_2") + assert _is_embedding_column("diffmap1") + assert not _is_embedding_column("donor") + assert not _is_embedding_column("FA") + assert not _is_embedding_column("batch") + + +def test_characterize_covariates_directions_only(tmp_path: Path) -> None: + store = _store_with_design(tmp_path) + result = characterize_covariates( + store, + studyContext="Case/control retina study across two donors.", + model=None, + directions={ + "columnDomains": { + "donor": "design", + "sample": "design", + "batch": "technical", + "sequencing_depth": "technical", + "umi_noise": "technical", + "disease": "biological", + "cell_type": "biological", + "author_cell_type": "biological", + }, + "coefficientsOfInterest": ["disease", "cell_type"], + "unitsOfInference": { + "disease": {"observationUnit": "sample", "independentUnit": "donor"}, + "cell_type": {"observationUnit": "sample"}, + }, + }, + ) + assert isinstance(result, CovariateCharacterization) + assert result.status == "done" + names = {column["name"]: column for column in result.columns} + assert names["disease"]["aliases"] == ["disease_ontology_term_id"] + assert "disease_ontology_term_id" not in { + column["name"] for column in result.columns if column["domain"] != "ignore" + } + assert names["X_umap1"]["domain"] == "ignore" + assert names["X_pca-1"]["domain"] == "ignore" + assert names["batch"]["kind"] == "categorical" + assert names["sequencing_depth"]["kind"] == "continuous" + + drops = { + entry["column"]: entry["kind"] + for entry in result.auditLog + if entry["kind"].startswith("drop") + } + assert drops["X_umap1"] == "dropEmbedding" + assert any(kind == "dropAssayStat" for kind in drops.values()) + + # donor, batch and disease partition the cells identically but sit in three + # different domains, so they are a confounding finding rather than aliases. + across = next( + entry for entry in result.auditLog if entry["kind"] == "equivalentAcrossDomains" + ) + assert set(across["columns"]) == {"donor", "batch", "disease"} + assert across["domains"] == ["biological", "design", "technical"] + # Same-domain equivalence cannot be resolved without a model, so it holds. + kept_apart = next( + entry for entry in result.auditLog if entry["kind"] == "equivalentKeptApart" + ) + assert set(kept_apart["columns"]) == {"cell_type", "author_cell_type"} + assert "author_cell_type" in {column["name"] for column in result.columns} + + scopes = {item["name"]: item["scope"] for item in result.coefficients} + assert scopes["disease"] == "betweenUnit" + assert scopes["cell_type"] == "withinUnit" + assert any(entry["kind"] == "withinUnit" for entry in result.auditLog) + assert len(result.confounding) == 1 + assert result.confounding[0]["coefficient"] == "disease" + measures = { + pair["technical"]: pair["association"].get("measure") + for pair in result.confounding[0]["pairs"] + } + assert measures == {"batch": "cramersV", "sequencing_depth": "etaSquared"} + varying = [ + entry + for entry in result.auditLog + if entry["kind"] == "technicalVariesWithinUnit" + ] + assert {entry["column"] for entry in varying} == {"umi_noise"} + assert all(entry["coefficient"] == "disease" for entry in varying) + + +def test_characterize_covariates_invalid_direction_fails(tmp_path: Path) -> None: + store = _store_with_design(tmp_path) + result = characterize_covariates( + store, + directions={"columnDomains": {"not_a_column": "biological"}}, + ) + assert result.status == "failed" + assert any("unknown columns" in note for note in result.notes) + + +def test_characterize_covariates_rejects_unsupported_domain_value( + tmp_path: Path, +) -> None: + store = _store_with_design(tmp_path) + result = characterize_covariates( + store, + directions={"columnDomains": {"batch": "nuisance"}}, + ) + assert result.status == "failed" + assert any("unsupported values" in note for note in result.notes) + + +def test_characterize_covariates_stays_headless_without_model(tmp_path: Path) -> None: + store = _store_with_design(tmp_path) + result = characterize_covariates(store) + assert result.status == "done" + assert result.decisions == [] + assert result.coefficients == [] + assert result.confounding == [] + unresolved = { + entry["column"] for entry in result.auditLog if entry["kind"] == "domainUnknown" + } + assert {"donor", "sample", "batch", "disease"} <= unresolved + + +def test_characterize_covariates_function_model_path(tmp_path: Path) -> None: + store = _store_with_design(tmp_path) + answers = { + "Assign a domain for cell metadata column donor": Decision( + selectedId="domain:design", + rationale="donor is sampling unit", + evidenceIds=["domain:design"], + ), + "Assign a domain for cell metadata column sample": Decision( + selectedId="domain:design", + rationale="sample is observation unit", + evidenceIds=["domain:design"], + ), + "Assign a domain for cell metadata column batch": Decision( + selectedId="domain:technical", + rationale="batch is technical", + evidenceIds=["domain:technical"], + ), + "Assign a domain for cell metadata column disease": Decision( + selectedId="domain:biological", + rationale="disease is biology", + evidenceIds=["domain:biological"], + ), + "Assign a domain for cell metadata column cell_type": Decision( + selectedId="domain:biological", + rationale="cell type is biology", + evidenceIds=["domain:biological"], + ), + "Assign a domain for cell metadata column author_cell_type": Decision( + selectedId="domain:biological", + rationale="author annotation is biology", + evidenceIds=["domain:biological"], + ), + "assign every cell to the same groups": Decision( + selectedId="equivalent:cell_type", + rationale="cell_type carries the readable labels", + evidenceIds=["equivalent:cell_type"], + ), + "Assign a domain for cell metadata column sequencing_depth": Decision( + selectedId="domain:technical", + rationale="depth is technical", + evidenceIds=["domain:technical"], + ), + "Assign a domain for cell metadata column umi_noise": Decision( + selectedId="domain:technical", + rationale="umi noise is technical", + evidenceIds=["domain:technical"], + ), + "Should biological column disease": Decision( + selectedId="coefficient:yes", + rationale="primary contrast", + evidenceIds=["coefficient:yes"], + ), + "Should biological column cell_type": Decision( + selectedId="coefficient:no", + rationale="composition only", + evidenceIds=["coefficient:no"], + ), + "Choose the observation unit for coefficient disease": Decision( + selectedId="unit:sample", + rationale="one row per sample", + evidenceIds=["unit:sample"], + ), + "Optional independent unit for coefficient disease": Decision( + selectedId="independentUnit:donor", + rationale="donor repeats", + evidenceIds=["independentUnit:donor"], + ), + } + result = characterize_covariates( + store, + studyContext="Case control across donors.", + model=_function_model(answers), + ) + assert result.status == "done" + assert any(decision["task"] == "columnDomain" for decision in result.decisions) + + columns = {column["name"]: column for column in result.columns} + assert columns["cell_type"]["aliases"] == ["author_cell_type"] + assert "author_cell_type" not in columns + collapsed = next( + entry for entry in result.auditLog if entry["kind"] == "equivalentColumns" + ) + assert collapsed["representative"] == "cell_type" + assert collapsed["levels"] == "AC = alpha; BC = beta" + + coeffs = {item["name"]: item for item in result.coefficients} + assert "disease" in coeffs + assert coeffs["disease"]["scope"] == "betweenUnit" + assert coeffs["disease"]["observationUnit"] == "sample" + assert coeffs["disease"]["independentUnit"] == "donor" + + +def test_characterize_covariates_does_not_mutate_store(tmp_path: Path) -> None: + store = _store_with_design(tmp_path) + before = list(store.cells.columns) + result = characterize_covariates( + store, + directions={ + "columnDomains": {"disease": "biological", "batch": "technical"}, + "coefficientsOfInterest": ["disease"], + "unitsOfInference": {"disease": {"observationUnit": "batch"}}, + }, + ) + assert result.status == "done" + assert set(store.cells.columns) == set(before) diff --git a/tests/test_metrics_association.py b/tests/test_metrics_association.py new file mode 100644 index 00000000..d1caa3cd --- /dev/null +++ b/tests/test_metrics_association.py @@ -0,0 +1,337 @@ +"""Tests for association and estimability helpers.""" + +import numpy as np +import pandas as pd +import pytest + +from scarf.metrics.association import ( + association_pair, + coefficient_estimability, + cramers_v, + directional_mapping, + eta_squared, + report_confounding, + report_technical_nesting, + spearman_rho, +) + + +def _categorical_table(counts: dict[tuple[str, str], int]) -> tuple[np.ndarray, ...]: + left: list[str] = [] + right: list[str] = [] + for (level_left, level_right), count in counts.items(): + left.extend([level_left] * count) + right.extend([level_right] * count) + return np.array(left), np.array(right) + + +def test_cramers_v_exact_mapping_is_one() -> None: + left = np.array(["a", "a", "b", "b", "c", "c"]) + right = np.array(["x", "x", "y", "y", "z", "z"]) + result = cramers_v(left, right) + assert result["status"] == "ok" + assert result["value"] == pytest.approx(1.0) + assert result["directionalMapping"]["nesting"] == "equivalent" + + +def test_cramers_v_matches_hand_computed_chi_square() -> None: + # 2x2 table [[30, 20], [20, 30]] has chi-square 4 at n=100, so V = 0.2. + left, right = _categorical_table( + {("a", "x"): 30, ("a", "y"): 20, ("b", "x"): 20, ("b", "y"): 30} + ) + result = cramers_v(left, right) + assert result["valueUncorrected"] == pytest.approx(0.2) + # The bias correction shrinks the estimate toward zero. + assert result["value"] < result["valueUncorrected"] + + +def test_cramers_v_is_degenerate_when_levels_reach_rows() -> None: + # Disease is perfectly determined by donor, but with one donor per row the + # bias correction has no residual degrees of freedom left to divide by. + disease = np.array(["case"] * 3 + ["ctrl"] * 3) + donor = np.array([f"d{index}" for index in range(6)]) + result = cramers_v(disease, donor) + assert result["status"] == "notComputed" + assert result["reason"] == "degenerateCorrection" + assert result["valueUncorrected"] == pytest.approx(1.0) + assert result["directionalMapping"]["nesting"] == "rightInLeft" + + +def test_cramers_v_constant_is_not_computed() -> None: + left = np.array(["a", "a", "a", "a"]) + right = np.array(["x", "y", "x", "y"]) + result = cramers_v(left, right) + assert result["status"] == "notComputed" + assert result["reason"] == "constantOrSingleLevel" + + +def test_directional_mapping_left_in_right() -> None: + left = np.array(["s1", "s1", "s2", "s2", "s3", "s3"]) + right = np.array(["b1", "b1", "b1", "b1", "b2", "b2"]) + mapping = directional_mapping(left, right) + assert mapping["leftMapsToRight"] is True + assert mapping["rightMapsToLeft"] is False + assert mapping["nesting"] == "leftInRight" + + +def test_directional_mapping_ignores_constant_column() -> None: + # Everything is trivially nested inside a constant, which is not a finding. + mapping = directional_mapping( + np.array(["a", "a", "a", "a"]), + np.array(["x", "y", "x", "y"]), + ) + assert mapping["nesting"] == "none" + assert mapping["reason"] == "constantColumn" + + +def test_directional_mapping_excludes_missing_rows() -> None: + left = np.array(["s1", "s1", "s2", "s2", "s3", None], dtype=object) + right = np.array(["b1", "b1", "b1", "b1", "b2", "b2"], dtype=object) + mapping = directional_mapping(left, right) + assert mapping["rowsUsed"] == 5 + assert mapping["rowsMissing"] == 1 + assert mapping["nesting"] == "leftInRight" + + +def test_eta_squared_matches_hand_computed_sums_of_squares() -> None: + # Grand mean 4, SS total 20, SS between 16. + result = eta_squared( + np.array([1.0, 3.0, 5.0, 7.0]), + np.array(["a", "a", "b", "b"]), + ) + assert result["status"] == "ok" + assert result["value"] == pytest.approx(0.8) + assert result["saturated"] is False + + +def test_eta_squared_flags_singleton_groups() -> None: + # One observation per level forces the ratio to 1 with no real effect. + result = eta_squared(np.array([1.0, 2.0, 3.0]), np.array(["a", "b", "c"])) + assert result["value"] == pytest.approx(1.0) + assert result["saturated"] is True + + +def test_eta_squared_rejects_non_numeric_values() -> None: + result = eta_squared(np.array(["low", "high", "low"]), np.array(["a", "b", "a"])) + assert result["status"] == "notComputed" + assert result["reason"] == "nonNumeric" + + +def test_spearman_rho_perfect_monotone() -> None: + left = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + right = np.array([2.0, 4.0, 6.0, 8.0, 10.0]) + result = spearman_rho(left, right) + assert result["status"] == "ok" + assert result["value"] == pytest.approx(1.0) + assert result["tiedLeft"] is False + + +def test_spearman_rho_reports_ties_and_missing() -> None: + left = np.array([1.0, 2.0, 2.0, 3.0, np.nan]) + right = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + result = spearman_rho(left, right) + assert result["rowsUsed"] == 4 + assert result["rowsMissing"] == 1 + assert result["tiedLeft"] is True + assert result["tiedRight"] is False + + +def test_spearman_rho_rejects_non_numeric_values() -> None: + result = spearman_rho(np.array(["a", "b", "c"]), np.array([1.0, 2.0, 3.0])) + assert result["status"] == "notComputed" + assert result["reason"] == "nonNumeric" + + +def test_association_pair_dispatches_kinds() -> None: + categorical = np.array(["a", "a", "b", "b"]) + continuous = np.array([1.0, 1.2, 8.0, 8.5]) + assert ( + association_pair( + continuous, + categorical, + leftKind="continuous", + rightKind="categorical", + )["measure"] + == "etaSquared" + ) + assert ( + association_pair( + categorical, + continuous, + leftKind="categorical", + rightKind="continuous", + )["measure"] + == "etaSquared" + ) + assert ( + association_pair( + continuous, + continuous, + leftKind="continuous", + rightKind="continuous", + )["measure"] + == "spearmanRho" + ) + + +def test_report_technical_nesting_finds_nested_batch() -> None: + columns = { + "sample": np.array(["s1", "s1", "s2", "s2", "s3", "s3"]), + "batch": np.array(["b1", "b1", "b1", "b1", "b2", "b2"]), + "chemistry": np.array(["v3"] * 6), + } + reports = report_technical_nesting(columns) + assert len(reports) == 1 + assert (reports[0]["left"], reports[0]["right"]) == ("sample", "batch") + assert reports[0]["nesting"] == "leftInRight" + + +def test_coefficient_estimability_detects_alias() -> None: + # Coefficient equals technical batch: not estimable after technicals. + batch = np.array(["b1", "b1", "b2", "b2", "b3", "b3"]) + disease = np.array(["d1", "d1", "d2", "d2", "d3", "d3"]) + result = coefficient_estimability( + disease, + coefficientKind="categorical", + technicals={"batch": batch}, + technicalKinds={"batch": "categorical"}, + ) + assert result["status"] == "ok" + assert result["coefficientEstimable"] is False + assert result["rankDeficient"] is True + + +def test_coefficient_estimability_detects_complementary_alias() -> None: + # Level order is reversed between the two factors. Without an intercept in + # the model matrix the indicators look independent and this alias is missed. + batch = np.array(["b1", "b1", "b2", "b2"]) + disease = np.array(["ctrl", "ctrl", "case", "case"]) + result = coefficient_estimability( + disease, + coefficientKind="categorical", + technicals={"batch": batch}, + technicalKinds={"batch": "categorical"}, + ) + assert result["coefficientEstimable"] is False + assert result["rankTechnical"] == result["rankWithCoefficient"] == 2 + + +def test_coefficient_estimability_keeps_crossed_factor() -> None: + batch = np.array(["b1", "b1", "b2", "b2"]) + disease = np.array(["case", "ctrl", "case", "ctrl"]) + result = coefficient_estimability( + disease, + coefficientKind="categorical", + technicals={"batch": batch}, + technicalKinds={"batch": "categorical"}, + ) + assert result["coefficientEstimable"] is True + assert result["rankWithCoefficient"] == result["rankTechnical"] + 1 + + +def test_coefficient_estimability_detects_continuous_alias() -> None: + # A continuous coefficient that is a per-batch constant lies in the span of + # the intercept plus the batch indicator. + batch = np.array(["b1", "b1", "b2", "b2"]) + dose = np.array([1.0, 1.0, 2.0, 2.0]) + result = coefficient_estimability( + dose, + coefficientKind="continuous", + technicals={"batch": batch}, + technicalKinds={"batch": "categorical"}, + ) + assert result["coefficientEstimable"] is False + + +def test_coefficient_estimability_without_technicals() -> None: + result = coefficient_estimability( + np.array(["case", "case", "ctrl", "ctrl"]), + coefficientKind="categorical", + technicals={}, + technicalKinds={}, + ) + assert result["coefficientEstimable"] is True + + +def test_coefficient_estimability_gates_when_columns_reach_rows() -> None: + # Three technical factors with many levels relative to six rows. + rows = 6 + technicals = { + "t1": np.array([f"a{i}" for i in range(rows)]), + "t2": np.array([f"b{i}" for i in range(rows)]), + "t3": np.array([f"c{i}" for i in range(rows)]), + } + result = coefficient_estimability( + np.array(["x", "x", "y", "y", "z", "z"]), + coefficientKind="categorical", + technicals=technicals, + technicalKinds={name: "categorical" for name in technicals}, + ) + assert result["status"] == "notComputed" + assert result["reason"] == "encodedColumnsReachRows" + + +def test_report_confounding_on_design_table() -> None: + design = pd.DataFrame( + { + "disease": ["case", "case", "ctrl", "ctrl"], + "batch": ["b1", "b1", "b2", "b2"], + "chemistry": ["v2", "v3", "v2", "v3"], + } + ) + report = report_confounding( + design, + coefficient="disease", + technicalColumns=["batch", "chemistry"], + columnKinds={ + "disease": "categorical", + "batch": "categorical", + "chemistry": "categorical", + }, + associationFloor=0.1, + ) + assert report["coefficient"] == "disease" + assert report["nRows"] == 4 + selected = {pair["technical"]: pair["selected"] for pair in report["pairs"]} + assert selected == {"batch": True, "chemistry": False} + assert report["estimability"]["coefficientEstimable"] is False + + +def test_report_confounding_selects_deterministic_pair_without_effect_size() -> None: + # The corrected effect size is unavailable here, so selection has to fall + # back on the exact mapping or the aliasing would be reported as estimable. + design = pd.DataFrame( + { + "disease": ["case"] * 3 + ["ctrl"] * 3, + "donor": [f"d{index}" for index in range(6)], + } + ) + report = report_confounding( + design, + coefficient="disease", + technicalColumns=["donor"], + columnKinds={"disease": "categorical", "donor": "categorical"}, + ) + pair = report["pairs"][0] + assert pair["association"]["status"] == "notComputed" + assert pair["selected"] is True + assert report["estimability"]["status"] == "notComputed" + assert "coefficientEstimable" not in report["estimability"] + + +def test_report_confounding_requires_present_columns() -> None: + design = pd.DataFrame({"disease": ["case", "ctrl"]}) + with pytest.raises(KeyError): + report_confounding( + design, + coefficient="missing", + technicalColumns=[], + columnKinds={"missing": "categorical"}, + ) + with pytest.raises(KeyError): + report_confounding( + design, + coefficient="disease", + technicalColumns=["batch"], + columnKinds={"disease": "categorical", "batch": "categorical"}, + ) From 125b55b24009d788405d814a71c9454eab722f27 Mon Sep 17 00:00:00 2001 From: parashardhapola Date: Wed, 5 Aug 2026 15:19:15 +0200 Subject: [PATCH 4/5] Add scarf.agent Phase 2b feature identity and harden covariate units. Introduce Ensembl-backed species/family observation with catalogSuspect warnings, and drop constant columns plus invalid finer independent units so design confounding stays general without clustering metadata. Also fix UTF-8 cell metadata, biotype assay-split gating, and decide() coercion for live H5AD runs. --- scarf/agent/__init__.py | 6 + scarf/agent/characterize_covariates.py | 227 +++++++- scarf/agent/characterize_features.py | 537 ++++++++++++++++++ scarf/agent/decide.py | 36 +- scarf/features/gene_reference.py | 374 +++++++++++++ scarf/features/identity.py | 575 ++++++++++++++++++++ scarf/quality_control/__init__.py | 9 +- scarf/quality_control/cell_cycle_genes.py | 18 +- scarf/readers/_h5ad_inspect.py | 35 +- scarf/storage/arrays.py | 43 +- tests/test_agent_characterize_covariates.py | 66 ++- tests/test_agent_characterize_features.py | 193 +++++++ tests/test_agent_decide.py | 11 + tests/test_features_gene_reference.py | 63 +++ tests/test_features_identity.py | 247 +++++++++ tests/test_readers.py | 38 +- tests/test_selections.py | 8 + 17 files changed, 2421 insertions(+), 65 deletions(-) create mode 100644 scarf/agent/characterize_features.py create mode 100644 scarf/features/gene_reference.py create mode 100644 scarf/features/identity.py create mode 100644 tests/test_agent_characterize_features.py create mode 100644 tests/test_features_gene_reference.py create mode 100644 tests/test_features_identity.py diff --git a/scarf/agent/__init__.py b/scarf/agent/__init__.py index a0ba1c03..8c5e3a5d 100644 --- a/scarf/agent/__init__.py +++ b/scarf/agent/__init__.py @@ -4,6 +4,10 @@ CovariateCharacterization, characterize_covariates, ) +from .characterize_features import ( + FeatureCharacterization, + characterize_features, +) from .decide import DecisionValidationError, decide from .ingest import IngestResult, detect_format, ingest from .runtime import check_runtime, load_env @@ -20,11 +24,13 @@ "Decision", "DecisionValidationError", "EvidenceItem", + "FeatureCharacterization", "IngestResult", "NeedsInput", "StageResult", "StageStatus", "characterize_covariates", + "characterize_features", "check_runtime", "decide", "detect_format", diff --git a/scarf/agent/characterize_covariates.py b/scarf/agent/characterize_covariates.py index 594f4afb..fc4bdf7b 100644 --- a/scarf/agent/characterize_covariates.py +++ b/scarf/agent/characterize_covariates.py @@ -8,10 +8,14 @@ import numpy as np import pandas as pd -from ..metrics.association import report_confounding, report_technical_nesting +from ..metrics.association import ( + directional_mapping, + report_confounding, + report_technical_nesting, +) from ..storage.types import as_zarr_array, as_zarr_group from ._deps import AGENT_INSTALL_HINT -from .decide import decide +from .decide import DecisionValidationError, decide from .types import Decision, EvidenceItem, StageStatus try: @@ -55,6 +59,7 @@ "dropAssayStat": "Scarf assay statistic column", "dropProvenance": "analysis-linked column", "dropEmbedding": "embedding-style column", + "dropConstant": "single-level column", } _DOMAIN_EVIDENCE = [ @@ -140,7 +145,16 @@ def ask( """Run one grounded decision, or return None when it cannot be asked.""" if self.model is None or len(evidence) < 2: return None - decision = decide(model=self.model, question=question, evidence=evidence) + try: + decision = decide(model=self.model, question=question, evidence=evidence) + except DecisionValidationError as exc: + self.note( + kind="decisionInvalid", + detail=str(exc), + task=task, + column=column, + ) + return None record: dict[str, Any] = { "task": task, "selectedId": decision.selectedId, @@ -528,6 +542,55 @@ def _unit_evidence(run: _Run, names: Sequence[str], prefix: str) -> list[Evidenc ] +def _coefficient_constant_within( + frame: pd.DataFrame, + coefficient: str, + unit: str, +) -> bool: + """True when the coefficient does not vary inside each unit level.""" + if coefficient not in frame.columns or unit not in frame.columns: + return False + return bool( + frame.groupby(unit, dropna=False)[coefficient].nunique(dropna=False).le(1).all() + ) + + +def _observation_unit_candidates( + run: _Run, + coefficient: str, + pool: Sequence[str], +) -> list[str]: + """Design/technical columns that can host a between-unit coefficient. + + A valid observation unit is a metadata fact, not a judgement: the coefficient + must be constant within each level. No clustering column is required; when + nothing in the pool works, the coefficient stays within-unit or unresolved. + """ + return [ + name + for name in pool + if name != coefficient + and _coefficient_constant_within(run.frame, coefficient, name) + ] + + +def _independent_is_coarser( + frame: pd.DataFrame, + *, + observation: str, + independent: str, +) -> bool: + """True when each observation level maps to one independent level. + + The independent unit must be coarser (or equal), never finer. A finer + independent unit inflates the design table with pseudo-replicated rows. + """ + if observation not in frame.columns or independent not in frame.columns: + return False + nesting = directional_mapping(frame[observation], frame[independent]).get("nesting") + return nesting in {"leftInRight", "equivalent"} + + def _resolve_units( run: _Run, coefficient: str, @@ -539,33 +602,107 @@ def _resolve_units( unit_map = dict(directed.get(coefficient) or {}) observation = unit_map.get("observationUnit") independent = unit_map.get("independentUnit") + valid_observation = _observation_unit_candidates(run, coefficient, unit_candidates) - if observation is None: + if observation is not None: + if observation not in run.frame.columns: + run.note( + kind="invalidObservationUnit", + detail=f"Directed observation unit {observation!r} is missing", + column=coefficient, + observationUnit=observation, + ) + observation = None + elif not _coefficient_constant_within(run.frame, coefficient, observation): + # Keep the directed unit; _characterize_coefficient records withinUnit. + pass + elif observation not in valid_observation: + valid_observation = [observation, *valid_observation] + elif len(valid_observation) == 1: + observation = valid_observation[0] + run.actions.append(f"observationUnit:{coefficient}->{observation}") + elif len(valid_observation) >= 2: decision = run.ask( task="observationUnit", column=coefficient, question=( f"Choose the observation unit for coefficient {coefficient}. " - "Each distinct value of this column is one design-table row. " + "Only columns where this coefficient is constant within each " + "level are listed. Each distinct value is one design-table row. " f"Study context: {run.context or 'none provided'}." ), - evidence=_unit_evidence( - run, - [name for name in unit_candidates if name != coefficient], - "unit", - ), + evidence=_unit_evidence(run, valid_observation, "unit"), ) if decision is not None: observation = decision.selectedId.removeprefix("unit:") - run.actions.append(f"observationUnit:{coefficient}->{observation}") + if observation not in valid_observation: + run.note( + kind="invalidObservationUnit", + detail=( + f"Model chose {observation!r}, which is not a valid " + f"observation unit for {coefficient}" + ), + column=coefficient, + observationUnit=observation, + ) + observation = None + else: + run.actions.append(f"observationUnit:{coefficient}->{observation}") + else: + run.note( + kind="noValidObservationUnit", + detail=( + f"No design/technical column keeps {coefficient} constant; " + "cannot build a between-unit design table from available metadata" + ), + column=coefficient, + ) + + if observation is None: + return None, None + + valid_independent = [ + name + for name in design_columns + if name not in {coefficient, observation} + and _independent_is_coarser( + run.frame, observation=observation, independent=name + ) + ] - if observation is not None and independent is None: + if independent is not None: + if independent not in run.frame.columns: + run.note( + kind="invalidIndependentUnit", + detail=f"Directed independent unit {independent!r} is missing", + column=coefficient, + independentUnit=independent, + ) + independent = None + elif not _independent_is_coarser( + run.frame, observation=observation, independent=independent + ): + run.note( + kind="independentUnitFiner", + detail=( + f"Independent unit {independent!r} is finer than observation " + f"unit {observation!r}; dropped to avoid pseudo-replicated " + "design rows" + ), + column=coefficient, + observationUnit=observation, + independentUnit=independent, + ) + independent = None + elif valid_independent: decision = run.ask( task="independentUnit", column=coefficient, question=( f"Optional independent unit for coefficient {coefficient} " - f"with observation unit {observation}." + f"with observation unit {observation}. Listed columns are " + "coarser than the observation unit (each observation level " + "maps to one independent level)." ), evidence=[ EvidenceItem( @@ -573,20 +710,24 @@ def _resolve_units( label="none", summary="No separate independent unit or subject column", ), - *_unit_evidence( - run, - [ - name - for name in design_columns - if name not in {coefficient, observation} - ], - "independentUnit", - ), + *_unit_evidence(run, valid_independent, "independentUnit"), ], ) if decision is not None and decision.selectedId != "independentUnit:none": - independent = decision.selectedId.removeprefix("independentUnit:") - run.actions.append(f"independentUnit:{coefficient}->{independent}") + chosen = decision.selectedId.removeprefix("independentUnit:") + if chosen not in valid_independent: + run.note( + kind="invalidIndependentUnit", + detail=( + f"Model chose {chosen!r}, which is not coarser than " + f"observation unit {observation!r}" + ), + column=coefficient, + independentUnit=chosen, + ) + else: + independent = chosen + run.actions.append(f"independentUnit:{coefficient}->{independent}") return observation, independent @@ -656,7 +797,26 @@ def _characterize_coefficient( group_cols = [observation_unit] if independent_unit is not None and independent_unit in run.frame.columns: - group_cols.append(independent_unit) + if _independent_is_coarser( + run.frame, + observation=observation_unit, + independent=independent_unit, + ): + group_cols.append(independent_unit) + else: + run.note( + kind="independentUnitFiner", + detail=( + f"Independent unit {independent_unit!r} is finer than " + f"observation unit {observation_unit!r}; omitted from " + "design table" + ), + column=coefficient, + observationUnit=observation_unit, + independentUnit=independent_unit, + ) + independent_unit = None + record["independentUnit"] = None columns = list(dict.fromkeys([*group_cols, coefficient, *unit_constant])) design = ( run.frame.loc[:, columns] @@ -770,13 +930,28 @@ def characterize_covariates( frame = store.cells.to_pandas_dataframe([*candidates, cellKey], key=cellKey) reviewed = len(candidates) + len(dropped) candidates = [name for name in candidates if name in frame.columns] + # Single-level columns are not covariates. Leaving them in creates spurious + # "perfect confounding" among every pair of constants. + directed_coefficients = set(direction_map.get("coefficientsOfInterest") or []) + varying: list[str] = [] + for name in candidates: + if int(frame[name].nunique(dropna=False)) <= 1: + dropped.append((name, "dropConstant")) + continue + varying.append(name) + candidates = varying candidates, aliases, alias_notes = _collapse_ontology_aliases(candidates, frame) run = _Run(frame=frame, context=_bounded_context(studyContext), model=model) for name, reason in dropped: + detail = f"Dropped {_DROP_REASONS[reason]} {name}" + if reason == "dropConstant" and name in directed_coefficients: + detail = ( + f"{detail}; also listed in coefficientsOfInterest but has no variation" + ) run.note( kind=reason, - detail=f"Dropped {_DROP_REASONS[reason]} {name}", + detail=detail, column=name, ) run.audit.extend(alias_notes) diff --git a/scarf/agent/characterize_features.py b/scarf/agent/characterize_features.py new file mode 100644 index 00000000..74ddd9ea --- /dev/null +++ b/scarf/agent/characterize_features.py @@ -0,0 +1,537 @@ +"""Characterize feature identity, species, families, and exogenous candidates.""" + +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from ..assay import RNAassay +from ..features.gene_reference import ( + GeneReference, + default_cache_dir, + ensure_reference, + load_reference, + species_registry, +) +from ..features.identity import ( + audit_feature_identity, + backfill_symbols, + exogenous_candidates, + observe_families, + reference_misses, + resolve_species, +) +from ..quality_control.cell_cycle_genes import ( + g2m_phase_genes, + g2m_phase_genes_mouse, + s_phase_genes, + s_phase_genes_mouse, +) +from ._deps import AGENT_INSTALL_HINT +from .decide import decide +from .types import Decision, EvidenceItem, StageStatus + +try: + from pydantic import BaseModel, Field +except ImportError as exc: + raise ImportError(AGENT_INSTALL_HINT) from exc + +__all__ = [ + "FeatureCharacterization", + "characterize_features", +] + +_DEFAULT_MAX_EXOGENOUS = 25 +_CONTEXT_LIMIT = 1200 +_AUTO_DOWNLOAD_SPECIES = frozenset({"homo_sapiens", "mus_musculus"}) +_CELL_CYCLE = { + "homo_sapiens": {"s": s_phase_genes, "g2m": g2m_phase_genes}, + "mus_musculus": {"s": s_phase_genes_mouse, "g2m": g2m_phase_genes_mouse}, +} +_SEX_COEFFICIENT_TOKENS = frozenset({"sex", "gender", "Sex", "Gender"}) + + +class FeatureCharacterization(BaseModel): + status: StageStatus + auditLog: list[dict[str, Any]] = Field(default_factory=list) + actions: list[str] = Field(default_factory=list) + notes: list[str] = Field(default_factory=list) + decisions: list[dict[str, Any]] = Field(default_factory=list) + assays: list[dict[str, Any]] = Field(default_factory=list) + + +def _bounded_context(study_context: str | None) -> str: + text = (study_context or "").strip() + return text if len(text) <= _CONTEXT_LIMIT else text[: _CONTEXT_LIMIT - 3] + "..." + + +def _audit( + audit_log: list[dict[str, Any]], + *, + kind: str, + detail: str, + **fields: Any, +) -> None: + audit_log.append({"kind": kind, "detail": detail, **fields}) + + +def _ask( + *, + model: Any | None, + question: str, + evidence: Sequence[EvidenceItem], + decisions: list[dict[str, Any]], + task: str, + assay: str | None = None, +) -> Decision | None: + if model is None or len(evidence) < 2: + return None + decision = decide(model=model, question=question, evidence=evidence) + record: dict[str, Any] = { + "task": task, + "selectedId": decision.selectedId, + "rationale": decision.rationale, + "evidenceIds": list(decision.evidenceIds), + } + if assay is not None: + record["assay"] = assay + decisions.append(record) + return decision + + +def _sex_coefficient_note( + covariates: Any | None, +) -> str | None: + if covariates is None: + return None + coefficients = getattr(covariates, "coefficients", None) or [] + for item in coefficients: + name = item.get("name") if isinstance(item, Mapping) else None + if name in _SEX_COEFFICIENT_TOKENS: + return ( + f"Prior covariate characterization marked {name!r} as a coefficient " + "of interest; tracked sex-chromosome genes must not be excluded later" + ) + return None + + +def _load_or_fetch_reference( + species: str, + *, + cache_dir: Path, + allow_download: bool, + audit_log: list[dict[str, Any]], + assay: str, +) -> GeneReference | None: + if species == "unknown": + return None + cached = load_reference(species, cacheDir=cache_dir) + if cached is not None: + return cached + if not allow_download: + _audit( + audit_log, + kind="referenceUnavailable", + detail=f"No cached reference for {species}; download disabled", + assay=assay, + species=species, + ) + return None + try: + reference = ensure_reference(species, cacheDir=cache_dir) + except Exception as exc: + _audit( + audit_log, + kind="referenceDownloadFailed", + detail=f"Failed to download reference for {species}: {exc}", + assay=assay, + species=species, + ) + return None + _audit( + audit_log, + kind="referenceDownloaded", + detail=f"Cached gene reference for {species} release {reference.release}", + assay=assay, + species=species, + release=reference.release, + ) + return reference + + +def _assist_species( + *, + model: Any | None, + unresolved: Mapping[str, Any], + context: str, + decisions: list[dict[str, Any]], + assay: str, +) -> str | None: + # Only ask among species that already have overlap evidence. Expanding to the + # full registry would let a guess trigger a non-human/mouse download. + candidates = [ + key for key in (unresolved.get("candidates") or []) if key in species_registry() + ] + if len(candidates) < 2: + return None + evidence = [ + EvidenceItem( + id=f"species:{key}", + label=species_registry()[key].label, + summary=( + f"overlap hits=" + f"{(unresolved.get('overlap') or {}).get('scores', {}).get(key, {}).get('hits', 0)}; " + f"prefix count=" + f"{(unresolved.get('prefixCounts') or {}).get(key, 0)}" + ), + ) + for key in candidates + ] + evidence.append( + EvidenceItem( + id="species:unknown", + label="unknown", + summary="Cannot settle species from available evidence", + ) + ) + decision = _ask( + model=model, + question=( + f"Choose the species for assay {assay}. " + f"Identity notes: {unresolved.get('reason', '')}. " + f"Study context: {context or 'none provided'}." + ), + evidence=evidence, + decisions=decisions, + task="species", + assay=assay, + ) + if decision is None: + return None + selected = decision.selectedId.removeprefix("species:") + return selected if selected in species_registry() or selected == "unknown" else None + + +def _classify_exogenous( + *, + model: Any | None, + candidates: Sequence[Mapping[str, Any]], + context: str, + decisions: list[dict[str, Any]], + assay: str, + species: str, +) -> list[dict[str, Any]]: + classified: list[dict[str, Any]] = [] + for item in candidates: + label = item.get("name") or item.get("id") or "feature" + evidence = [ + EvidenceItem( + id="exogenous:potentialExogenous", + label="potentialExogenous", + summary="Spike-in, transgene, guide, antibody tag, or other non-endogenous feature", + ), + EvidenceItem( + id="exogenous:unresolved", + label="unresolved", + summary="Not enough evidence to treat as exogenous", + ), + ] + decision = _ask( + model=model, + question=( + f"Classify feature {label!r} (id={item.get('id')!r}) for assay {assay} " + f"under species {species}. Study context: {context or 'none provided'}." + ), + evidence=evidence, + decisions=decisions, + task="exogenous", + assay=assay, + ) + record = dict(item) + if decision is None: + record["class"] = "unresolved" + else: + record["class"] = decision.selectedId.removeprefix("exogenous:") + classified.append(record) + return classified + + +def _characterize_assay( + store: Any, + assay_name: str, + *, + model: Any | None, + context: str, + directions: Mapping[str, Any], + cache_dir: Path, + allow_download: bool, + audit_log: list[dict[str, Any]], + actions: list[str], + decisions: list[dict[str, Any]], + sex_note: str | None, +) -> dict[str, Any]: + assay = store.get_assay(assay_name) + ids = [str(value) for value in assay.feats.fetch_all("ids")] + names = [str(value) for value in assay.feats.fetch_all("names")] + identity = audit_feature_identity(ids, names) + record: dict[str, Any] = { + "assay": assay_name, + "assayKind": type(assay).__name__, + "identity": identity, + "species": "unknown", + "speciesMethod": None, + "families": [], + "exogenous": [], + "symbolBackfill": None, + } + + if not isinstance(assay, RNAassay): + record["skipped"] = "familyPlanningNotApplicable" + _audit( + audit_log, + kind="nonRnaAssay", + detail=f"Assay {assay_name} is not RNA; stopped after identity audit", + assay=assay_name, + ) + return record + + species_by_assay = dict(directions.get("speciesByAssay") or {}) + directed_species = species_by_assay.get(assay_name) + resolution = resolve_species( + ids, + names, + directed=directed_species, + cacheDir=cache_dir, + allowDownload=allow_download, + ) + species = resolution["species"] + if species == "unknown" and resolution.get("method") == "inconclusive": + assisted = _assist_species( + model=model, + unresolved=resolution, + context=context, + decisions=decisions, + assay=assay_name, + ) + if assisted is not None: + species = assisted + resolution = { + **resolution, + "species": species, + "method": "llmAssist", + "reason": "model choice among inconclusive candidates", + } + actions.append(f"species:{assay_name}->{species}") + + record["species"] = species + record["speciesMethod"] = resolution.get("method") + record["speciesResolution"] = { + key: value + for key, value in resolution.items() + if key not in {"overlap"} or value is not None + } + _audit( + audit_log, + kind="speciesResolved", + detail=resolution.get("reason", f"species={species}"), + assay=assay_name, + species=species, + method=resolution.get("method"), + ) + + if species == "unknown": + record["families"] = observe_families( + species="unknown", + ids=ids, + symbols=names, + reference=None, + ) + _audit( + audit_log, + kind="speciesUnknown", + detail=f"Skipped species-dependent steps for assay {assay_name}", + assay=assay_name, + ) + return record + + # Only human/mouse auto-download; any other species needs an explicit direction. + may_download = allow_download and ( + species in _AUTO_DOWNLOAD_SPECIES or directed_species == species + ) + reference = _load_or_fetch_reference( + species, + cache_dir=cache_dir, + allow_download=may_download, + audit_log=audit_log, + assay=assay_name, + ) + + symbols = list(names) + if reference is not None: + backfill = backfill_symbols(ids, names, reference) + if backfill["nRecovered"]: + symbols = backfill["symbols"] + record["symbolBackfill"] = { + "nRecovered": backfill["nRecovered"], + "joinRate": backfill["joinRate"], + } + actions.append( + f"symbolBackfill:{assay_name}:{backfill['nRecovered']}/{backfill['nFeatures']}" + ) + elif identity.get("idsEqualNames") or identity.get("nEmptyNames", 0) > 0: + _audit( + audit_log, + kind="familiesNotAssessable", + detail=( + f"Names are empty or ID-shaped on {assay_name} and no reference " + "is available to recover symbols" + ), + assay=assay_name, + ) + + record["families"] = observe_families( + species=species, + ids=ids, + symbols=symbols, + reference=reference, + cellCycleGenes=_CELL_CYCLE, + ) + if sex_note is not None: + record["notes"] = [sex_note] + _audit( + audit_log, + kind="sexCoefficientNote", + detail=sex_note, + assay=assay_name, + ) + elif species != "unknown": + _audit( + audit_log, + kind="sexChromosomeTracked", + detail=( + "Sex-chromosome genes are tracked with defaultExclude=false; " + "Phase 3 may exclude them when sex is not a coefficient of interest" + ), + assay=assay_name, + ) + + raw_max = directions.get("maxExogenousCandidates", _DEFAULT_MAX_EXOGENOUS) + try: + max_exogenous = int(raw_max) + except (TypeError, ValueError): + _audit( + audit_log, + kind="invalidDirection", + detail=f"maxExogenousCandidates={raw_max!r}; using {_DEFAULT_MAX_EXOGENOUS}", + assay=assay_name, + ) + max_exogenous = _DEFAULT_MAX_EXOGENOUS + if reference is not None: + misses = reference_misses(ids, symbols, reference) + if misses["count"]: + record["referenceMisses"] = misses + _audit( + audit_log, + kind="referenceMiss", + detail=( + f"{misses['count']} Ensembl-shaped id(s) absent from the " + f"{species} reference (release drift, not exogenous)" + ), + assay=assay_name, + count=misses["count"], + examples=misses["examples"], + ) + candidates = exogenous_candidates( + ids, + symbols, + reference=reference, + maxCandidates=max_exogenous, + ) + if reference is None and not candidates: + _audit( + audit_log, + kind="exogenousUnresolved", + detail=( + f"No reference and no structural exogenous candidates for {assay_name}" + ), + assay=assay_name, + ) + record["exogenous"] = _classify_exogenous( + model=model, + candidates=candidates, + context=context, + decisions=decisions, + assay=assay_name, + species=species, + ) + actions.append(f"families:{assay_name}") + return record + + +def characterize_features( + store: Any, + *, + studyContext: str | None = None, + model: Any | None = None, + assays: Sequence[str] | None = None, + directions: Mapping[str, Any] | None = None, + covariates: Any | None = None, + cacheDir: Path | str | None = None, + allowDownload: bool = False, +) -> FeatureCharacterization: + """Label feature identity, species, families, and exogenous candidates.""" + direction_map = dict(directions or {}) + audit_log: list[dict[str, Any]] = [] + actions: list[str] = [] + decisions: list[dict[str, Any]] = [] + notes: list[str] = [] + context = _bounded_context(studyContext) + cache_dir = Path(cacheDir) if cacheDir is not None else default_cache_dir() + + available = list(store.assay_names) + selected = list(assays) if assays is not None else available + unknown = sorted(set(selected) - set(available)) + if unknown: + return FeatureCharacterization( + status="failed", + notes=[f"unknown assays: {unknown}"], + ) + species_by_assay = direction_map.get("speciesByAssay") + if species_by_assay is not None: + if not isinstance(species_by_assay, Mapping): + return FeatureCharacterization( + status="failed", + notes=["speciesByAssay must be a mapping"], + ) + bad = sorted(set(species_by_assay) - set(available)) + if bad: + return FeatureCharacterization( + status="failed", + notes=[f"speciesByAssay cites unknown assays: {bad}"], + ) + + sex_note = _sex_coefficient_note(covariates) + assay_records = [ + _characterize_assay( + store, + assay_name, + model=model, + context=context, + directions=direction_map, + cache_dir=cache_dir, + allow_download=allowDownload, + audit_log=audit_log, + actions=actions, + decisions=decisions, + sex_note=sex_note, + ) + for assay_name in selected + ] + notes.append(f"Characterized {len(assay_records)} assay(s)") + return FeatureCharacterization( + status="done", + auditLog=audit_log, + actions=actions, + notes=notes, + decisions=decisions, + assays=assay_records, + ) diff --git a/scarf/agent/decide.py b/scarf/agent/decide.py index f5c6af67..ce8b54c5 100644 --- a/scarf/agent/decide.py +++ b/scarf/agent/decide.py @@ -19,13 +19,23 @@ class DecisionValidationError(ValueError): """Raised when a model decision cites unknown or invalid evidence.""" -def _coerce_selected_id(selected_id: str, allowed: set[str]) -> str: - if selected_id in allowed: - return selected_id - matches = [evidence_id for evidence_id in allowed if evidence_id in selected_id] +def _coerce_evidence_id(evidence_id: str, allowed: set[str]) -> str: + """Map a model-emitted id onto an allowed evidence id when unambiguous. + + Live models often echo prompt scaffolding such as ``id=domain:biological`` + instead of the bare id. Accept that when exactly one allowed id is embedded. + """ + if evidence_id in allowed: + return evidence_id + stripped = evidence_id.strip() + if stripped.startswith("id="): + stripped = stripped[3:].strip() + if stripped in allowed: + return stripped + matches = [allowed_id for allowed_id in allowed if allowed_id in evidence_id] if len(matches) == 1: return matches[0] - return selected_id + return evidence_id def validate_decision( @@ -35,13 +45,17 @@ def validate_decision( allowed = {item.id for item in evidence} if not allowed: raise DecisionValidationError("evidence must contain at least one item") - selected_id = _coerce_selected_id(decision.selectedId, allowed) - evidence_ids = list(decision.evidenceIds) - if selected_id != decision.selectedId or ( - selected_id not in evidence_ids and selected_id in allowed + selected_id = _coerce_evidence_id(decision.selectedId, allowed) + evidence_ids = [ + _coerce_evidence_id(evidence_id, allowed) + for evidence_id in decision.evidenceIds + ] + if selected_id not in evidence_ids and selected_id in allowed: + evidence_ids = [selected_id, *evidence_ids] + if ( + selected_id != decision.selectedId + or evidence_ids != list(decision.evidenceIds) ): - if selected_id not in evidence_ids: - evidence_ids = [selected_id, *evidence_ids] decision = Decision( selectedId=selected_id, rationale=decision.rationale, diff --git a/scarf/features/gene_reference.py b/scarf/features/gene_reference.py new file mode 100644 index 00000000..b20682df --- /dev/null +++ b/scarf/features/gene_reference.py @@ -0,0 +1,374 @@ +"""Per-species Ensembl gene reference download and local lookup.""" + +import gzip +import os +import re +import urllib.request +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, TextIO + +__all__ = [ + "GeneReference", + "SpeciesSpec", + "cached_species", + "default_cache_dir", + "ensure_reference", + "load_reference", + "parse_gff3_genes", + "prefix_species", + "reference_summary", + "species_registry", + "write_reference_fixture", +] + +_ENSEMBL_GFF3 = "https://ftp.ensembl.org/pub/current_gff3/{species}/" +_ENSEMBL_GENOMES_GFF3 = ( + "https://ftp.ensemblgenomes.ebi.ac.uk/pub/{division}/current/gff3/{species}/" +) +_GENE_TYPES = frozenset({"gene", "ncRNA_gene", "pseudogene"}) +_GFF_NAME = re.compile( + r"^(?P