diff --git a/README.md b/README.md index c6f44356..37efbc12 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,7 @@ pip install "flowcept[mongo]" # MongoDB support pip install "flowcept[webservice]" # REST API and web UI pip install "flowcept[dask]" # Dask adapter pip install "flowcept[mlflow]" # MLflow adapter +pip install "flowcept[codex]" # Codex session-log adapter pip install "flowcept[rabbitmq]" # RabbitMQ MQ pip install "flowcept[kafka]" # Kafka MQ pip install "flowcept[telemetry]" # CPU/memory telemetry diff --git a/docs/README.md b/docs/README.md index 02a76809..c55f66fc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -142,6 +142,7 @@ Adapter setup is additive: flowcept --init-settings --dask -y flowcept --init-settings --mlflow -y flowcept --init-settings --tensorboard -y +flowcept --init-settings --codex -y ``` These commands add `adapters.` to the current settings file. @@ -262,6 +263,10 @@ Adapters: - `examples/tensorboard_example.py` - `notebooks/tensorboard.ipynb` - `tests/adapters/test_tensorboard.py` +- Codex adapter: + - `examples/codex_example.py` + - `tests/adapters/test_codex_interceptor.py` + - DPL skill docs: `resources/skills/agent-loop-provenance/README.md` Agentic provenance / MCP: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 3a2b7ec6..82b44cb7 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -63,6 +63,7 @@ Adapter flags are additive: flowcept --init-settings --dask -y flowcept --init-settings --mlflow -y flowcept --init-settings --tensorboard -y + flowcept --init-settings --codex -y They add ``adapters.`` to the current settings file instead of replacing it. diff --git a/docs/prov_capture.rst b/docs/prov_capture.rst index 6af769fb..b4d55002 100644 --- a/docs/prov_capture.rst +++ b/docs/prov_capture.rst @@ -170,6 +170,7 @@ Supported adapters: - **MLflow** — `MLflow example `_ - **Dask** — `Dask example `_ - **TensorBoard** — `TensorBoard example `_ +- **Codex** — `Codex example `_ Install the extras you need (see `installation `_), then configure the adapter in your settings file. Adapters capture runs, tasks, metrics, and artifacts and push them through Flowcept’s pipeline (MQ → DB). @@ -663,6 +664,7 @@ References & Examples - MLflow adapter: https://github.com/ORNL/flowcept/blob/main/examples/mlflow_example.py - Dask adapter: https://github.com/ORNL/flowcept/blob/main/examples/dask_example.py - TensorBoard adapter: https://github.com/ORNL/flowcept/blob/main/examples/tensorboard_example.py +- Codex adapter: https://github.com/ORNL/flowcept/blob/main/examples/codex_example.py - Loop instrumentation: https://github.com/ORNL/flowcept/blob/main/examples/instrumented_loop_example.py - LLM/PyTorch model: https://github.com/ORNL/flowcept/blob/main/examples/llm_complex/llm_model.py - MCP Agent tasks: https://github.com/ORNL/flowcept/blob/main/examples/agents/aec_agent_mock.py diff --git a/docs/schemas.rst b/docs/schemas.rst index 55ae2e59..39030273 100644 --- a/docs/schemas.rst +++ b/docs/schemas.rst @@ -21,7 +21,7 @@ It is described in: PROV-AGENT names the main building blocks you see in modern AI systems: -- **Activities** such as Campaign, Workflow, Task, AIModelInvocation, and AgentTool +- **Activities** such as Campaign, Workflow, Task, AIModelInvocation, and ToolInvocation - **Agents** such as an AI agent or a human user - **Data Objects** such as domain data, prompts, responses, scheduling info, and telemetry - **Relations** such as *used*, *wasGeneratedBy*, *wasAssociatedWith*, *wasAttributedTo*, and *wasInformedBy* @@ -64,21 +64,21 @@ Use the :class:`~flowcept.commons.vocabulary.PROV_AGENT` enum to set these value Captured automatically by :class:`~flowcept.instrumentation.flowcept_agent_task.FlowceptLLM`. ``used.prompt`` stores the input; ``generated.response`` stores the output; ``custom_metadata.llm_usage`` stores token counts. - * - ``PROV_AGENT.AGENT_TOOL`` - - ``agent_tool`` - - A tool execution by an AI agent (*AgentTool* in PROV-AGENT). + * - ``PROV_AGENT.TOOL_INVOCATION`` + - ``tool_invocation`` + - A tool execution by an AI agent (*ToolInvocation* in PROV-AGENT). Captured automatically by the :func:`~flowcept.instrumentation.flowcept_agent_task.agent_flowcept_task` decorator applied to MCP tools and LangGraph tool nodes. ``used`` stores tool arguments; ``generated`` stores the return value. -The ``wasInformedBy`` relation — an ``AgentTool`` activity informing an ``AIModelInvocation`` — is +The ``wasInformedBy`` relation — a ``ToolInvocation`` activity informing an ``AIModelInvocation`` — is the key link for root-cause analysis and downstream impact tracing in PROV-AGENT. In Flowcept this is expressed through the ``agent_id`` field: every task with the same ``agent_id`` belongs to the same AI agent and can be queried together to reconstruct the full agent provenance graph. The UI uses ``subtype`` to visually distinguish AI agent activities from regular workflow tasks. -Filter for ``subtype == "ai_model_invocation"`` or ``subtype == "agent_tool"`` to isolate agent +Filter for ``subtype == "ai_model_invocation"`` or ``subtype == "tool_invocation"`` to isolate agent interactions from the provenance database. Figure diff --git a/docs/setup.rst b/docs/setup.rst index 042e85b7..d50f6488 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -31,6 +31,7 @@ Good practice is to cherry-pick the extras relevant to your workflow instead of pip install flowcept[mlflow] # MLflow adapter pip install flowcept[dask] # Dask adapter pip install flowcept[tensorboard] # TensorBoard adapter + pip install flowcept[codex] # Codex session-log adapter pip install flowcept[rabbitmq] # RabbitMQ message queue pip install flowcept[kafka] # Kafka message queue pip install flowcept[nvidia] # NVIDIA GPU runtime capture @@ -250,6 +251,20 @@ Adapter flags are additive: flowcept --init-settings --dask -y flowcept --init-settings --mlflow -y flowcept --init-settings --tensorboard -y + flowcept --init-settings --codex -y + +Codex declared provenance +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Codex adapter can capture standard Codex session logs without extra +assistant instructions. To capture declared PROV-Agent-Loop semantics such as +execution plans, plan steps, loop iterations, evaluations, criteria, decisions, +beliefs, memories, and lessons learned, install the bundled +``resources/skills/agent-loop-provenance`` skill and enable +``adapters.codex.declared_provenance_enabled`` in the settings file. + +See ``resources/skills/agent-loop-provenance/README.md`` for the skill +installation and usage instructions. Custom Settings File --------------------- diff --git a/examples/codex_example.py b/examples/codex_example.py new file mode 100644 index 00000000..d3929d32 --- /dev/null +++ b/examples/codex_example.py @@ -0,0 +1,45 @@ +"""Observe a Codex session JSONL file with Flowcept. + +Setup: + pip install "flowcept[codex]" + flowcept --init-settings --codex -y + +Then edit your Flowcept settings file: + adapters.codex.file_path: /path/to/codex/session.jsonl + adapters.codex.declared_provenance_enabled: false + +With ``declared_provenance_enabled: false`` the adapter runs in OPL mode and +captures the provenance it can infer directly from the Codex JSONL log. + +For DPL mode, install the bundled agent-loop-provenance skill before starting +the Codex session, then set: + adapters.codex.declared_provenance_enabled: true + +The skill instructions live in: + resources/skills/agent-loop-provenance/README.md + +Run this observer while the Codex session is active, or replay an existing JSONL +by pointing ``adapters.codex.file_path`` to that file. +""" + +from time import sleep + +from flowcept import Flowcept +from flowcept.configs import settings + + +if __name__ == "__main__": + # Configure adapters.codex.file_path in settings.yaml before running this. + file_path = settings["adapters"]["codex"]["file_path"] + declared = settings["adapters"]["codex"].get("declared_provenance_enabled", False) + print(f"Codex JSONL path: {file_path}") + print(f"Declared provenance enabled: {declared}") + + with Flowcept("codex", save_workflow=False) as flowcept: + print("Codex adapter running. Press Ctrl+C to stop.") + try: + while True: + sleep(2) + print(f"records in buffer: {len(flowcept.get_buffer())}") + except KeyboardInterrupt: + print("Stopping Codex adapter...") diff --git a/examples/llm_tutorial/llm_train_campaign.py b/examples/llm_tutorial/llm_train_campaign.py index 2bcea8ae..034f4ea2 100644 --- a/examples/llm_tutorial/llm_train_campaign.py +++ b/examples/llm_tutorial/llm_train_campaign.py @@ -3,6 +3,8 @@ import argparse import json import sys +import os +from time import sleep import itertools import uuid import pandas as pd @@ -65,8 +67,8 @@ def generate_configs(params: dict): return result -def search_workflow(ntokens, dataset_ref, train_data_path, val_data_path, test_data_path, workflow_params, campaign_id=None, scheduler_file=None, start_dask_cluster=False, with_persistence=True, with_flowcept=True, dask_map_gpus=False): - client, cluster = start_dask(with_flowcept) +def search_workflow(ntokens, dataset_ref, train_data_path, val_data_path, test_data_path, workflow_params, campaign_id=None, scheduler_file=None, start_dask_cluster=False, with_persistence=True, with_flowcept=True, dask_map_gpus=False, with_slurm=False): + client, cluster = start_dask(scheduler_file, start_dask_cluster, with_flowcept, with_slurm) workflow_params["train_data_path"] = train_data_path workflow_params["val_data_path"] = val_data_path workflow_params["test_data_path"] = test_data_path @@ -114,40 +116,399 @@ def search_workflow(ntokens, dataset_ref, train_data_path, val_data_path, test_d file.write(f"{t2 - t1}\n") print("Done main loop. Closing dask.") + sleep(30) + print("Closing dask") close_dask(client, cluster, f) + print("Dask closed") + sleep(30) + print("Hi!") return search_wf_id, len(configs) -def start_dask(with_flowcept=True): +def start_dask( + scheduler_file=None, + start_dask_cluster=False, + with_flowcept=True, + with_slurm=False, + workers_per_node=8, +): + import os + import socket + import subprocess + import time + from distributed import Client - from distributed import LocalCluster - cluster = LocalCluster(n_workers=1) - scheduler = cluster.scheduler - client = Client(scheduler.address) - client.forward_logging() - # Registering Flowcept's worker adapters + + try: + import logging + logging.getLogger("distributed.worker").setLevel(logging.WARNING) + logging.getLogger("distributed.comm").setLevel(logging.WARNING) + except Exception: + pass + + def run_command( + command, + out_file="./cmd.out", + err_file="./cmd.err", + env=None, + ): + if env is None: + env = os.environ.copy() + + with open(out_file, "w") as out, open(err_file, "w") as err: + process = subprocess.Popen( + ["/bin/bash", "-c", command], + stdout=out, + stderr=err, + env=env, + ) + + return process + + def get_slurm_nodes(): + """ + Return the compute nodes belonging to the current Slurm allocation. + """ + if "SLURM_JOB_ID" not in os.environ: + raise RuntimeError( + "with_slurm=True, but this process is not running " + "inside a Slurm allocation." + ) + + nodelist = os.environ.get("SLURM_NODELIST") + + if not nodelist: + raise RuntimeError( + "SLURM_NODELIST is not defined." + ) + + result = subprocess.run( + ["scontrol", "show", "hostnames", nodelist], + capture_output=True, + text=True, + check=True, + ) + + nodes = [ + node.strip() + for node in result.stdout.splitlines() + if node.strip() + ] + + if not nodes: + raise RuntimeError( + f"Could not resolve Slurm nodes from {nodelist}" + ) + + return nodes + + # ------------------------------------------------------------------ + # Start a new Dask cluster + # ------------------------------------------------------------------ + + if start_dask_cluster: + + scheduler_file = os.path.abspath( + scheduler_file or "scheduler_file.json" + ) + + # Avoid accidentally connecting to an old scheduler. + if os.path.exists(scheduler_file): + os.remove(scheduler_file) + + llm_complex_dir = os.path.abspath( + os.path.dirname(__file__) + ) + + env = os.environ.copy() + + old_pythonpath = env.get("PYTHONPATH", "") + if old_pythonpath: + env["PYTHONPATH"] = ( + f"{llm_complex_dir}:{old_pythonpath}" + ) + else: + env["PYTHONPATH"] = llm_complex_dir + + # ============================================================== + # SLURM MODE + # ============================================================== + + if with_slurm: + + nodes = get_slurm_nodes() + + print( + f"Running inside Slurm job " + f"{os.environ['SLURM_JOB_ID']}" + ) + print(f"Allocated nodes: {nodes}") + + # ---------------------------------------------------------- + # Scheduler + # ---------------------------------------------------------- + + scheduler_node = nodes[0] + + print( + f"Starting Dask scheduler on " + f"{scheduler_node}" + ) + + scheduler_command = ( + f"srun " + f"--overlap " + f"--nodes=1 " + f"--ntasks=1 " + f"--nodelist={scheduler_node} " + f"dask scheduler " + f"--host {scheduler_node} " + f"--no-dashboard " + f"--no-show " + f"--scheduler-file {scheduler_file}" + ) + + scheduler_process = run_command( + scheduler_command, + out_file="dask_scheduler.out", + err_file="dask_scheduler.err", + env=env, + ) + + # Wait until scheduler_file appears rather than + # using a fixed sleep. + timeout = 60 + start = time.time() + + while not os.path.exists(scheduler_file): + if scheduler_process.poll() is not None: + raise RuntimeError( + "Dask scheduler exited before creating " + "the scheduler file. Check " + "dask_scheduler.err." + ) + + if time.time() - start > timeout: + raise RuntimeError( + f"Timeout waiting for " + f"{scheduler_file}" + ) + + time.sleep(1) + + print( + f"Dask scheduler started on " + f"{scheduler_node}" + ) + + # ---------------------------------------------------------- + # Workers + # ---------------------------------------------------------- + + worker_processes = [] + + for node in nodes: + + for gpu_id in range(workers_per_node): + + print( + f"Starting worker on " + f"{node}, GPU {gpu_id}" + ) + + worker_command = ( + f"srun " + f"--overlap " + f"--nodes=1 " + f"--ntasks=1 " + f"--nodelist={node} " + f"bash -c '" + f"export ROCR_VISIBLE_DEVICES={gpu_id}; " + f"dask worker " + f"--nthreads 1 " + f"--nworkers 1 " + f"--no-dashboard " + f"--scheduler-file {scheduler_file}" + f"'" + ) + + process = run_command( + worker_command, + out_file=( + f"dask_worker_{node}_" + f"{gpu_id}.out" + ), + err_file=( + f"dask_worker_{node}_" + f"{gpu_id}.err" + ), + env=env, + ) + + worker_processes.append(process) + + print( + f"Started {len(worker_processes)} " + f"Dask workers." + ) + + # ============================================================== + # NON-SLURM MODE + # ============================================================== + + else: + + print( + "Starting Dask cluster without Slurm." + ) + + print("Starting scheduler...") + + scheduler_command = ( + f"dask scheduler " + f"--host localhost " + f"--no-dashboard " + f"--no-show " + f"--scheduler-file {scheduler_file}" + ) + + scheduler_process = run_command( + scheduler_command, + out_file="dask_scheduler.out", + err_file="dask_scheduler.err", + env=env, + ) + + sleep(5) + + timeout = 60 + start = time.time() + + while not os.path.exists(scheduler_file): + if scheduler_process.poll() is not None: + raise RuntimeError( + "Dask scheduler exited before " + "creating scheduler file." + ) + + if time.time() - start > timeout: + raise RuntimeError( + "Timeout waiting for Dask scheduler." + ) + + time.sleep(1) + + print("Starting workers...") + + for gpu_id in range(workers_per_node): + + worker_command = ( + f"export ROCR_VISIBLE_DEVICES={gpu_id}; " + f"dask worker " + f"--nthreads 1 " + f"--nworkers 1 " + f"--no-dashboard " + f"--scheduler-file {scheduler_file}" + ) + + run_command( + worker_command, + out_file=f"dask_worker_{gpu_id}.out", + err_file=f"dask_worker_{gpu_id}.err", + env=env, + ) + + # Give the scheduler a moment to process worker registrations. + time.sleep(2) + + # ------------------------------------------------------------------ + # Connect client + # ------------------------------------------------------------------ + + if scheduler_file is None: + + from distributed import LocalCluster + + cluster = LocalCluster( + n_workers=1 + ) + + client = Client( + cluster.scheduler.address + ) + + else: + + cluster = None + + print( + f"Connecting to scheduler file " + f"{scheduler_file}", + flush=True, + ) + + client = Client( + scheduler_file=scheduler_file + ) + + print("Started Client.") + + # ------------------------------------------------------------------ + # Flowcept + # ------------------------------------------------------------------ + if with_flowcept: - from flowcept.flowceptor.adapters.dask.dask_plugins import FlowceptDaskWorkerAdapter - client.register_plugin(FlowceptDaskWorkerAdapter()) + + from flowcept.flowceptor.adapters.dask.dask_plugins import ( + FlowceptDaskWorkerAdapter, + ) + + client.register_plugin( + FlowceptDaskWorkerAdapter() + ) + + print("Registered Flowcept plugin.") return client, cluster -def close_dask(client, cluster, _flowcept=None): +def close_dask( + client, + cluster=None, + _flowcept=None, + shutdown_cluster=False, +): + print("Closing Dask...") + try: - print("Closing dask...") - client.close() - cluster.close() + if client is not None: + if shutdown_cluster: + print("Shutting down Dask scheduler and workers...") + client.shutdown() + else: + print("Closing Dask client...") + client.close() + + if cluster is not None: + print("Closing local Dask cluster...") + cluster.close() + print("Dask closed.") - if _flowcept: - print("Now closing flowcept consumer...") - _flowcept.stop() - print("Flowcept consumer closed.") + except Exception as e: - print(e) + print(f"Error while closing Dask: {e}") + finally: + if _flowcept is not None: + try: + print("Now closing Flowcept consumer...") + _flowcept.stop() + print("Flowcept consumer closed.") + except Exception as e: + print(f"Error while closing Flowcept: {e}") -def run_campaign(workflow_params, campaign_id=None, start_dask_cluster=False, with_persistence=True, with_flowcept=True): + +def run_campaign(workflow_params, campaign_id=None, start_dask_cluster=False, with_persistence=True, with_flowcept=True, with_slurm=False): _campaign_id = campaign_id or str(uuid.uuid4()) print(f"Campaign id={_campaign_id}") @@ -163,7 +524,7 @@ def run_campaign(workflow_params, campaign_id=None, start_dask_cluster=False, wi subset_size=subset_size, with_persistence=with_persistence) - _search_wf_id, n_configs = search_workflow(dataprep_generated["ntokens"], dataprep_generated["dataset_ref"], dataprep_generated["train_data_path"], dataprep_generated["val_data_path"], dataprep_generated["test_data_path"], workflow_params, campaign_id=_campaign_id, start_dask_cluster=start_dask_cluster, with_persistence=with_persistence, with_flowcept=with_flowcept) + _search_wf_id, n_configs = search_workflow(dataprep_generated["ntokens"], dataprep_generated["dataset_ref"], dataprep_generated["train_data_path"], dataprep_generated["val_data_path"], dataprep_generated["test_data_path"], workflow_params, campaign_id=_campaign_id, start_dask_cluster=start_dask_cluster, with_persistence=with_persistence, with_flowcept=with_flowcept, with_slurm=with_slurm) return _campaign_id, _dataprep_wf_id, _search_wf_id, dataprep_generated["train_n_batches"], dataprep_generated["val_n_batches"], n_configs @@ -188,6 +549,13 @@ def parse_args(): help=f"Use flowcept dask plugin (accepts: {', '.join(true_values)})", ) + arguments.add_argument( + "--with-slurm", + type=lambda v: v.lower() in true_values, + default=False, + help=f"Start Dask in a slurm job (accepts: {', '.join(true_values)})", + ) + arguments.add_argument("--start-dask-cluster", action="store_true", default=False, help="Start the dask cluster before execution. Use only for tests and not for real experiments") default_exp_param_settings = { @@ -229,7 +597,7 @@ def main(): workflow_params = json.loads(args.workflow_params) workflow_params["with_persistence"] = args.with_persistence print("TORCH SETTINGS: " + str(INSTRUMENTATION.get("torch"))) - run_campaign(workflow_params, campaign_id=args.campaign_id, start_dask_cluster=args.start_dask_cluster, with_persistence=args.with_persistence, with_flowcept=args.with_flowcept) + run_campaign(workflow_params, campaign_id=args.campaign_id, start_dask_cluster=args.start_dask_cluster, with_persistence=args.with_persistence, with_flowcept=args.with_flowcept, with_slurm=args.with_slurm) print("Alright! Congrats.") return 1 diff --git a/pyproject.toml b/pyproject.toml index f763ce57..af51021b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ docs = ["sphinx", "furo"] kafka = ["confluent-kafka<=2.8.0"] # As of today, 2/28/2025, version 2.8.1 is stale. When this gets fixed, let's remove the version constraint. https://pypi.org/project/confluent-kafka/#history rabbitmq = ["pika"] mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog", "cryptography"] +codex = ["watchdog"] nvidia = ["nvidia-ml-py"] amd = ["amdsmi"] mqtt = ["paho-mqtt"] diff --git a/resources/sample_settings.yaml b/resources/sample_settings.yaml index a479141e..cc4c1da1 100644 --- a/resources/sample_settings.yaml +++ b/resources/sample_settings.yaml @@ -1,4 +1,4 @@ -flowcept_version: 0.10.8 # Version of the Flowcept package. Do not update this manually. The CI updates it. This setting file is compatible with this version. +flowcept_version: 1.0.3 # Version of the Flowcept package. Do not update this manually. The CI updates it. This setting file is compatible with this version. project: debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later. @@ -166,6 +166,15 @@ adapters: log_metrics: ['accuracy'] watch_interval_sec: 5 + codex: + kind: codex + file_path: codex_events.jsonl + watch_interval_sec: 1 + recursive: true + include_developer_messages: true + include_reasoning: true + declared_provenance_enabled: false + dask: kind: dask worker_should_get_input: true diff --git a/resources/skills/agent-loop-provenance/AGENT_LOOP_PROVENANCE_SKILL.md b/resources/skills/agent-loop-provenance/AGENT_LOOP_PROVENANCE_SKILL.md new file mode 100644 index 00000000..05ab6967 --- /dev/null +++ b/resources/skills/agent-loop-provenance/AGENT_LOOP_PROVENANCE_SKILL.md @@ -0,0 +1,197 @@ +--- +name: agent-loop-provenance +description: Emit compact declared provenance annotations for Flowcept PROV-Agent-Loop capture when a code-assistant session needs explicit agentic-loop semantics such as objectives, plans, plan steps, loop iterations, evaluations, criteria, thoughts, observations, beliefs, decisions, mandates, memories, or lessons learned. +--- + +# Agent Loop Provenance + +Emit sparse Declared Provenance Layer (DPL) annotations for agentic-loop concepts the runtime log cannot reliably infer. The Observed Provenance Layer (OPL) already captures raw prompts, final responses, model/tool calls, command arguments, raw outputs, timestamps, token usage, files, errors, and status. Do not duplicate those as DPL. + +Use one compact JSON object inside each `...` block. Prefer internal/progress messages, avoid final user-visible answers, and read `references/flowcept-prov-agent-loop.md` only when you need the full Flowcept mapping. + +## Core Rules + +Emit DPL only when a semantic concept is created or materially changes. + +Do not emit DPL for raw user prompts, final user-visible responses, raw tool calls/outputs, timestamps, token counts, file paths, diffs, or exit codes already visible to the runtime. + +Do emit DPL for refined objectives/plans, plan-step and loop boundaries, evaluation start/end with criteria/result/decision, and declared `Thought`, `Observation`, `Belief`, `Decision`, `Mandate`, `Memory`, or `LessonLearned`. + +DPL boundary tags are control markers, not retrospective notes. Before the first tool call, code edit, validation command, or substantive execution for a planned step, emit: + +1. `PlanStepExecution started`. +2. `LoopIteration started`. +3. `Evaluation started` if the next action is a test, check, validation, metric inspection, or acceptance assessment. + +Never close a `LoopIteration` or `PlanStepExecution` and then run another tool for the same work. If more work is needed, keep the loop open. If switching steps, close the current loop/step first, then open the next step/loop before doing work. + +Keep at most one active `PlanStepExecution` and one active `LoopIteration` unless the user explicitly asks for nested work. Prefer no nesting for provenance clarity. + +Do not write normal prose like "Need start step..." as an internal reminder. Emit the correct DPL tag or omit the note. + +## Payload + +Use this generic shape: + +```json +{"layer":"DPL","class":"Evaluation","event":"started","label":"Run tests","criteria":["Tests pass"]} +``` + +Required: + +- `layer`: always `"DPL"`. +- `class`: one canonical concept. +- `event`: only for boundary activities, `"started"` or `"finished"`. + +Canonical classes: + +`Objective`, `Plan`, `PlanStepExecution`, `LoopIteration`, `Evaluation`, `EvaluationCriteria`, `EvaluationResult`, `Thought`, `Observation`, `Belief`, `Decision`, `Mandate`, `Memory`, `LessonLearned`. + +Useful fields: `label`, `summary`, `status`, `criteria`, `criteria_ids`, `command`, `result`, `decision`, `reason`, and `derived_from`. + +Use these controlled values where possible: `status` as `finished`, `passed`, `failed`, `inconclusive`, `skipped`, `error`, or `unknown`; `decision` as `continue`, `stop`, `retry`, `revise_plan`, `escalate`, or `fail`; `reason` as `success`, `timeout`, `max_iterations`, `budget_exceeded`, `approval_denied`, `test_failure`, or `guardrail_violation`. + +Do not invent runtime ids such as `session_id`, `turn_id`, `workflow_id`, `task_id`, or `parent_id`. The adapter owns ids and links. Use stable labels. + +## Planning + +When producing an implementation or experiment plan, make the execution structure explicit enough to replay as provenance. + +Every plan should include one objective/summary, a `Steps` section with one bullet per executable unit, and evaluation criteria for the whole plan when relevant. + +Each executable step must have a stable label and, if needed, a short description. + +Recommended visible shape: + +```markdown +**Steps** +- `Inspect tutorial` + Identify runnable entrypoints and constraints. +- `Create local config` + Create Conda/project configuration. +- `Run Step 1` + Execute the small Step 1 workflow. +- `Validate persistence` + Query the configured database. + +**Evaluation criteria** +- Config files parse successfully. +- Step 1 completes or fails with a documented blocker. +- Expected workflow/task records are present. +``` + +When executing the plan, copy the label exactly from `Steps` into each `PlanStepExecution`. Do not start a step whose label was not present unless the plan was revised or the user introduced it. + +If the plan changes materially, emit a `Decision` with `decision:"revise_plan"` and then produce the revised visible step labels before continuing. + +## Step And Loop Boundaries + +For every planned step actually reached: + +```text +{"layer":"DPL","class":"PlanStepExecution","event":"started","label":"Create local config"} +{"layer":"DPL","class":"LoopIteration","event":"started","label":"Create local config loop","summary":"Create and inspect the local configuration."} +``` + +Then do the work. Finish in reverse order: + +```text +{"layer":"DPL","class":"LoopIteration","event":"finished","label":"Create local config loop","status":"finished","summary":"Configuration was created and inspected."} +{"layer":"DPL","class":"PlanStepExecution","event":"finished","label":"Create local config","status":"finished","summary":"Local configuration is ready."} +``` + +Do not emit `PlanStepExecution finished` until every loop, tool invocation, and evaluation belonging to that step has finished and been interpreted. + +A loop is one bounded agentic cycle: assemble context, invoke the model or choose an action, run tools/actions, capture observations, evaluate when relevant, and decide whether to continue, retry, revise, escalate, or stop. + +Use one loop per attempt. Start a new loop when: + +- an evaluation finishes with `decision:"retry"`; +- a decision is `retry`, `revise_plan`, `escalate`, `fail`, or `stop`; +- a failed action changes command, environment, dependency, parameter, resource request, or strategy; +- an observation changes the working belief used by the next action; +- validation ends and a new validation attempt begins; +- a step switches between implementation, validation, or final inspection. + +Bad ordering: + +```text +{"layer":"DPL","class":"LoopIteration","event":"finished","label":"Run tests loop"} +``` + +Then running `pytest`. The tool is outside the loop. + +Good ordering: + +```text +{"layer":"DPL","class":"PlanStepExecution","event":"started","label":"Run tests"} +{"layer":"DPL","class":"LoopIteration","event":"started","label":"Run tests loop"} +{"layer":"DPL","class":"Evaluation","event":"started","label":"Run tests","criteria":["Tests pass"],"command":"pytest"} +``` + +Run the command, interpret it, then emit `Evaluation finished`, semantic evidence, `LoopIteration finished`, and `PlanStepExecution finished`. + +## Evaluation + +Criteria describe what will be checked. An `Evaluation` task exists only when the check actually runs or the assessment is actually performed. + +Before a validation/test/check command: + +```text +{"layer":"DPL","class":"Evaluation","event":"started","label":"Focused tests","criteria":["Focused tests pass"],"command":"pytest tests/test_parser.py"} +``` + +After interpreting the result: + +```text +{"layer":"DPL","class":"Evaluation","event":"finished","label":"Focused tests","status":"passed","decision":"continue","reason":"success","result":"Focused tests passed."} +``` + +After an evaluation, usually emit: + +- `Observation`: what the result shows. +- `Belief`: accepted working knowledge used later. +- `Decision`: continue, retry, revise, stop, escalate, fail, or accept. +- `LessonLearned`: reusable knowledge from evaluation/failure/debugging/comparison. Required when a failure/retry/workaround teaches something useful beyond the immediate command. +- `Memory`: only information intentionally retained for later steps, sessions, or reruns. + +Always emit `LessonLearned` when an evaluation/debugging cycle discovers a reusable rule, workaround, environment constraint, dependency constraint, command preference, validation shortcut, resource behavior, or failure cause. In particular, if an evaluation fails and the next attempt changes command, dependency, environment, parameter, resource request, or strategy, emit `LessonLearned` after the retry is understood. If useful for future local/Frontier reruns, also emit `Memory`. + +Lesson-learned retry example: + +```text +{"layer":"DPL","class":"Evaluation","event":"finished","label":"Run local tests","status":"failed","decision":"retry","reason":"test_failure","result":"pytest was unavailable in the active environment."} +{"layer":"DPL","class":"Observation","summary":"The validation failed before tests ran because pytest is not installed.","derived_from":"Run local tests"} +{"layer":"DPL","class":"Decision","decision":"retry","reason":"test_failure","summary":"Retry with uv run --with pytest."} +{"layer":"DPL","class":"LessonLearned","summary":"Use uv run --with pytest for this repo when pytest is not declared in the local environment."} +``` + +## Semantic Messages + +Use semantic tags sparsely, but do not leave them absent in nontrivial DPL runs. + +- `Thought`: concise reasoning-in-progress or next investigative direction. +- `Observation`: interpreted feedback from tool result, test, metric, UI state, log, or error. Raw output is not an observation. +- `Belief`: accepted working knowledge derived from evidence and used later. +- `Decision`: explicit control choice, scope choice, approval outcome, stop/continue/retry/revise/escalate/fail. +- `Mandate`: human/system instruction, permission, restriction, approval, denial, budget, policy, or constraint. +- `Memory`: information intentionally retained for later steps, turns, sessions, or experiments. +- `LessonLearned`: reusable operational knowledge derived from evaluation, failure, repair, retry, workaround, or comparison. + +Examples: + +```text +{"layer":"DPL","class":"Observation","summary":"The test failed because the config file was missing.","derived_from":"last_tool_result"} +{"layer":"DPL","class":"Belief","summary":"The project requires creating the config before running Step 1.","derived_from":"last_observation"} +{"layer":"DPL","class":"Decision","decision":"retry","reason":"test_failure","summary":"Create the config and rerun Step 1."} +{"layer":"DPL","class":"LessonLearned","summary":"Validate local configuration before launching larger Frontier runs."} +``` + +Minimum DPL evidence for evaluation-oriented runs: + +- every executed plan step has `PlanStepExecution started/finished`; +- every executed plan step contains at least one `LoopIteration started/finished`; +- every meaningful tool/test/log/metric loop emits at least one `Observation`; +- every control-flow change emits a `Decision`; +- every executed validation emits `Evaluation started/finished`; +- nontrivial runs emit `Memory` or `LessonLearned` when reusable knowledge appears. diff --git a/resources/skills/agent-loop-provenance/README.md b/resources/skills/agent-loop-provenance/README.md new file mode 100644 index 00000000..4a8bf2d5 --- /dev/null +++ b/resources/skills/agent-loop-provenance/README.md @@ -0,0 +1,47 @@ +# Agent Loop Provenance Skill + +This directory contains the agent skill used by the Flowcept Code assistant adapters to capture the Declared Provenance Layer (DPL) for PROV-Agent-Loop experiments. + +The Flowcept code assistant adapters can always capture the Observed Provenance Layer (OPL) from the code assistant session JSONL logs. The OPL includes structural information such as prompts, assistant responses, model calls, tool calls, command inputs and outputs, timestamps, statuses, errors, and token usage. + +The DPL requires the assistant to explicitly emit provenance annotations for semantic concepts that cannot be reliably inferred from raw logs, such as objectives, plans, plan steps, loop iterations, evaluations, criteria and results, thoughts, observations, beliefs, decisions, mandates, memories, and lessons learned. + +## Files + +- `AGENT_LOOP_PROVENANCE_SKILL.md`: Source text for the agent skill. It is intentionally not named `SKILL.md` because this repository ignores `SKILL.md` files. +- `agents/openai.yaml`: Optional agent metadata for the skill. +- `references/flowcept-prov-agent-loop.md`: Detailed Flowcept/PROV-Agent-Loop mapping used by the skill. + +## Using the Codex Adapter + +### Installing for Codex + +Codex expects installed skills to contain a file named `SKILL.md`. To install this skill locally, copy this directory into your Codex skills directory and rename the source file to `SKILL.md` in the installed copy: + +```bash +mkdir -p ~/.codex/skills/agent-loop-provenance +cp AGENT_LOOP_PROVENANCE_SKILL.md ~/.codex/skills/agent-loop-provenance/SKILL.md +cp -R agents references ~/.codex/skills/agent-loop-provenance/ +``` + +Start a new Codex session after installing or updating the skill. + +### Configuring the Codex Adapter + +Configure Flowcept to use the Codex adapter and point it to the Codex session JSONL file: + +```yaml +adapters: + codex: + kind: codex + file_path: /path/to/codex/session.jsonl + watch_interval_sec: 1 + recursive: true + include_developer_messages: true + include_reasoning: true + declared_provenance_enabled: true +``` + +Set `declared_provenance_enabled: true` for DPL runs. Set it to `false` for OPL-only runs. + +In the Codex session used to perform the experiment, enable or invoke the `agent-loop-provenance` skill before asking the assistant to configure and run the workflow. The adapter parses the skill's `...` annotations from the Codex JSONL and maps them to Flowcept workflow, task, and entity records. \ No newline at end of file diff --git a/resources/skills/agent-loop-provenance/agents/openai.yaml b/resources/skills/agent-loop-provenance/agents/openai.yaml new file mode 100644 index 00000000..d3d41282 --- /dev/null +++ b/resources/skills/agent-loop-provenance/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Agent Loop Provenance" + short_description: "Log Flowcept agent loop provenance" + default_prompt: "Use $agent-loop-provenance while working so Codex logs contain Flowcept PROV-Agent-Loop execution plans, steps, loop iterations, evaluations, and results." diff --git a/resources/skills/agent-loop-provenance/references/flowcept-prov-agent-loop.md b/resources/skills/agent-loop-provenance/references/flowcept-prov-agent-loop.md new file mode 100644 index 00000000..08bb3ec0 --- /dev/null +++ b/resources/skills/agent-loop-provenance/references/flowcept-prov-agent-loop.md @@ -0,0 +1,149 @@ +# Flowcept PROV-Agent-Loop Mapping + +## Capture Layers + +Flowcept separates two capture layers: + +- Observed Provenance Layer (OPL): deterministic records already present in the assistant/runtime log. +- Declared Provenance Layer (DPL): compact semantic annotations emitted by the agent or human when the runtime cannot infer meaning. + +This skill only emits DPL. The adapter captures OPL separately. + +## OPL Records + +The adapter should capture these without DPL: + +- `Session` +- `UserPrompt` when a real user request is observed +- `AgentResponse` from the final user-visible answer +- `AIModelInvocation` +- `ToolInvocation` +- raw tool results +- commands, arguments, working directories, sandbox/approval metadata +- timestamps, token usage, model/provider metadata +- file edits, errors, exit codes, diffs, and observable ids +- structural `LoopIteration` per turn when no semantic loop boundary is declared +- heuristic `Evaluation` for obvious test/check commands + +Do not emit DPL just to repeat these records. + +## DPL Records + +Emit DPL for concepts that require semantic declaration: + +- `Objective` +- `PlanStepExecution` +- semantic `LoopIteration` +- `Evaluation`, `EvaluationCriteria`, and `EvaluationResult` +- `Thought` +- `Observation` +- `Belief` +- `Decision` +- `Mandate` +- `Memory` +- `LessonLearned` + +Use DPL only when a concept is created or materially changed. + +## Flowcept Objects + +Map PROV-Agent-Loop concepts to Flowcept records: + +- `Session` -> `WorkflowObject`, `subtype: "session"`. +- `ExecutionPlan` -> `WorkflowObject`, `subtype: "execution_plan"`. +- `PlanStepExecution` -> `TaskObject`, `subtype: "plan_step_execution"`. +- `LoopIteration` -> `TaskObject`, `subtype: "loop_iteration"`. +- `AIModelInvocation` -> `TaskObject`, `subtype: "ai_model_invocation"`. +- `ToolInvocation` -> `TaskObject`, `subtype: "tool_invocation"`. +- `Evaluation` -> `TaskObject`, `subtype: "evaluation"`. +- `AIAgent` and `Human` -> `AgentObject`. +- `Objective`, `Plan`, `EvaluationCriteria`, `EvaluationResult`, raw tool inputs, and raw tool results -> entity dicts in `used.entities[]` or `generated.entities[]`. +- `Message`, `UserPrompt`, `AgentResponse`, `Thought`, `Observation`, `Belief`, `Decision`, `Mandate`, `Memory`, and `LessonLearned` -> message-like entity dicts in `used.messages[]` or `generated.messages[]`. + +## DPL Payload Shape + +Use one JSON object inside `...`: + +```json +{"layer":"DPL","class":"Evaluation","event":"started","label":"Run tests","criteria":["Focused tests pass"]} +``` + +Required: + +- `layer: "DPL"` +- `class`: the PROV-Agent-Loop class + +Boundary activities use: + +- `event: "started"` +- `event: "finished"` + +Do not invent `session_id`, `turn_id`, `workflow_id`, `task_id`, or `parent_id`. The adapter creates these from the runtime log. + +## Message Specializations + +Use these distinctions: + +- `Thought`: tentative intermediate reasoning or deliberation. +- `Observation`: perceived feedback derived from a tool, environment, evaluation, or other evidence. +- `Belief`: working knowledge accepted as true or likely true in the current loop/session. +- `Decision`: explicit control-flow choice, approval outcome, stop/retry/escalation/revision. +- `Mandate`: instruction, authorization, policy, or delegated constraint. +- `Memory`: persistent context promoted for reuse. +- `LessonLearned`: reusable operational learning. +Raw user prompts and final responses are OPL, not DPL. + +## Used And Generated Shape + +Flowcept keeps message-like entities under a stable `messages` key and non-message entities under an `entities` key: + +```json +{ + "used": { + "messages": [ + {"type": "observation", "content": "The previous validation failed because stderr was missing."} + ], + "entities": [ + {"type": "evaluation_criteria", "content": "Focused tests pass."} + ] + }, + "generated": { + "entities": [ + {"type": "evaluation_result", "status": "passed", "content": "Focused tests passed."} + ] + } +} +``` + +Tool inputs and outputs are raw OPL entity records, not messages. Emit `Observation` only when the agent interprets the output: + +```json +{"layer":"DPL","class":"Observation","summary":"The test failure shows the parser does not handle missing stderr.","derived_from":"last_tool_result"} +``` + +## Workflow And Task Links + +The adapter infers links: + +- `Session -> ExecutionPlan` +- `ExecutionPlan -> PlanStepExecution` +- `PlanStepExecution -> LoopIteration` +- `LoopIteration -> AIModelInvocation` +- `LoopIteration -> ToolInvocation` +- `LoopIteration -> Evaluation` + +The agent helps by keeping labels stable: + +- `label`: active plan step, loop, or evaluation label. +- `criteria`: evaluation criteria text. +- `criteria_ids`: only when naturally declared in a plan. +- `command`: test/check command. +- `derived_from`: obvious evidence label. + +## Boundary Rules + +- Prefer explicit start/end tags for semantic steps, loops, and evaluations. +- Every explicit test/check/validation should be associated with a loop iteration and evaluation when possible. +- Use `Decision` for retry, stop, revise-plan, escalate, or fail choices. +- Use `Memory` and `LessonLearned` only when information is intentionally promoted for future reuse. +- Keep annotations short; long outputs stay in raw logs or external records. diff --git a/src/flowcept/agents/README.md b/src/flowcept/agents/README.md index 36fa1fbf..ccfd07b6 100644 --- a/src/flowcept/agents/README.md +++ b/src/flowcept/agents/README.md @@ -118,12 +118,12 @@ agent-specific activities in the task database: | Enum | Stored string | What it captures | |---|---|---| | `PROV_AGENT.AI_MODEL_INVOCATION` | `"ai_model_invocation"` | One LLM prompt → response call | -| `PROV_AGENT.AGENT_TOOL` | `"agent_tool"` | One tool execution by an AI agent | +| `PROV_AGENT.TOOL_INVOCATION` | `"tool_invocation"` | One tool execution by an AI agent | ### Automatic capture **MCP tools** — every `@mcp_flowcept.tool()` function in `mcp_tools/` is also -decorated with `@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL)`. No extra +decorated with `@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION)`. No extra code needed; tool calls are stored automatically when the interceptor is running. **LLM calls** — wrap any LangChain model with `FlowceptLLM` to record every @@ -139,7 +139,7 @@ response = wrapped.invoke("How many tasks failed?") wraps each graph execution in a `Flowcept` context (`workflow_name="Flowcept LangGraph Chat"`, `start_persistence=True`). This gives every chat turn its own `workflow_id`. Within the graph, `call_model` uses `FlowceptLLM` and `call_tools` uses -`FlowceptTask(subtype=PROV_AGENT.AGENT_TOOL)` — both inherit +`FlowceptTask(subtype=PROV_AGENT.TOOL_INVOCATION)` — both inherit `Flowcept.current_workflow_id` automatically. ### Querying agent provenance @@ -149,7 +149,7 @@ Within the graph, `call_model` uses `FlowceptLLM` and `call_tools` uses Flowcept.db.task_query(filter={"subtype": "ai_model_invocation", "agent_id": my_agent_id}) # All tool executions in a chat session (workflow) -Flowcept.db.task_query(filter={"subtype": "agent_tool", "workflow_id": thread_id}) +Flowcept.db.task_query(filter={"subtype": "tool_invocation", "workflow_id": thread_id}) ``` The UI uses `subtype` to display AI agent workflows differently from regular diff --git a/src/flowcept/agents/chat_orchestration/graph_builder.py b/src/flowcept/agents/chat_orchestration/graph_builder.py index fc28b75e..bfe628c9 100644 --- a/src/flowcept/agents/chat_orchestration/graph_builder.py +++ b/src/flowcept/agents/chat_orchestration/graph_builder.py @@ -72,7 +72,7 @@ def call_tools(state: MessagesState): tool_fn = tools_by_name.get(name) with FlowceptTask( activity_id=name, - subtype=PROV_AGENT.AGENT_TOOL, + subtype=PROV_AGENT.TOOL_INVOCATION, used=sanitize_json_like(args, mongo_safe_keys=True), agent_id=agent_id, ) as task: diff --git a/src/flowcept/agents/mcp/context_manager.py b/src/flowcept/agents/mcp/context_manager.py index 68f1d1ad..a95b0d9c 100644 --- a/src/flowcept/agents/mcp/context_manager.py +++ b/src/flowcept/agents/mcp/context_manager.py @@ -224,15 +224,15 @@ def message_handler(self, msg_obj: Dict): if msg_type == "task": task_msg = TaskObject.from_dict(msg_obj) - # Filter agent-internal tasks (AI_MODEL_INVOCATION and AGENT_TOOL) that must not + # Filter agent-internal tasks (AI_MODEL_INVOCATION and TOOL_INVOCATION) that must not # pollute the user-workflow DataFrame. Two cases warrant filtering: # 1. The task belongs to this agent (original self-filter). # 2. A user workflow is already loaded and this task belongs to a different workflow # — i.e. it was emitted by an external agent (e.g. the chat orchestrator) running # its own session workflow alongside the user's workflow. - # User workflow tasks that happen to carry AGENT_TOOL (e.g. submit_gridsearch_job) + # User workflow tasks that happen to carry TOOL_INVOCATION (e.g. submit_gridsearch_job) # are preserved because their workflow_id matches the loaded workflow. - if task_msg.subtype in (PROV_AGENT.AI_MODEL_INVOCATION, PROV_AGENT.AGENT_TOOL): + if task_msg.subtype in (PROV_AGENT.AI_MODEL_INVOCATION, PROV_AGENT.TOOL_INVOCATION): loaded_wf_id = (self.context.workflow_msg_obj or {}).get("workflow_id") task_wf_id = msg_obj.get("workflow_id") if task_msg.agent_id == self.agent_id or (loaded_wf_id and task_wf_id != loaded_wf_id): @@ -255,7 +255,7 @@ def message_handler(self, msg_obj: Dict): FlowceptTask( agent_id=self.agent_id, generated={"msg": "Provenance Agent reset context."}, - subtype=PROV_AGENT.AGENT_TOOL, + subtype=PROV_AGENT.TOOL_INVOCATION, activity_id="reset_user_context", ).send() return True diff --git a/src/flowcept/agents/mcp/mcp_tools/dashboard_mcp_tools.py b/src/flowcept/agents/mcp/mcp_tools/dashboard_mcp_tools.py index e7fe3738..751ce82a 100644 --- a/src/flowcept/agents/mcp/mcp_tools/dashboard_mcp_tools.py +++ b/src/flowcept/agents/mcp/mcp_tools/dashboard_mcp_tools.py @@ -15,21 +15,21 @@ @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_make_chart(card_spec: Dict[str, Any], context: Optional[Dict[str, Any]] = None) -> ToolResult: """Build a chart from a declarative dashboard card spec; the UI renders the result.""" return dashboard_tools.make_chart(card_spec=card_spec, context=context) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_get_dashboard(dashboard_id: str) -> ToolResult: """Get a stored dashboard spec by id.""" return dashboard_tools.get_dashboard(dashboard_id=dashboard_id) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_update_dashboard(dashboard_id: str, spec: Dict[str, Any]) -> ToolResult: """Replace a stored dashboard spec with a complete revised spec.""" return dashboard_tools.update_dashboard(dashboard_id=dashboard_id, spec=spec) @@ -41,7 +41,7 @@ def db_update_dashboard(dashboard_id: str, spec: Dict[str, Any]) -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_make_chart(result_code: str, plot_code: str = "") -> ToolResult: """Generate a chart from the in-memory tasks DataFrame. @@ -62,14 +62,14 @@ def df_make_chart(result_code: str, plot_code: str = "") -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_get_dashboard(dashboard_id: str) -> ToolResult: """Get a stored dashboard spec by id (DF path — delegates to the same dashboard store).""" return dashboard_tools.get_dashboard(dashboard_id=dashboard_id) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_update_dashboard(dashboard_id: str, spec: Dict[str, Any]) -> ToolResult: """Replace a stored dashboard spec with a complete revised spec (DF path).""" return dashboard_tools.update_dashboard(dashboard_id=dashboard_id, spec=spec) diff --git a/src/flowcept/agents/mcp/mcp_tools/db_query_mcp_tools.py b/src/flowcept/agents/mcp/mcp_tools/db_query_mcp_tools.py index 752c3169..5b19bdb4 100644 --- a/src/flowcept/agents/mcp/mcp_tools/db_query_mcp_tools.py +++ b/src/flowcept/agents/mcp/mcp_tools/db_query_mcp_tools.py @@ -14,7 +14,7 @@ @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_query_tasks( filter: Optional[Dict[str, Any]] = None, projection: Optional[List[str]] = None, @@ -26,35 +26,35 @@ def db_query_tasks( @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_query_workflows(filter: Optional[Dict[str, Any]] = None, limit: int = 100) -> ToolResult: """Query workflow provenance records in the database with a Mongo-style filter.""" return db_query_tools.query_workflows(filter=filter, limit=limit) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_get_task_summary(filter: Optional[Dict[str, Any]] = None) -> ToolResult: """Summarize tasks matching a filter: status counts, per-activity durations, time range.""" return db_query_tools.get_task_summary(filter=filter) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_list_campaigns(campaign_id: Optional[str] = None) -> ToolResult: """List derived campaign summaries (campaigns group workflows and tasks).""" return db_query_tools.list_campaigns(campaign_id=campaign_id) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_list_agents(filter: Optional[Dict[str, Any]] = None) -> ToolResult: """List derived agent summaries (agents observed in task provenance).""" return db_query_tools.list_agents(filter=filter) @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_query_objects( filter: Optional[Dict[str, Any]] = None, projection: Optional[Any] = None, @@ -70,7 +70,7 @@ def db_query_objects( @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_highlight_lineage( task_ids: Optional[List[str]] = None, filter: Optional[Dict[str, Any]] = None, @@ -81,7 +81,7 @@ def db_highlight_lineage( @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def db_fix_query( query_params: Dict[str, Any], error: str, diff --git a/src/flowcept/agents/mcp/mcp_tools/df_query_mcp_tools.py b/src/flowcept/agents/mcp/mcp_tools/df_query_mcp_tools.py index ead7d384..bef87c4d 100644 --- a/src/flowcept/agents/mcp/mcp_tools/df_query_mcp_tools.py +++ b/src/flowcept/agents/mcp/mcp_tools/df_query_mcp_tools.py @@ -33,7 +33,7 @@ @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def run_df_query(code: str, context_kind: str = "tasks") -> ToolResult: """Execute pandas code against the current context DataFrame. @@ -59,7 +59,7 @@ def run_df_query(code: str, context_kind: str = "tasks") -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_query_tasks(code: str) -> ToolResult: """Query task provenance using pandas code against the in-memory tasks DataFrame. @@ -70,7 +70,7 @@ def df_query_tasks(code: str) -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_query_workflows() -> ToolResult: """Return the workflow record(s) loaded in the agent's in-memory context. @@ -98,7 +98,7 @@ def df_query_workflows() -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_query_objects(code: str) -> ToolResult: """Query stored data-object records using pandas code against the in-memory objects DataFrame. @@ -109,7 +109,7 @@ def df_query_objects(code: str) -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_get_task_summary() -> ToolResult: """Summarize tasks in the in-memory DataFrame: activity types, status counts, time range. @@ -136,7 +136,7 @@ def df_get_task_summary() -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_get_objects_summary() -> ToolResult: """Summarize stored objects in the in-memory objects DataFrame: available types, counts, and tracked columns. @@ -157,7 +157,7 @@ def df_get_objects_summary() -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_list_campaigns() -> ToolResult: """List campaign summaries derived from the in-memory tasks DataFrame. @@ -180,7 +180,7 @@ def df_list_campaigns() -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_list_agents() -> ToolResult: """List agent summaries derived from the in-memory tasks DataFrame. @@ -199,7 +199,7 @@ def df_list_agents() -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_highlight_lineage(task_ids: list = None, code: str = None) -> ToolResult: """Return seed task IDs for UI lineage highlighting from the in-memory tasks DataFrame. @@ -222,7 +222,7 @@ def df_highlight_lineage(task_ids: list = None, code: str = None) -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def df_fix_query(raw_text: str, runtime_error: str = None) -> ToolResult: """Extract or repair pandas code using the current agent DataFrame columns. diff --git a/src/flowcept/agents/mcp/mcp_tools/report_tools.py b/src/flowcept/agents/mcp/mcp_tools/report_tools.py index c0a3ffb1..8ae58ea3 100644 --- a/src/flowcept/agents/mcp/mcp_tools/report_tools.py +++ b/src/flowcept/agents/mcp/mcp_tools/report_tools.py @@ -11,7 +11,7 @@ @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def generate_workflow_card( workflow_id: str = None, campaign_id: str = None, diff --git a/src/flowcept/agents/mcp/mcp_tools/schema_mcp_tools.py b/src/flowcept/agents/mcp/mcp_tools/schema_mcp_tools.py index fe7df713..d9d4d0cd 100644 --- a/src/flowcept/agents/mcp/mcp_tools/schema_mcp_tools.py +++ b/src/flowcept/agents/mcp/mcp_tools/schema_mcp_tools.py @@ -10,7 +10,7 @@ @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def get_schema_context(tool_context: str = "db", workflow_id: Optional[str] = None) -> ToolResult: """Return schema context for the active query path. @@ -66,7 +66,7 @@ def get_schema_context(tool_context: str = "db", workflow_id: Optional[str] = No @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def get_df_schema_context(context_kind: str = "tasks") -> ToolResult: """Return the in-memory DataFrame schema context for the DF query path. @@ -105,7 +105,7 @@ def get_df_schema_context(context_kind: str = "tasks") -> ToolResult: @mcp_flowcept.tool() -@agent_flowcept_task(subtype=PROV_AGENT.AGENT_TOOL) +@agent_flowcept_task(subtype=PROV_AGENT.TOOL_INVOCATION) def get_workflow_schema_context(workflow_id: str) -> ToolResult: """Return the workflow-scoped schema context for the DB query path. diff --git a/src/flowcept/cli.py b/src/flowcept/cli.py index 8ae0c26b..11e71efe 100644 --- a/src/flowcept/cli.py +++ b/src/flowcept/cli.py @@ -125,6 +125,7 @@ def init_settings( dask: bool = False, mlflow: bool = False, tensorboard: bool = False, + codex: bool = False, ): """ Create or extend the user settings file. @@ -142,6 +143,8 @@ def init_settings( Add default mlflow adapter settings under `adapters.mlflow`. tensorboard : bool, optional Add default tensorboard adapter settings under `adapters.tensorboard`. + codex : bool, optional + Add default Codex adapter settings under `adapters.codex`. Notes ----- @@ -151,7 +154,7 @@ def init_settings( and only writes adapter sections. - `--full` only copies the full sample file. It does not apply a runtime profile. """ - add_adapters = dask or mlflow or tensorboard + add_adapters = dask or mlflow or tensorboard or codex settings_path_env = os.getenv("FLOWCEPT_SETTINGS_PATH", None) if settings_path_env is not None: @@ -206,6 +209,12 @@ def init_settings( TensorboardSettings().save_settings() print("Added adapters.tensorboard settings.") + if codex: + from flowcept.flowceptor.adapters.code_assistants.codex.codex_dataclasses import CodexSettings + + CodexSettings().save_settings() + print("Added adapters.codex settings.") + def _resolve_user_settings_path() -> Path: """Resolve writable user settings path honoring FLOWCEPT_SETTINGS_PATH.""" diff --git a/src/flowcept/commons/daos/docdb_dao/lmdb_dao.py b/src/flowcept/commons/daos/docdb_dao/lmdb_dao.py index c0b69527..ad2cfb81 100644 --- a/src/flowcept/commons/daos/docdb_dao/lmdb_dao.py +++ b/src/flowcept/commons/daos/docdb_dao/lmdb_dao.py @@ -715,7 +715,7 @@ def _ts(val): agent_ids = [a["agent_id"] for a in stored if "agent_id" in a] docs = ( self.task_query( - filter={"agent_id": {"$in": agent_ids}}, + filter={"$or": [{"agent_id": {"$in": agent_ids}}, {"source_agent_id": {"$in": agent_ids}}]}, projection=[ "agent_id", "activity_id", @@ -730,33 +730,32 @@ def _ts(val): stats_map: Dict = {} for doc in docs: - agent_id = doc.get("agent_id") - if not agent_id: - continue - record = stats_map.setdefault( - agent_id, - { - "task_count": 0, - "activities": set(), - "source_agent_ids": set(), - "campaign_ids": set(), - "workflow_ids": set(), - "last_active": None, - }, - ) - record["task_count"] += 1 - for key, field in ( - ("activities", "activity_id"), - ("source_agent_ids", "source_agent_id"), - ("campaign_ids", "campaign_id"), - ("workflow_ids", "workflow_id"), - ): - if doc.get(field): - record[key].add(doc[field]) - ts = _ts(doc.get("registered_at")) - if ts is not None: - current = record["last_active"] - record["last_active"] = ts if current is None else max(current, ts) + participants = [doc.get("agent_id"), doc.get("source_agent_id")] + for agent_id in {agent_id for agent_id in participants if agent_id in agent_ids}: + record = stats_map.setdefault( + agent_id, + { + "task_count": 0, + "activities": set(), + "source_agent_ids": set(), + "campaign_ids": set(), + "workflow_ids": set(), + "last_active": None, + }, + ) + record["task_count"] += 1 + for key, field in ( + ("activities", "activity_id"), + ("source_agent_ids", "source_agent_id"), + ("campaign_ids", "campaign_id"), + ("workflow_ids", "workflow_id"), + ): + if doc.get(field): + record[key].add(doc[field]) + ts = _ts(doc.get("registered_at")) + if ts is not None: + current = record["last_active"] + record["last_active"] = ts if current is None else max(current, ts) for record in stats_map.values(): for key in ("activities", "source_agent_ids", "campaign_ids", "workflow_ids"): record[key] = sorted(record[key]) diff --git a/src/flowcept/commons/daos/docdb_dao/mongodb_dao.py b/src/flowcept/commons/daos/docdb_dao/mongodb_dao.py index ead79557..be70ca86 100644 --- a/src/flowcept/commons/daos/docdb_dao/mongodb_dao.py +++ b/src/flowcept/commons/daos/docdb_dao/mongodb_dao.py @@ -2004,10 +2004,29 @@ def _ts(val): rows = ( self.raw_pipeline( [ - {"$match": {"agent_id": {"$in": agent_ids}}}, + { + "$match": { + "$or": [ + {"agent_id": {"$in": agent_ids}}, + {"source_agent_id": {"$in": agent_ids}}, + ] + } + }, + { + "$project": { + "participants": ["$agent_id", "$source_agent_id"], + "activity_id": 1, + "source_agent_id": 1, + "campaign_id": 1, + "workflow_id": 1, + "registered_at": 1, + } + }, + {"$unwind": "$participants"}, + {"$match": {"participants": {"$in": agent_ids}}}, { "$group": { - "_id": "$agent_id", + "_id": "$participants", "task_count": {"$sum": 1}, "activities": {"$addToSet": "$activity_id"}, "source_agent_ids": {"$addToSet": "$source_agent_id"}, diff --git a/src/flowcept/commons/flowcept_dataclasses/task_object.py b/src/flowcept/commons/flowcept_dataclasses/task_object.py index 6e830ed4..49e94fe8 100644 --- a/src/flowcept/commons/flowcept_dataclasses/task_object.py +++ b/src/flowcept/commons/flowcept_dataclasses/task_object.py @@ -37,7 +37,7 @@ class TaskObject: - ``"ai_model_invocation"`` (:attr:`~flowcept.commons.vocabulary.PROV_AGENT.AI_MODEL_INVOCATION`) — a single LLM prompt→response call. Captured automatically by :class:`~flowcept.instrumentation.flowcept_agent_task.FlowceptLLM`. - - ``"agent_tool"`` (:attr:`~flowcept.commons.vocabulary.PROV_AGENT.AGENT_TOOL`) — + - ``"tool_invocation"`` (:attr:`~flowcept.commons.vocabulary.PROV_AGENT.TOOL_INVOCATION`) — a tool execution by an AI agent. Captured automatically by the :func:`~flowcept.instrumentation.flowcept_agent_task.agent_flowcept_task` decorator. diff --git a/src/flowcept/commons/settings_factory.py b/src/flowcept/commons/settings_factory.py index f32c54e5..dfd18ea9 100644 --- a/src/flowcept/commons/settings_factory.py +++ b/src/flowcept/commons/settings_factory.py @@ -16,12 +16,16 @@ from flowcept.flowceptor.adapters.dask.dask_dataclasses import ( DaskSettings, ) +from flowcept.flowceptor.adapters.code_assistants.codex.codex_dataclasses import ( + CodexSettings, +) SETTINGS_CLASSES = { Vocabulary.Settings.MLFLOW_KIND: MLFlowSettings, Vocabulary.Settings.TENSORBOARD_KIND: TensorboardSettings, Vocabulary.Settings.DASK_KIND: DaskSettings, + Vocabulary.Settings.CODEX_KIND: CodexSettings, } diff --git a/src/flowcept/commons/vocabulary.py b/src/flowcept/commons/vocabulary.py index 1e1d2c77..9db48084 100644 --- a/src/flowcept/commons/vocabulary.py +++ b/src/flowcept/commons/vocabulary.py @@ -16,6 +16,7 @@ class Settings: MLFLOW_KIND = "mlflow" TENSORBOARD_KIND = "tensorboard" DASK_KIND = "dask" + CODEX_KIND = "codex" class Status(str, Enum): @@ -85,7 +86,7 @@ class PROV_AGENT(str, Enum): ``ResponseData`` (entity) *wasGeneratedBy* the invocation. The invocation *wasAssociatedWith* the ``AIAgent``. - - **AgentTool** *used* tool input arguments (``DomainData``). + - **ToolInvocation** *used* tool input arguments (``DomainData``). Return values *wasGeneratedBy* the tool call. The tool *wasAssociatedWith* the ``AIAgent``. An ``AIModelInvocation`` that the tool triggers *wasInformedBy* the tool @@ -96,7 +97,7 @@ class PROV_AGENT(str, Enum): ----- >>> from flowcept.commons.vocabulary import PROV_AGENT >>> task_obj.subtype = PROV_AGENT.AI_MODEL_INVOCATION - >>> task_obj.subtype = PROV_AGENT.AGENT_TOOL + >>> task_obj.subtype = PROV_AGENT.TOOL_INVOCATION """ AI_MODEL_INVOCATION = "ai_model_invocation" @@ -108,11 +109,39 @@ class PROV_AGENT(str, Enum): ``custom_metadata.response_metadata``. """ - AGENT_TOOL = "agent_tool" - """A tool execution by an AI agent (``AgentTool`` in PROV-AGENT). + TOOL_INVOCATION = "tool_invocation" + """A tool execution by an AI agent (``ToolInvocation`` in PROV-AGENT). Captured automatically by the :func:`~flowcept.instrumentation.flowcept_agent_task.agent_flowcept_task` decorator (applied to MCP tools and LangGraph tool nodes). Recorded fields: ``used`` = tool input arguments, ``generated`` = tool return value. """ + + +class PROV_AGENT_LOOP(str, Enum): + """Activity subtype vocabulary for PROV-AGENT-LOOP workflows. + + PROV-AGENT-LOOP extends PROV-AGENT with activities that describe the + structure of an agentic loop. Flowcept records these values as ``subtype`` + on :class:`~flowcept.commons.flowcept_dataclasses.workflow_object.WorkflowObject` + or :class:`~flowcept.commons.flowcept_dataclasses.task_object.TaskObject`. + Entities consumed or generated by these activities, such as plans, + checkpoints, messages, criteria, and results, should be represented in + ``used``, ``generated``, or ``custom_metadata`` rather than as task subtypes. + """ + + SESSION = "session" + """Runtime conversational workflow.""" + + EXECUTION_PLAN = "execution_plan" + """Executable plan workflow inside a session.""" + + PLAN_STEP_EXECUTION = "plan_step_execution" + """Execution activity for one step of an execution plan.""" + + LOOP_ITERATION = "loop_iteration" + """One agentic loop cycle.""" + + EVALUATION = "evaluation" + """Activity that checks progress, correctness, safety, limits, or completion.""" diff --git a/src/flowcept/flowceptor/adapters/base_interceptor.py b/src/flowcept/flowceptor/adapters/base_interceptor.py index 59c7b0fc..240f9923 100644 --- a/src/flowcept/flowceptor/adapters/base_interceptor.py +++ b/src/flowcept/flowceptor/adapters/base_interceptor.py @@ -42,6 +42,10 @@ def build(kind: str) -> "BaseInterceptor": from flowcept.flowceptor.adapters.tensorboard.tensorboard_interceptor import TensorboardInterceptor return TensorboardInterceptor() + elif kind == "codex": + from flowcept.flowceptor.adapters.code_assistants.codex.codex_interceptor import CodexInterceptor + + return CodexInterceptor() elif kind == "broker_mqtt": from flowcept.flowceptor.adapters.brokers.mqtt_interceptor import MQTTBrokerInterceptor diff --git a/src/flowcept/flowceptor/adapters/code_assistants/__init__.py b/src/flowcept/flowceptor/adapters/code_assistants/__init__.py new file mode 100644 index 00000000..ad188a8f --- /dev/null +++ b/src/flowcept/flowceptor/adapters/code_assistants/__init__.py @@ -0,0 +1 @@ +"""Code assistant adapter subpackages.""" diff --git a/src/flowcept/flowceptor/adapters/code_assistants/codex/__init__.py b/src/flowcept/flowceptor/adapters/code_assistants/codex/__init__.py new file mode 100755 index 00000000..1b285034 --- /dev/null +++ b/src/flowcept/flowceptor/adapters/code_assistants/codex/__init__.py @@ -0,0 +1 @@ +"""Codex subpackage.""" diff --git a/src/flowcept/flowceptor/adapters/code_assistants/codex/codex_dataclasses.py b/src/flowcept/flowceptor/adapters/code_assistants/codex/codex_dataclasses.py new file mode 100644 index 00000000..65e91ba3 --- /dev/null +++ b/src/flowcept/flowceptor/adapters/code_assistants/codex/codex_dataclasses.py @@ -0,0 +1,26 @@ +"""Codex adapter settings.""" + +from dataclasses import dataclass + +from flowcept.commons.flowcept_dataclasses.base_settings_dataclasses import ( + BaseSettings, +) + + +@dataclass +class CodexSettings(BaseSettings): + """Codex session log settings.""" + + key: str = "codex" + kind: str = "codex" + file_path: str = "codex_events.jsonl" + watch_interval_sec: int = 1 + recursive: bool = True + include_developer_messages: bool = True + include_reasoning: bool = True + declared_provenance_enabled: bool = False + + def __post_init__(self): + """Set runtime observer metadata.""" + self.observer_type = "file" + self.observer_subtype = "jsonl" diff --git a/src/flowcept/flowceptor/adapters/code_assistants/codex/codex_interceptor.py b/src/flowcept/flowceptor/adapters/code_assistants/codex/codex_interceptor.py new file mode 100644 index 00000000..4f841f2c --- /dev/null +++ b/src/flowcept/flowceptor/adapters/code_assistants/codex/codex_interceptor.py @@ -0,0 +1,2523 @@ +"""Codex session log interceptor.""" + +from __future__ import annotations + +import json +import re +import unicodedata +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from time import sleep, time +from typing import Any + +from flowcept.commons.flowcept_dataclasses.agent_object import AgentObject +from flowcept.commons.flowcept_dataclasses.task_object import TaskObject +from flowcept.commons.flowcept_dataclasses.workflow_object import WorkflowObject +from flowcept.commons.utils import get_utc_now +from flowcept.commons.vocabulary import PROV_AGENT, PROV_AGENT_LOOP, Status +from flowcept.flowceptor.adapters.base_interceptor import BaseInterceptor + + +JsonObject = dict[str, Any] + +PLAN_SUMMARY_SECTION_NAMES = { + "resumo", + "summary", +} + +PLAN_EVALUATION_SECTION_NAMES = { + "verificacao", + "verificacoes", + "verification", + "verifications", + "validation", + "validacao", + "test", + "tests", + "testing", + "teste", + "testes", + "evaluation", + "avaliacao", + "evaluation_criteria", + "criterios_de_avaliacao", + "criterios_avaliacao", +} + +PLAN_STEP_SECTION_NAMES = { + "steps", + "step", + "etapas", + "etapa", + "passos", + "passo", +} + +FLOWCEPT_MESSAGE_EVENT_TYPES = { + "user_prompt", + "agent_response", + "thought", + "observation", + "belief", + "decision", + "mandate", + "message", + "checkpoint", + "memory", + "lesson_learned", +} + +FLOWCEPT_ENTITY_EVENT_TYPES = { + "objective", + "plan", + "checkpoint", + "evaluation_criteria", + "evaluation_result", +} + +FLOWCEPT_TASK_EVENT_TYPES = { + "plan_step_started", + "plan_step_finished", + "loop_iteration_started", + "loop_iteration_finished", + "evaluation_started", + "evaluation_finished", +} + +DPL_MESSAGE_CLASSES = { + "Thought", + "Observation", + "Belief", + "Decision", + "Mandate", + "Memory", + "LessonLearned", +} + +DPL_ENTITY_CLASSES = { + "Objective", + "Plan", + "Checkpoint", + "EvaluationCriteria", + "EvaluationResult", +} + +DPL_TASK_CLASSES = { + "PlanStepExecution", + "LoopIteration", + "Evaluation", +} + +DPL_CLASS_TO_MESSAGE_TYPE = { + "Thought": "thought", + "Observation": "observation", + "Belief": "belief", + "Decision": "decision", + "Mandate": "mandate", + "Memory": "memory", + "LessonLearned": "lesson_learned", +} + +DPL_CLASS_TO_ENTITY_TYPE = { + "Objective": "objective", + "Plan": "plan", + "Checkpoint": "checkpoint", + "EvaluationCriteria": "evaluation_criteria", + "EvaluationResult": "evaluation_result", +} + +DPL_CLASS_TO_TASK_PREFIX = { + "PlanStepExecution": "plan_step", + "LoopIteration": "loop_iteration", + "Evaluation": "evaluation", +} + +DPL_CLASS_ALIASES = { + "thought": "Thought", + "observation": "Observation", + "belief": "Belief", + "decision": "Decision", + "mandate": "Mandate", + "memory": "Memory", + "lessonlearned": "LessonLearned", + "objective": "Objective", + "plan": "Plan", + "checkpoint": "Checkpoint", + "evaluationcriteria": "EvaluationCriteria", + "evaluationcriterion": "EvaluationCriteria", + "evaluationresult": "EvaluationResult", + "planstepexecution": "PlanStepExecution", + "planstep": "PlanStepExecution", + "loopiteration": "LoopIteration", + "evaluation": "Evaluation", +} + +TOOL_DATA_SCHEDULING_PATTERNS = re.compile( + r"\b(" + r"slurm|sbatch|squeue|sacct|scontrol|scancel|sstat|qsub|qstat|qdel|pbs|lsf|bsub|bjobs|job_id|" + r"allocation|accounting|node-hours?|queue|partition" + r")\b", + re.IGNORECASE, +) + +TOOL_DATA_TELEMETRY_PATTERNS = re.compile( + r"\b(" + r"telemetry|nvidia-smi|rocm-smi|amd-smi|amdsmi|gpu|cpu|memory|mem|ram|vram|utilization|" + r"power|energy|temperature|throughput|latency|iostat|vmstat|top|psutil" + r")\b", + re.IGNORECASE, +) + + +@dataclass +class ToolInvocationState: + """State for a Codex function_call/function_call_output pair.""" + + call_id: str + task_id: str + tool_name: str | None + arguments: Any + started_at: float | None + workflow_id: str | None = None + parent_task_id: str | None = None + response_item_id: str | None = None + ended_at: float | None = None + output: Any = None + emitted: bool = False + + +@dataclass +class TaggedTaskState: + """State for explicit flowcept_event start/finish task boundaries.""" + + task_id: str + subtype: str + activity_id: str + workflow_id: str | None + parent_task_id: str | None + agent_id: str | None + source_agent_id: str | None + started_at: float | None + used_messages: list[JsonObject] = field(default_factory=list) + used_entities: list[JsonObject] = field(default_factory=list) + generated_messages: list[JsonObject] = field(default_factory=list) + generated_entities: list[JsonObject] = field(default_factory=list) + metadata: JsonObject = field(default_factory=dict) + emitted_started: bool = False + + +@dataclass +class ModelInvocationState: + """State for a Codex model invocation within a turn.""" + + task_id: str + turn_id: str + workflow_id: str | None + parent_task_id: str | None + agent_id: str | None + source_agent_id: str | None + started_at: float | None + prompt: str | None = None + ai_model: JsonObject = field(default_factory=dict) + messages: list[JsonObject] = field(default_factory=list) + used_entities: list[JsonObject] = field(default_factory=list) + generated_messages: list[JsonObject] = field(default_factory=list) + generated_entities: list[JsonObject] = field(default_factory=list) + plans: list[JsonObject] = field(default_factory=list) + response: str | None = None + token_usage: JsonObject | None = None + ended_at: float | None = None + tools: dict[str, ToolInvocationState] = field(default_factory=dict) + emitted_started: bool = False + emitted: bool = False + + +@dataclass +class TurnState: + """State for one Codex turn, mapped to a loop iteration.""" + + task_id: str + workflow_id: str | None + agent_id: str | None + source_agent_id: str | None + started_at: float | None + submitted_at: float | None = None + ended_at: float | None = None + prompt: str | None = None + final_response: str | None = None + metadata: JsonObject = field(default_factory=dict) + mandates: list[JsonObject] = field(default_factory=list) + messages: list[JsonObject] = field(default_factory=list) + invocations: list[ModelInvocationState] = field(default_factory=list) + current_invocation: ModelInvocationState | None = None + emitted: bool = False + + +def _compact_dict(data: JsonObject) -> JsonObject: + return {key: value for key, value in data.items() if value is not None} + + +def _epoch_seconds(value: Any) -> float | None: + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str): + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except ValueError: + return None + + +def _parse_json_or_raw(value: Any) -> Any: + if not isinstance(value, str): + return value + try: + return json.loads(value) + except json.JSONDecodeError: + return value + + +def _message_text(payload: JsonObject) -> str | None: + content = payload.get("content") or [] + parts = [] + for item in content: + if isinstance(item, dict): + text = item.get("text") + if text: + parts.append(text) + return "\n".join(parts) if parts else None + + +def _extract_proposed_plans(text: str | None) -> list[str]: + if not text: + return [] + return [ + match.group(1).strip() + for match in re.finditer(r"(.*?)", text, flags=re.DOTALL) + if match.group(1).strip() + ] + + +def _extract_flowcept_events(text: str | None) -> list[JsonObject]: + if not text: + return [] + text = re.sub(r".*?", "", text, flags=re.DOTALL) + events = [] + for match in re.finditer(r"(.*?)", text, flags=re.DOTALL): + raw_event = match.group(1).strip() + if not raw_event: + continue + try: + event = json.loads(raw_event) + except json.JSONDecodeError: + continue + if isinstance(event, dict) and (event.get("type") or (event.get("layer") == "DPL" and event.get("class"))): + events.append(event) + return events + + +def _normalize_declared_event(event: JsonObject) -> JsonObject | None: + if event.get("layer") == "DPL" and event.get("class"): + raw_class = str(event.get("class") or "") + class_key = re.sub(r"[\s_-]+", "", raw_class).lower() + event_class = DPL_CLASS_ALIASES.get(class_key, raw_class) + event_name = event.get("event") + if event_class in DPL_MESSAGE_CLASSES: + normalized = {key: value for key, value in event.items() if key not in {"layer", "class", "event"}} + normalized["type"] = DPL_CLASS_TO_MESSAGE_TYPE[event_class] + normalized["classification_source"] = "declared" + return normalized + if event_class in DPL_ENTITY_CLASSES: + normalized = {key: value for key, value in event.items() if key not in {"layer", "class", "event"}} + normalized["type"] = DPL_CLASS_TO_ENTITY_TYPE[event_class] + normalized["classification_source"] = "declared" + return normalized + if event_class in DPL_TASK_CLASSES and event_name in {"started", "finished"}: + normalized = {key: value for key, value in event.items() if key not in {"layer", "class", "event"}} + normalized["type"] = f"{DPL_CLASS_TO_TASK_PREFIX[event_class]}_{event_name}" + normalized["classification_source"] = "declared" + return normalized + return None + if event.get("type"): + normalized = dict(event) + normalized.setdefault("classification_source", "declared") + return normalized + return None + + +def _strip_flowcept_events(text: str | None) -> str | None: + if text is None: + return None + stripped = re.sub(r".*?", "", text, flags=re.DOTALL).strip() + return stripped or None + + +def _is_flowcept_event_only_text(text: str | None) -> bool: + return bool(_extract_flowcept_events(text)) and _strip_flowcept_events(text) is None + + +def _normalize_plan_content(content: str | None) -> str | None: + if content is None: + return None + return content.strip() + + +def _normalized_section_name(name: str) -> str: + normalized = unicodedata.normalize("NFKD", name.strip().lower()) + ascii_name = "".join(char for char in normalized if not unicodedata.combining(char)) + return re.sub(r"[^a-z0-9]+", "_", ascii_name).strip("_") + + +def _parse_plan_content(content: str) -> JsonObject: + title = None + current_section = None + sections: dict[str, list[str]] = {} + raw_sections: dict[str, str] = {} + + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line: + continue + heading = re.fullmatch(r"\*\*(.+?)\*\*", line) + if heading: + heading_text = heading.group(1).strip() + if title is None: + title = heading_text + continue + current_section = _normalized_section_name(heading_text) + raw_sections[current_section] = heading_text + sections.setdefault(current_section, []) + continue + if current_section is None: + continue + bullet = re.match(r"^[-*]\s+(.*)$", line) + sections[current_section].append((bullet.group(1) if bullet else line).strip()) + + return _compact_dict( + { + "title": title, + "sections": {key: value for key, value in sections.items() if value}, + "section_names": raw_sections or None, + } + ) + + +def _section_items(sections: JsonObject, aliases: set[str]) -> list[str]: + items = [] + for name, section_items in sections.items(): + if name in aliases: + items.extend(section_items) + return items + + +def _plan_summary(plan: JsonObject) -> str | None: + summary_items = _section_items(plan.get("sections") or {}, PLAN_SUMMARY_SECTION_NAMES) + return "\n".join(summary_items) if summary_items else None + + +def _canonical_plan_step_label(label: str | None) -> str | None: + if not label: + return None + label = label.strip() + backtick = re.match(r"`([^`]+)`", label) + if backtick: + label = backtick.group(1) + label = re.sub(r"^\*\*(.+?)\*\*$", r"\1", label).strip() + normalized = unicodedata.normalize("NFKD", label.lower()) + ascii_label = "".join(char for char in normalized if not unicodedata.combining(char)) + canonical = re.sub(r"[^a-z0-9]+", " ", ascii_label).strip() + return canonical or None + + +def _plan_step_labels(plan: JsonObject) -> set[str]: + content = plan.get("content") + if not isinstance(content, str): + return set() + current_section = None + labels: set[str] = set() + for raw_line in content.splitlines(): + stripped = raw_line.strip() + if not stripped: + continue + heading = re.fullmatch(r"\*\*(.+?)\*\*", stripped) + if heading: + current_section = _normalized_section_name(heading.group(1)) + continue + if current_section not in PLAN_STEP_SECTION_NAMES: + continue + bullet = re.match(r"^[-*]\s+(.*)$", raw_line) + if not bullet: + continue + label = _canonical_plan_step_label(bullet.group(1)) + if label: + labels.add(label) + return labels + + +def _task_status(ended_at: float | None, output: Any = None) -> Status: + if ended_at is None: + return Status.UNKNOWN + if isinstance(output, str): + exit_codes = re.findall(r"Process exited with code (\d+)", output) + if exit_codes and any(code != "0" for code in exit_codes): + return Status.ERROR + if '"interrupted":true' in output or "'interrupted': True" in output: + return Status.ERROR + if isinstance(output, dict) and (output.get("interrupted") or output.get("is_error")): + return Status.ERROR + return Status.FINISHED + + +def _entity(kind: str, **kwargs) -> JsonObject: + return _compact_dict({"type": kind, **kwargs}) + + +def _message(content: str | None, attributed_to: str | None, role: str | None = None, **kwargs) -> JsonObject: + return _entity("message", content=content, attributed_to=attributed_to, role=role, **kwargs) + + +def _thought(content: str | None, attributed_to: str | None, role: str | None = None, **kwargs) -> JsonObject: + return _entity("thought", content=content, attributed_to=attributed_to, role=role, **kwargs) + + +def _mandate(content: str | None, attributed_to: str | None, role: str | None = None, **kwargs) -> JsonObject: + return _entity("mandate", content=content, attributed_to=attributed_to, role=role, **kwargs) + + +def _is_context_message(content: str | None) -> bool: + if content is None: + return False + stripped = content.strip() + return stripped.startswith("") or stripped.startswith("") + + +def _is_injected_context_message(content: str | None) -> bool: + if content is None: + return False + stripped = content.strip() + return ( + stripped.startswith("") + or stripped.startswith("") + or stripped.startswith("") + ) + + +class CodexInterceptor(BaseInterceptor): + """Interceptors Codex JSONL session logs into Flowcept provenance records.""" + + def __init__(self, plugin_key: str = "codex"): + super().__init__(plugin_key) + self._observer: Any | None = None + self._processed_lines: dict[str, int] = {} + self._session_id: str | None = None + self._codex_agent_id: str | None = None + self._human_agent_id: str | None = None + self._model_provider: str | None = None + self._current_turn_id: str | None = None + self._active_execution_plan_workflow_id: str | None = None + self._pending_execution_plan_workflow_id: str | None = None + self._pending_execution_plan_turn_id: str | None = None + self._execution_plan_criteria: dict[str, list[JsonObject]] = {} + self._execution_plan_steps: dict[str, set[str]] = {} + self._execution_plan_finished_steps: dict[str, set[str]] = {} + self._tagged_task_counts: dict[str, int] = {} + self._active_tagged_tasks: dict[tuple[str, str], TaggedTaskState] = {} + self._processed_declared_events: set[str] = set() + self._emitted_task_bounds: dict[str, tuple[float | None, float | None]] = {} + self._turns: dict[str, TurnState] = {} + self._emitted_records_count = 0 + + def callback(self) -> int: + """Read newly appended Codex JSONL events and emit Flowcept messages.""" + sleep(self.settings.watch_interval_sec) + intercepted = 0 + for path in self._log_paths(): + intercepted += self._process_log(path) + return intercepted + + def intercept(self, obj_msg: JsonObject): + """Intercept and count emitted Codex provenance records.""" + obj_msg.setdefault("custom_metadata", {}) + if isinstance(obj_msg["custom_metadata"], dict): + obj_msg["custom_metadata"].setdefault("flowcept_capture_observed_at", time()) + super().intercept(obj_msg) + self._emitted_records_count += 1 + + def start(self, bundle_exec_id, check_safe_stops: bool = True) -> "CodexInterceptor": + """Start observing Codex logs.""" + super().start(bundle_exec_id, check_safe_stops) + self.observe() + return self + + def stop(self, check_safe_stops: bool = True) -> bool: + """Stop observing Codex logs.""" + self.logger.debug("Codex interceptor stopping...") + if self._observer is not None: + self._observer.stop() + self._observer.join(timeout=1) + try: + self.callback() + self._flush_open_records() + except Exception as e: + self.logger.exception(e) + super().stop(check_safe_stops) + self.logger.debug("Codex interceptor stopped.") + return True + + def observe(self): + """Observe Codex log files.""" + from watchdog.observers.polling import PollingObserver + + from flowcept.flowceptor.adapters.mlflow.interception_event_handler import ( + InterceptionEventHandler, + ) + + watch_path = Path(self.settings.file_path) + parent_to_watch = watch_path if watch_path.is_dir() else watch_path.parent + while not parent_to_watch.exists(): + self.logger.debug(f"I can't watch {parent_to_watch}, as it does not exist.") + sleep(self.settings.watch_interval_sec) + + event_handler = InterceptionEventHandler(self, str(parent_to_watch), self.__class__.callback) + self._observer = PollingObserver() + self._observer.schedule(event_handler, str(parent_to_watch), recursive=self.settings.recursive) + self._observer.start() + sleep(0.2) + self.logger.debug(f"Watching Codex logs under {parent_to_watch}") + self.callback() + + def _log_paths(self) -> list[Path]: + path = Path(self.settings.file_path) + if path.is_file(): + return [path] + if not path.exists(): + return [] + if path.is_dir(): + return sorted(item for item in path.rglob("*.jsonl") if item.is_file()) + return [] + + def _process_log(self, path: Path) -> int: + path_key = str(path.resolve()) + processed = self._processed_lines.get(path_key, 0) + intercepted_before = self._emitted_records_count + with path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if line_number <= processed: + continue + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + self.logger.warning(f"Skipping invalid Codex JSONL line {line_number} in {path}: {exc}") + continue + self._map_event(event, line_number, path) + self._processed_lines[path_key] = line_number + return self._emitted_records_count - intercepted_before + + def _map_event(self, event: JsonObject, line_number: int, path: Path): + event_type = event.get("type") + payload = event.get("payload") or {} + timestamp = _epoch_seconds(event.get("timestamp")) + if event_type == "session_meta": + self._handle_session_meta(payload, timestamp, line_number, path) + elif event_type == "turn_context": + self._handle_turn_context(payload) + elif event_type == "event_msg": + self._handle_event_msg(payload, timestamp, line_number) + elif event_type == "response_item": + self._handle_response_item(payload, timestamp, line_number) + + def _handle_session_meta(self, payload: JsonObject, timestamp: float | None, line_number: int, path: Path): + session_id = payload.get("session_id") or payload.get("id") + if not session_id: + return + self._session_id = session_id + self._codex_agent_id = f"codex:{session_id}" + self._human_agent_id = f"human:{session_id}" + self._model_provider = payload.get("model_provider") + + workflow_obj = WorkflowObject(workflow_id=session_id, name=payload.get("title") or session_id) + workflow_obj.subtype = PROV_AGENT_LOOP.SESSION.value + workflow_obj.campaign_id = self._campaign_id() + workflow_obj.started_at = _epoch_seconds(payload.get("timestamp")) or timestamp + workflow_obj.status = Status.UNKNOWN + workflow_obj.used = { + "entities": [ + _entity( + "mandate", + content=(payload.get("base_instructions") or {}).get("text"), + attributed_to=self._human_agent_id, + ) + ] + } + workflow_obj.workflow_description = "Codex session imported from a Codex JSONL transcript." + self._send_workflow_message(workflow_obj) + self.send_agent_message(self._agent_obj(self._codex_agent_id, payload.get("originator") or "Codex")) + self.send_agent_message(self._agent_obj(self._human_agent_id, "Human")) + + def _handle_turn_context(self, payload: JsonObject): + turn_id = payload.get("turn_id") or self._current_turn_id + turn = self._turns.get(turn_id) if turn_id else None + if not turn: + return + collaboration_mode = payload.get("collaboration_mode") or {} + sandbox_policy = payload.get("sandbox_policy") or {} + turn.metadata.update( + _compact_dict( + { + "turn_id": turn_id, + "cwd": payload.get("cwd"), + "model": payload.get("model"), + "effort": payload.get("effort"), + "approval_policy": payload.get("approval_policy"), + "sandbox_policy": sandbox_policy, + "collaboration_mode_kind": collaboration_mode.get("mode"), + "workspace_roots": payload.get("workspace_roots"), + "current_date": payload.get("current_date"), + "timezone": payload.get("timezone"), + } + ) + ) + for invocation in turn.invocations: + self._apply_model_metadata(invocation, turn) + + def _handle_event_msg(self, payload: JsonObject, timestamp: float | None, line_number: int): + payload_type = payload.get("type") + if payload_type == "task_started": + self._start_turn(payload, timestamp, line_number) + elif payload_type == "user_message": + self._set_turn_prompt(payload.get("message"), timestamp) + elif payload_type == "agent_message": + if payload.get("phase") == "final_answer": + turn = self._current_turn() + if turn: + turn.final_response = _strip_flowcept_events(payload.get("message")) or turn.final_response + return + turn = self._current_turn() + if _is_flowcept_event_only_text(payload.get("message")) and turn and not turn.current_invocation: + if self.settings.declared_provenance_enabled: + self._handle_declared_annotation_text(payload.get("message"), timestamp) + return + invocation = self._ensure_invocation(timestamp) + if invocation: + self._add_assistant_text(invocation, payload.get("message"), payload.get("phase"), timestamp) + elif payload_type == "item_completed": + self._handle_item_completed(payload, timestamp) + elif payload_type == "token_count": + self._close_invocation_with_tokens(payload, timestamp) + elif payload_type == "task_complete": + self._complete_turn(payload, timestamp, line_number) + elif payload_type == "turn_aborted": + self._abort_turn(payload, timestamp, line_number) + + def _handle_response_item(self, payload: JsonObject, timestamp: float | None, line_number: int): + metadata = payload.get("internal_chat_message_metadata_passthrough") or {} + turn_id = metadata.get("turn_id") + if turn_id in self._turns: + self._current_turn_id = turn_id + + payload_type = payload.get("type") + if payload_type == "message": + self._handle_message(payload, timestamp, line_number) + elif payload_type == "reasoning" and self.settings.include_reasoning: + invocation = self._ensure_invocation(timestamp) + if invocation: + self._append_unique_message( + invocation.generated_messages, + _entity( + "thought", + content=payload.get("summary") or payload.get("content"), + encrypted_content=payload.get("encrypted_content"), + attributed_to=invocation.agent_id, + ), + ) + elif payload_type == "function_call": + self._handle_function_call(payload, timestamp) + elif payload_type == "function_call_output": + self._handle_function_call_output(payload, timestamp) + elif payload_type == "custom_tool_call": + self._handle_custom_tool_call(payload, timestamp) + elif payload_type == "custom_tool_call_output": + self._handle_function_call_output(payload, timestamp) + + def _start_turn(self, payload: JsonObject, timestamp: float | None, line_number: int): + turn_id = payload.get("turn_id") + if not turn_id: + return + self._current_turn_id = turn_id + started_at = _epoch_seconds(payload.get("started_at")) or timestamp + turn = TurnState( + task_id=turn_id, + workflow_id=self._session_id, + agent_id=self._codex_agent_id, + source_agent_id=self._human_agent_id, + started_at=started_at, + metadata=_compact_dict( + { + "turn_id": turn_id, + "collaboration_mode_kind": payload.get("collaboration_mode_kind"), + "model_context_window": payload.get("model_context_window"), + "line_number": line_number, + } + ), + ) + self._turns[turn_id] = turn + + def _set_turn_prompt(self, prompt: str | None, timestamp: float | None): + turn = self._current_turn() + if not turn or prompt is None: + return + if _is_injected_context_message(prompt): + return + if _is_context_message(prompt): + mandate = _mandate(prompt, attributed_to=turn.agent_id, role="user") + self._append_unique_message(turn.mandates, mandate) + if turn.current_invocation: + self._append_unique_message(turn.current_invocation.messages, mandate) + return + turn.prompt = prompt + turn.submitted_at = timestamp + message = _entity("user_prompt", content=prompt, attributed_to=turn.source_agent_id, role="user") + self._append_unique_message(turn.messages, message) + invocation = self._ensure_invocation(timestamp) + if invocation and (invocation.prompt is None or _is_context_message(invocation.prompt)): + invocation.prompt = prompt + self._append_unique_message(invocation.messages, message) + + def _handle_message(self, payload: JsonObject, timestamp: float | None, line_number: int): + role = payload.get("role") + text = _message_text(payload) + if not text: + return + turn = self._current_turn() + if role == "user": + self._set_turn_prompt(text, timestamp) + elif role == "developer" and self.settings.include_developer_messages and turn: + if _is_context_message(text): + mandate = _mandate(text, attributed_to=turn.agent_id, role=role) + self._append_unique_message(turn.mandates, mandate) + if turn.current_invocation: + self._append_unique_message(turn.current_invocation.messages, mandate) + else: + message = _message( + text, + attributed_to=turn.source_agent_id, + role=role, + ) + self._append_unique_message(turn.messages, message) + if turn.current_invocation: + self._append_unique_message(turn.current_invocation.messages, message) + elif role == "assistant": + if _is_flowcept_event_only_text(text) and turn and not turn.current_invocation: + if self.settings.declared_provenance_enabled: + self._handle_declared_annotation_text(text, timestamp) + return + invocation = self._ensure_invocation(timestamp) + if invocation: + self._add_assistant_text(invocation, text, payload.get("phase"), timestamp) + + def _handle_item_completed(self, payload: JsonObject, timestamp: float | None): + item = payload.get("item") or {} + if item.get("type") != "Plan": + return + invocation = self._ensure_invocation(timestamp) + if invocation: + self._register_plan( + invocation, + content=item.get("text"), + item_id=item.get("id"), + timestamp=timestamp, + ) + + def _add_assistant_text( + self, + invocation: ModelInvocationState, + text: str | None, + phase: str | None, + timestamp: float | None, + ): + if not text: + return + if self.settings.declared_provenance_enabled: + for event in _extract_flowcept_events(text): + self._handle_flowcept_event(invocation, event, timestamp) + clean_text = _strip_flowcept_events(text) + for plan_text in _extract_proposed_plans(text): + self._register_plan(invocation, content=plan_text, item_id=None, timestamp=timestamp) + if phase == "final_answer": + invocation.response = clean_text + invocation.ended_at = timestamp or invocation.ended_at + elif clean_text: + message = _thought(clean_text, attributed_to=invocation.agent_id, role="assistant", phase=phase) + self._append_unique_message(invocation.generated_messages, message) + turn = self._turns.get(invocation.turn_id) + close_at = turn.metadata.pop("close_current_invocation_at", None) if turn else None + if close_at and turn and turn.current_invocation is invocation and not invocation.emitted: + invocation.ended_at = close_at + turn.current_invocation = None + + def _handle_declared_annotation_text(self, text: str | None, timestamp: float | None): + turn = self._current_turn() + if not turn: + return + invocation = self._last_invocation(turn) + if not invocation: + return + for event in _extract_flowcept_events(text): + self._handle_flowcept_event(invocation, event, timestamp) + + def _handle_flowcept_event( + self, + invocation: ModelInvocationState, + event: JsonObject, + timestamp: float | None, + ): + event = _normalize_declared_event(event) + if not event: + return + event_key = json.dumps( + { + "turn_id": invocation.turn_id, + "event": event, + }, + sort_keys=True, + default=str, + ) + if event_key in self._processed_declared_events: + return + self._processed_declared_events.add(event_key) + event_type = event.get("type") + if event_type in {"user_prompt", "agent_response"}: + return + if event_type in FLOWCEPT_MESSAGE_EVENT_TYPES: + self._handle_flowcept_message_event(invocation, event) + elif event_type in FLOWCEPT_ENTITY_EVENT_TYPES: + self._handle_flowcept_entity_event(invocation, event) + elif event_type in FLOWCEPT_TASK_EVENT_TYPES: + self._handle_flowcept_task_event(invocation, event, timestamp) + + def _handle_flowcept_message_event(self, invocation: ModelInvocationState, event: JsonObject): + event_type = event.get("type") + attributed_to = self._event_attributed_to(event, invocation) + message = _entity( + event_type, + **_compact_dict( + { + ("content" if key == "summary" and "content" not in event else key): value + for key, value in event.items() + if key not in {"type", "attributed_to"} + } + ), + attributed_to=attributed_to, + ) + if event_type == "user_prompt" and "role" not in message: + message["role"] = "user" + elif event_type == "agent_response" and "role" not in message: + message["role"] = "assistant" + if event_type == "user_prompt": + invocation.prompt = event.get("content") or invocation.prompt + self._append_unique_message(invocation.messages, message) + turn = self._turns.get(invocation.turn_id) + if turn: + turn.prompt = event.get("content") or turn.prompt + self._append_unique_message(turn.messages, message) + elif event_type in {"mandate", "objective"}: + self._append_unique_message(invocation.messages, message) + turn = self._turns.get(invocation.turn_id) + if turn and event_type == "mandate": + self._append_unique_message(turn.mandates, message) + elif turn: + self._append_unique_message(turn.messages, message) + elif event_type == "agent_response": + invocation.response = event.get("content") or invocation.response + turn = self._turns.get(invocation.turn_id) + if turn: + turn.final_response = event.get("content") or turn.final_response + else: + if not invocation.emitted: + self._append_unique_message(invocation.generated_messages, message) + else: + self._append_declared_entity_to_active_context(invocation, message, event_type) + + def _handle_flowcept_entity_event(self, invocation: ModelInvocationState, event: JsonObject): + event_type = event.get("type") + entity = _entity( + event_type, + **_compact_dict( + { + ("content" if key == "summary" and "content" not in event else key): value + for key, value in event.items() + if key != "type" + } + ), + attributed_to=self._event_attributed_to(event, invocation), + ) + if event_type == "plan" and entity.get("content"): + self._register_plan( + invocation, + content=entity.get("content"), + item_id=entity.get("item_id"), + timestamp=invocation.started_at, + ) + return + if not invocation.emitted: + self._append_unique_message(invocation.generated_entities, entity) + else: + self._append_declared_entity_to_active_context(invocation, entity, event_type) + + def _handle_flowcept_task_event( + self, + invocation: ModelInvocationState, + event: JsonObject, + timestamp: float | None, + ): + event_type = event.get("type") + if event_type.endswith("_started"): + self._start_tagged_task(invocation, event, timestamp) + elif event_type.endswith("_finished"): + self._finish_tagged_task(invocation, event, timestamp) + + def _start_tagged_task( + self, + invocation: ModelInvocationState, + event: JsonObject, + timestamp: float | None, + ): + subtype, activity_id = self._tagged_task_kind(event["type"]) + key = self._tagged_task_key(event) + if key in self._active_tagged_tasks: + return + workflow_id = self._workflow_for_tagged_task(invocation, subtype) + if subtype == PROV_AGENT_LOOP.LOOP_ITERATION.value: + self._start_observed_loop_context(invocation, event, workflow_id, timestamp) + return + started_at = timestamp or invocation.started_at + if started_at is not None and subtype == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value: + started_at -= 0.002 + parent_task_id = self._parent_for_tagged_task(subtype) + if started_at is not None and subtype == PROV_AGENT_LOOP.EVALUATION.value and parent_task_id is not None: + parent = self._active_tagged_task(PROV_AGENT_LOOP.LOOP_ITERATION.value) + if parent and parent.task_id == parent_task_id and parent.started_at is not None: + started_at = max(started_at, parent.started_at + 0.001) + task = TaggedTaskState( + task_id=self._next_tagged_task_id(invocation, subtype), + subtype=subtype, + activity_id=activity_id, + workflow_id=workflow_id, + parent_task_id=parent_task_id, + agent_id=invocation.agent_id, + source_agent_id=invocation.source_agent_id, + started_at=started_at, + used_messages=self._tagged_task_used_messages(event, invocation, subtype, workflow_id), + used_entities=self._tagged_task_used_entities(event, invocation, subtype, workflow_id), + metadata=self._tagged_task_metadata(event, invocation), + ) + self._active_tagged_tasks[key] = task + self._remember_tagged_context(invocation, task) + self._emit_tagged_task(task, timestamp, final=False) + + def _finish_tagged_task( + self, + invocation: ModelInvocationState, + event: JsonObject, + timestamp: float | None, + ): + start_type = event["type"].replace("_finished", "_started") + key = self._tagged_task_key({**event, "type": start_type}) + state = self._active_tagged_tasks.pop(key, None) + if state is not None and state.metadata.get("observed_loop_iteration"): + self._finish_observed_loop_context(invocation, event, state, timestamp) + return + if state is None: + subtype, activity_id = self._tagged_task_kind(event["type"]) + workflow_id = self._workflow_for_tagged_task(invocation, subtype) + if subtype == PROV_AGENT_LOOP.LOOP_ITERATION.value: + self._finish_observed_loop_context(invocation, event, None, timestamp) + return + state = TaggedTaskState( + task_id=self._next_tagged_task_id(invocation, subtype), + subtype=subtype, + activity_id=activity_id, + workflow_id=workflow_id, + parent_task_id=self._parent_for_tagged_task(subtype), + agent_id=invocation.agent_id, + source_agent_id=invocation.source_agent_id, + started_at=timestamp or invocation.started_at, + used_messages=self._tagged_task_used_messages(event, invocation, subtype, workflow_id), + used_entities=self._tagged_task_used_entities(event, invocation, subtype, workflow_id), + metadata=self._tagged_task_metadata(event, invocation), + ) + state.generated_messages.extend(self._tagged_task_generated_messages(event, invocation, state.subtype)) + state.generated_entities.extend(self._tagged_task_generated_entities(event, invocation, state.subtype)) + state.metadata.update(self._tagged_task_metadata(event, invocation)) + self._emit_tagged_task(state, timestamp, final=True) + self._clear_finished_tagged_context(invocation, state, timestamp) + + def _emit_tagged_task(self, state: TaggedTaskState, timestamp: float | None, final: bool = True): + if not final and state.emitted_started: + return + if not final and self._current_task_buffer() is None: + return + task = TaskObject() + task.task_id = state.task_id + task.workflow_id = state.workflow_id + task.parent_task_id = state.parent_task_id + task.campaign_id = self._campaign_id() + task.agent_id = state.agent_id + task.source_agent_id = state.source_agent_id + task.subtype = state.subtype + task.activity_id = state.metadata.get("label") or state.activity_id + task.used = _compact_dict( + { + "entities": (state.used_messages + state.used_entities) or None, + } + ) + if final: + task.generated = _compact_dict( + { + "entities": (state.generated_messages + state.generated_entities) or None, + } + ) + task.started_at = state.started_at + task.ended_at = (timestamp or state.started_at) if final else None + task.utc_timestamp = task.started_at + self._normalize_child_task_bounds(task) + task.status = _task_status(task.ended_at, state.metadata.get("result")) if final else Status.RUNNING + task.enrich(self.plugin_key) + self._intercept_task(task) + if not final: + state.emitted_started = True + + def _handle_function_call(self, payload: JsonObject, timestamp: float | None): + invocation = self._ensure_invocation(timestamp) + if not invocation: + return + self._apply_active_context_to_invocation(invocation) + self._emit_invocation_start(invocation) + call_id = payload.get("call_id") or payload.get("id") + if not call_id: + return + tool_name = payload.get("name") + arguments = _parse_json_or_raw(payload.get("arguments")) + invocation.tools[call_id] = ToolInvocationState( + call_id=call_id, + task_id=f"{invocation.task_id}:tool_invocation:{len(invocation.tools) + 1}", + tool_name=tool_name, + arguments=arguments, + started_at=timestamp, + workflow_id=invocation.workflow_id, + parent_task_id=invocation.parent_task_id, + response_item_id=payload.get("id"), + ) + if tool_name == "update_plan": + self._register_update_plan(invocation, arguments, timestamp) + + def _handle_custom_tool_call(self, payload: JsonObject, timestamp: float | None): + invocation = self._ensure_invocation(timestamp) + if not invocation: + return + self._apply_active_context_to_invocation(invocation) + self._emit_invocation_start(invocation) + call_id = payload.get("call_id") or payload.get("id") + if not call_id: + return + invocation.tools[call_id] = ToolInvocationState( + call_id=call_id, + task_id=f"{invocation.task_id}:tool_invocation:{len(invocation.tools) + 1}", + tool_name=payload.get("name") or "custom_tool_call", + arguments=_parse_json_or_raw(payload.get("input")), + started_at=timestamp, + workflow_id=invocation.workflow_id, + parent_task_id=invocation.parent_task_id, + response_item_id=payload.get("id"), + ended_at=timestamp if payload.get("status") == "completed" else None, + ) + + def _register_update_plan( + self, + invocation: ModelInvocationState, + arguments: Any, + timestamp: float | None, + ): + if self._active_execution_plan_workflow_id or self._pending_execution_plan_workflow_id: + return + if not isinstance(arguments, dict): + return + plan_items = arguments.get("plan") + if not isinstance(plan_items, list) or not plan_items: + return + lines = ["**Execution Plan**", "", "**Steps**"] + for item in plan_items: + if not isinstance(item, dict) or not item.get("step"): + continue + lines.append(f"- `{item['step']}`") + content = "\n".join(lines) + self._register_plan(invocation, content=content, item_id="codex:update_plan", timestamp=timestamp) + + def _handle_function_call_output(self, payload: JsonObject, timestamp: float | None): + call_id = payload.get("call_id") + invocation = self._find_invocation_for_call(call_id) or self._ensure_invocation(timestamp) + if not invocation or not call_id: + return + tool = invocation.tools.get(call_id) + if tool is None: + tool = ToolInvocationState( + call_id=call_id, + task_id=f"{invocation.task_id}:tool_invocation:{len(invocation.tools) + 1}", + tool_name=None, + arguments=None, + started_at=timestamp, + workflow_id=invocation.workflow_id, + parent_task_id=invocation.parent_task_id, + ) + invocation.tools[call_id] = tool + tool.output = _parse_json_or_raw(payload.get("output")) + tool.ended_at = timestamp + self._emit_tool(tool, invocation) + + def _close_invocation_with_tokens(self, payload: JsonObject, timestamp: float | None): + turn = self._current_turn() + invocation = turn.current_invocation if turn else None + if not invocation or not turn: + return + info = payload.get("info") or {} + usage = info.get("last_token_usage") or info + invocation.token_usage = self._normalize_llm_usage(usage, invocation, info) + invocation.ai_model["context_window"] = info.get("model_context_window") or invocation.ai_model.get( + "context_window" + ) + invocation.ended_at = timestamp + turn.current_invocation = None + + def _complete_turn(self, payload: JsonObject, timestamp: float | None, line_number: int): + turn_id = payload.get("turn_id") or self._current_turn_id + turn = self._turns.get(turn_id) if turn_id else None + if not turn: + return + turn.final_response = _strip_flowcept_events(payload.get("last_agent_message")) + turn.ended_at = _epoch_seconds(payload.get("completed_at")) or timestamp + turn.metadata.update( + _compact_dict( + { + "duration_ms": payload.get("duration_ms"), + "time_to_first_token_ms": payload.get("time_to_first_token_ms"), + "line_number_completed": line_number, + } + ) + ) + response_invocation = self._last_unemitted_invocation(turn) or turn.current_invocation + if response_invocation and turn.final_response: + response_invocation.response = response_invocation.response or turn.final_response + if turn.current_invocation: + turn.current_invocation.ended_at = turn.current_invocation.ended_at or turn.ended_at + turn.current_invocation = None + for invocation in turn.invocations: + if not invocation.emitted: + invocation.ended_at = invocation.ended_at or turn.ended_at + self._emit_invocation(invocation) + self._discard_empty_started_tagged_tasks() + self._emit_turn(turn) + self._activate_pending_execution_plan(turn.task_id) + self._current_turn_id = None + + def _abort_turn(self, payload: JsonObject, timestamp: float | None, line_number: int): + turn_id = payload.get("turn_id") or self._current_turn_id + turn = self._turns.get(turn_id) if turn_id else None + if not turn: + return + turn.ended_at = _epoch_seconds(payload.get("completed_at")) or timestamp + turn.metadata.update( + _compact_dict( + { + "interrupted": True, + "abort_reason": payload.get("reason"), + "line_number_aborted": line_number, + } + ) + ) + if turn.current_invocation: + turn.current_invocation.ended_at = turn.current_invocation.ended_at or turn.ended_at + if self._invocation_has_observed_work(turn.current_invocation): + self._emit_invocation(turn.current_invocation) + elif turn.current_invocation in turn.invocations: + turn.invocations.remove(turn.current_invocation) + turn.current_invocation = None + self._discard_empty_started_tagged_tasks() + self._emit_turn(turn) + self._activate_pending_execution_plan(turn.task_id) + self._current_turn_id = None + + def _ensure_invocation(self, timestamp: float | None) -> ModelInvocationState | None: + turn = self._current_turn() + if not turn: + return None + if turn.current_invocation: + self._apply_active_context_to_invocation(turn.current_invocation) + return turn.current_invocation + self._emit_closed_invocations(turn) + workflow_id = self._workflow_for_turn(turn) + fallback_loop_id = turn.task_id if self._should_emit_fallback_loop(turn, workflow_id) else None + invocation = ModelInvocationState( + task_id=f"{turn.task_id}:ai_model_invocation:{len(turn.invocations) + 1}", + turn_id=turn.task_id, + workflow_id=workflow_id, + parent_task_id=self._parent_for_observed_task( + PROV_AGENT.AI_MODEL_INVOCATION.value, + workflow_id=workflow_id, + fallback=fallback_loop_id, + ), + agent_id=turn.agent_id, + source_agent_id=turn.source_agent_id, + started_at=timestamp or turn.started_at, + prompt=turn.prompt, + messages=list(turn.mandates + turn.messages), + ) + self._apply_active_context_to_invocation(invocation) + self._apply_model_metadata(invocation, turn) + turn.invocations.append(invocation) + turn.current_invocation = invocation + return invocation + + def _last_unemitted_invocation(self, turn: TurnState) -> ModelInvocationState | None: + for invocation in reversed(turn.invocations): + if not invocation.emitted: + return invocation + return None + + def _last_invocation(self, turn: TurnState) -> ModelInvocationState | None: + return turn.invocations[-1] if turn.invocations else None + + def _emit_closed_invocations(self, turn: TurnState): + for invocation in turn.invocations: + if not invocation.emitted and invocation.ended_at is not None: + self._emit_invocation(invocation) + + def _apply_model_metadata(self, invocation: ModelInvocationState, turn: TurnState): + invocation.ai_model.update( + _compact_dict( + { + "type": "ai_model", + "model": turn.metadata.get("model"), + "provider": self._workflow_provider(), + "effort": turn.metadata.get("effort"), + "context_window": turn.metadata.get("model_context_window"), + } + ) + ) + + def _normalize_llm_usage(self, usage: JsonObject, invocation: ModelInvocationState, info: JsonObject) -> JsonObject: + return _compact_dict( + { + "model": invocation.ai_model.get("model"), + "input_tokens": usage.get("input_tokens"), + "output_tokens": usage.get("output_tokens"), + "total_tokens": usage.get("total_tokens"), + "cached_input_tokens": usage.get("cached_input_tokens"), + "reasoning_output_tokens": usage.get("reasoning_output_tokens"), + "model_context_window": info.get("model_context_window"), + "token_count_source": "codex_token_count", + } + ) + + def _emit_turn(self, turn: TurnState): + if turn.emitted: + return + workflow_id = self._workflow_for_turn(turn) + if not self._should_emit_fallback_loop(turn, workflow_id): + turn.emitted = True + return + if not any(self._invocation_has_observed_work(invocation) for invocation in turn.invocations): + turn.emitted = True + return + task = TaskObject() + task.task_id = turn.task_id + task.workflow_id = workflow_id + task.parent_task_id = self._parent_for_observed_loop(turn) + task.campaign_id = self._campaign_id() + task.agent_id = turn.agent_id + task.source_agent_id = turn.source_agent_id + task.subtype = PROV_AGENT_LOOP.LOOP_ITERATION.value + task.activity_id = "loop_iteration" + if turn.metadata.get("declared_loop_label"): + task.activity_id = turn.metadata.get("declared_loop_label") + if turn.metadata.get("declared_loop_seen"): + turn.emitted = True + return + task.used = _compact_dict( + { + "entities": (turn.mandates + turn.messages) or None, + } + ) + task.submitted_at = turn.submitted_at + task.started_at = turn.metadata.get("declared_loop_started_at") or turn.started_at + child_ended_at = [ + ended_at for invocation in turn.invocations for ended_at in [invocation.ended_at] if ended_at is not None + ] + task.ended_at = turn.metadata.get("declared_loop_finished_at") or max( + [ended_at for ended_at in [turn.ended_at, *child_ended_at] if ended_at is not None], + default=None, + ) + task.utc_timestamp = task.started_at + self._normalize_child_task_bounds(task) + task.status = self._loop_iteration_status(task.ended_at, turn.metadata) + task.enrich(self.plugin_key) + self._intercept_task(task) + turn.emitted = True + + def _emit_invocation(self, invocation: ModelInvocationState): + if invocation.emitted: + return + if not self._invocation_has_observed_work(invocation): + invocation.emitted = True + return + task = self._invocation_task(invocation, final=True) + self._intercept_task(task) + invocation.emitted_started = True + invocation.emitted = True + for tool in invocation.tools.values(): + if tool.ended_at is not None: + self._emit_tool(tool, invocation) + + def _emit_invocation_start(self, invocation: ModelInvocationState): + if invocation.emitted_started or invocation.emitted: + return + if self._current_task_buffer() is None: + return + if not self._invocation_has_observed_work(invocation): + return + task = self._invocation_task(invocation, final=False) + self._intercept_task(task) + invocation.emitted_started = True + + def _invocation_task(self, invocation: ModelInvocationState, final: bool): + response = None + if final: + response = ( + invocation.response + or self._response_from_generated_messages(invocation) + or self._response_from_tool_requests(invocation) + ) + task = TaskObject() + task.task_id = invocation.task_id + task.workflow_id = invocation.workflow_id + task.parent_task_id = invocation.parent_task_id + task.campaign_id = self._campaign_id() + task.agent_id = invocation.agent_id + task.source_agent_id = invocation.source_agent_id + task.subtype = PROV_AGENT.AI_MODEL_INVOCATION.value + task.activity_id = "ai_model_invocation" + task.used = _compact_dict( + { + "prompt": invocation.prompt or self._prompt_from_messages(invocation.messages), + "entities": ( + invocation.messages + + invocation.used_entities + + ([invocation.ai_model] if invocation.ai_model else []) + ) + or None, + } + ) + if final: + task.generated = _compact_dict( + { + "response": response, + "entities": ( + self._generated_invocation_messages(invocation) + + self._generated_invocation_entities(invocation) + ) + or None, + } + ) + task.started_at = invocation.started_at + task.ended_at = invocation.ended_at if final else None + task.utc_timestamp = task.started_at + self._normalize_child_task_bounds(task) + task.status = _task_status(invocation.ended_at) if final else Status.RUNNING + if final: + task.custom_metadata = _compact_dict( + { + "llm_usage": invocation.token_usage, + "response_metadata": { + "model": invocation.ai_model.get("model"), + "provider": invocation.ai_model.get("provider"), + "effort": invocation.ai_model.get("effort"), + "context_window": invocation.ai_model.get("context_window"), + }, + } + ) + task.enrich(self.plugin_key) + return task + + def _response_from_generated_messages(self, invocation: ModelInvocationState) -> str | None: + for message in reversed(invocation.generated_messages + invocation.generated_entities + invocation.plans): + content = message.get("content") + if content: + return str(content) + return None + + def _response_from_tool_requests(self, invocation: ModelInvocationState) -> str | None: + if not invocation.tools: + return None + names = [tool.tool_name or "tool_invocation" for tool in invocation.tools.values()] + if len(names) == 1: + return f"Requested tool invocation: {names[0]}" + return "Requested tool invocations: " + ", ".join(names) + + def _invocation_has_observed_work(self, invocation: ModelInvocationState) -> bool: + return bool( + invocation.tools + or invocation.generated_messages + or invocation.generated_entities + or invocation.plans + or invocation.response + or invocation.token_usage + ) + + def _generated_invocation_messages(self, invocation: ModelInvocationState) -> list[JsonObject]: + messages = [] + if invocation.response: + messages.append( + _entity( + "agent_response", + content=invocation.response, + attributed_to=invocation.agent_id, + role="assistant", + ) + ) + messages.extend(invocation.generated_messages) + return messages + + def _generated_invocation_entities(self, invocation: ModelInvocationState) -> list[JsonObject]: + return list(invocation.generated_entities + invocation.plans) + + def _register_plan( + self, + invocation: ModelInvocationState, + content: str | None, + item_id: str | None, + timestamp: float | None, + ): + content = _normalize_plan_content(content) + if not content: + return + for existing in invocation.plans: + if existing.get("content") == content: + return + plan = _entity( + "plan", + content=content, + item_id=item_id, + attributed_to=invocation.agent_id, + generated_by=invocation.task_id, + **_parse_plan_content(content), + ) + invocation.plans.append(plan) + self._emit_execution_plan(plan, invocation, timestamp) + + def _emit_execution_plan( + self, + plan: JsonObject, + invocation: ModelInvocationState, + timestamp: float | None, + ): + if self._active_execution_plan_workflow_id or self._pending_execution_plan_workflow_id: + return + workflow_id = f"{invocation.task_id}:execution_plan:{len(invocation.plans)}" + workflow = WorkflowObject( + workflow_id=workflow_id, + name="execution_plan", + ) + workflow.parent_workflow_id = self._session_id or invocation.workflow_id + workflow.agent_id = invocation.agent_id + workflow.campaign_id = self._campaign_id() + workflow.subtype = PROV_AGENT_LOOP.EXECUTION_PLAN.value + workflow.started_at = timestamp or invocation.started_at + workflow.status = Status.RUNNING + criteria = self._evaluation_criteria_from_plan(plan) + if criteria: + self._execution_plan_criteria[workflow_id] = criteria + steps = _plan_step_labels(plan) + if steps: + self._execution_plan_steps[workflow_id] = steps + self._execution_plan_finished_steps[workflow_id] = set() + workflow.used = _compact_dict( + { + "entities": self._execution_plan_used_entities(plan, invocation, criteria) or None, + } + ) + workflow.workflow_description = ( + _plan_summary(plan) or "Codex execution plan extracted from a proposed_plan block or Plan item." + ) + self._send_workflow_message(workflow) + invocation.workflow_id = self._session_id or invocation.workflow_id + turn = self._turns.get(invocation.turn_id) + if turn: + turn.metadata["produced_execution_plan_id"] = workflow_id + self._pending_execution_plan_workflow_id = workflow_id + self._pending_execution_plan_turn_id = invocation.turn_id + + def _execution_plan_used_entities( + self, + plan: JsonObject, + invocation: ModelInvocationState, + criteria: list[JsonObject], + ) -> list[JsonObject]: + entities = [ + _entity( + "plan", + content=plan.get("content"), + title=plan.get("title"), + sections=plan.get("sections") or None, + item_id=plan.get("item_id"), + attributed_to=plan.get("attributed_to"), + generated_by=plan.get("generated_by"), + ) + ] + entities.extend(entity for entity in invocation.generated_entities if entity.get("type") == "objective") + prompt = invocation.prompt or self._prompt_from_messages(invocation.messages) + if prompt and not any(entity.get("type") == "objective" for entity in entities): + entities.append( + _entity( + "objective", + content=prompt, + attributed_to=invocation.source_agent_id, + role="user", + ) + ) + return entities + + def _evaluation_criteria_from_plan(self, plan: JsonObject) -> list[JsonObject]: + sections = plan.get("sections") or {} + criteria = _section_items(sections, PLAN_EVALUATION_SECTION_NAMES) + return [ + _entity( + "evaluation_criteria", + content=item, + attributed_to=plan.get("attributed_to"), + index=index, + ) + for index, item in enumerate(criteria, start=1) + ] + + def _emit_tool(self, tool: ToolInvocationState, invocation: ModelInvocationState): + if tool.emitted: + return + task = TaskObject() + task.task_id = tool.task_id + task.workflow_id = tool.workflow_id or self._active_workflow_id(invocation.workflow_id) + task.campaign_id = self._campaign_id() + is_evaluation = self._is_test_command(tool.arguments) + if is_evaluation and self._merge_tool_into_declared_evaluation(tool, invocation): + return + task.subtype = PROV_AGENT_LOOP.EVALUATION.value if is_evaluation else PROV_AGENT.TOOL_INVOCATION.value + task.parent_task_id = self._parent_for_observed_task( + task.subtype, + workflow_id=task.workflow_id, + fallback=tool.parent_task_id or invocation.parent_task_id, + ) + task.agent_id = invocation.agent_id + task.source_agent_id = invocation.source_agent_id + task.activity_id = "evaluation" if is_evaluation else tool.tool_name or "tool_invocation" + task.used = _compact_dict( + { + "entities": self._tool_used_entities(tool, task.workflow_id, is_evaluation) or None, + } + ) + task.generated = _compact_dict( + { + "entities": [ + ( + _entity( + "evaluation_result", + content=tool.output, + attributed_to=invocation.agent_id, + generated_by=tool.task_id, + ) + if is_evaluation + else _entity( + self._tool_data_entity_type(tool, tool.output), + content=tool.output, + attributed_to=invocation.agent_id, + generated_by=tool.task_id, + ) + ) + ] + if tool.output is not None + else None + } + ) + task.started_at = tool.started_at + task.ended_at = tool.ended_at or tool.started_at + task.utc_timestamp = task.started_at + self._normalize_against_active_parent(task) + self._normalize_child_task_bounds(task) + task.status = _task_status(task.ended_at, tool.output) + task.enrich(self.plugin_key) + self._intercept_task(task) + tool.emitted = True + + def _merge_tool_into_declared_evaluation( + self, + tool: ToolInvocationState, + invocation: ModelInvocationState, + ) -> bool: + evaluation = self._active_tagged_task(PROV_AGENT_LOOP.EVALUATION.value) + if evaluation is None: + return False + self._append_unique_message(evaluation.used_entities, _entity("tool", name=tool.tool_name)) + if tool.arguments is not None: + self._append_unique_message( + evaluation.used_entities, + _entity(self._tool_data_entity_type(tool, tool.arguments), content=tool.arguments), + ) + if tool.output is not None: + self._append_unique_message( + evaluation.generated_entities, + _entity( + "evaluation_result", + content=tool.output, + attributed_to=invocation.agent_id, + generated_by=evaluation.task_id, + ), + ) + tool.emitted = True + return True + + def _flush_open_records(self): + for turn in self._turns.values(): + for invocation in turn.invocations: + for tool in invocation.tools.values(): + if not tool.emitted: + self._emit_tool(tool, invocation) + if not invocation.emitted: + self._emit_invocation(invocation) + if not turn.emitted: + turn.ended_at = turn.ended_at or get_utc_now() + self._discard_empty_started_tagged_tasks() + self._emit_turn(turn) + + def _current_turn(self) -> TurnState | None: + if self._current_turn_id: + return self._turns.get(self._current_turn_id) + return None + + def _active_workflow_id(self, fallback_workflow_id: str | None) -> str | None: + return self._active_execution_plan_workflow_id or fallback_workflow_id + + def _workflow_for_turn(self, turn: TurnState) -> str | None: + if turn.metadata.get("collaboration_mode_kind") == "plan": + return turn.workflow_id + if turn.metadata.get("produced_execution_plan_id"): + return turn.workflow_id + if turn.metadata.get("active_execution_plan_id"): + return turn.metadata.get("active_execution_plan_id") + return self._active_workflow_id(turn.workflow_id) + + def _should_emit_fallback_loop(self, turn: TurnState, workflow_id: str | None) -> bool: + if turn.metadata.get("declared_loop_seen"): + return False + if turn.metadata.get("produced_execution_plan_id"): + return False + if workflow_id == self._session_id: + return False + return workflow_id is not None + + def _parent_for_observed_loop(self, turn: TurnState) -> str | None: + return turn.metadata.get("active_plan_step_id") + + def _active_loop_context(self) -> TaggedTaskState | None: + return self._active_tagged_task(PROV_AGENT_LOOP.LOOP_ITERATION.value) + + def _apply_active_context_to_invocation(self, invocation: ModelInvocationState): + loop = self._active_loop_context() + if not loop or invocation.emitted: + return + self._reparent_invocation_to_loop(invocation, loop) + + def _workflow_for_tagged_task(self, invocation: ModelInvocationState, subtype: str) -> str | None: + if subtype == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value: + return self._active_workflow_id(invocation.workflow_id) + return self._active_workflow_id(invocation.workflow_id) + + def _parent_for_tagged_task(self, subtype: str) -> str | None: + if subtype == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value: + return None + if subtype == PROV_AGENT_LOOP.LOOP_ITERATION.value: + step = self._active_tagged_task(PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + return step.task_id if step else None + if subtype == PROV_AGENT_LOOP.EVALUATION.value: + loop = self._active_tagged_task(PROV_AGENT_LOOP.LOOP_ITERATION.value) + if loop: + return loop.task_id + step = self._active_tagged_task(PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + return step.task_id if step else None + return None + + def _parent_for_observed_task( + self, + subtype: str, + workflow_id: str | None, + fallback: str | None, + ) -> str | None: + if subtype == PROV_AGENT_LOOP.EVALUATION.value: + declared_evaluation = self._active_tagged_task(PROV_AGENT_LOOP.EVALUATION.value) + if declared_evaluation and declared_evaluation.workflow_id == workflow_id: + return declared_evaluation.task_id + loop = self._active_tagged_task(PROV_AGENT_LOOP.LOOP_ITERATION.value) + if loop and loop.workflow_id == workflow_id: + return loop.task_id + turn = self._current_turn() + if turn and workflow_id and turn.metadata.get("last_loop_iteration_workflow_id") == workflow_id: + last_loop_id = turn.metadata.get("last_loop_iteration_id") + if last_loop_id: + return last_loop_id + return fallback + + def _active_tagged_task(self, subtype: str) -> TaggedTaskState | None: + for state in reversed(list(self._active_tagged_tasks.values())): + if state.subtype == subtype: + return state + return None + + def _append_declared_entity_to_active_context( + self, + invocation: ModelInvocationState, + entity: JsonObject, + entity_type: str, + ): + evaluation = self._active_tagged_task(PROV_AGENT_LOOP.EVALUATION.value) + if evaluation: + if entity_type in {"mandate", "evaluation_criteria"}: + self._append_unique_message(evaluation.used_entities, entity) + else: + self._append_unique_message(evaluation.generated_entities, entity) + return + if not invocation.emitted: + if entity_type in {"mandate", "evaluation_criteria"}: + self._append_unique_message(invocation.used_entities, entity) + else: + self._append_unique_message(invocation.generated_entities, entity) + + def _remember_tagged_context(self, invocation: ModelInvocationState, state: TaggedTaskState): + turn = self._turns.get(invocation.turn_id) + if not turn: + return + if state.workflow_id and state.workflow_id != self._session_id: + turn.metadata["active_execution_plan_id"] = state.workflow_id + invocation.workflow_id = state.workflow_id + if state.subtype == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value: + turn.metadata["active_plan_step_id"] = state.task_id + turn.metadata["active_plan_step_label"] = state.metadata.get("label") + + def _start_observed_loop_context( + self, + invocation: ModelInvocationState, + event: JsonObject, + workflow_id: str | None, + timestamp: float | None, + ): + turn = self._turns.get(invocation.turn_id) + self._emit_stale_invocation_before_new_loop(invocation, turn) + metadata = self._tagged_task_metadata(event, invocation) + started_at = timestamp or invocation.started_at + fallback_children_started_at = self._fallback_turn_children_started_at(turn) + if fallback_children_started_at is not None: + started_at = min(started_at or fallback_children_started_at, fallback_children_started_at - 0.001) + active_step = self._active_tagged_task(PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + if active_step and fallback_children_started_at is not None: + active_step.started_at = min( + active_step.started_at or fallback_children_started_at, + fallback_children_started_at - 0.002, + ) + if active_step.emitted_started: + self._update_buffered_task_context( + active_step.task_id, + workflow_id=active_step.workflow_id, + parent_task_id=active_step.parent_task_id, + started_at=active_step.started_at, + ) + active_step = self._active_tagged_task(PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + if active_step and active_step.started_at is not None and started_at is not None: + if started_at <= active_step.started_at: + started_at = active_step.started_at + 0.001 + elif started_at == invocation.started_at: + started_at -= 0.001 + state = TaggedTaskState( + task_id=self._next_tagged_task_id(invocation, PROV_AGENT_LOOP.LOOP_ITERATION.value), + subtype=PROV_AGENT_LOOP.LOOP_ITERATION.value, + activity_id="loop_iteration", + workflow_id=workflow_id, + parent_task_id=self._parent_for_tagged_task(PROV_AGENT_LOOP.LOOP_ITERATION.value), + agent_id=invocation.agent_id, + source_agent_id=invocation.source_agent_id, + started_at=started_at, + used_messages=self._tagged_task_used_messages( + event, + invocation, + PROV_AGENT_LOOP.LOOP_ITERATION.value, + workflow_id, + ), + used_entities=self._tagged_task_used_entities( + event, + invocation, + PROV_AGENT_LOOP.LOOP_ITERATION.value, + workflow_id, + ), + metadata={**metadata, "observed_loop_iteration": True}, + ) + self._active_tagged_tasks[self._tagged_task_key(event)] = state + if turn: + self._adopt_fallback_turn_children_into_loop(turn, state) + self._emit_tagged_task(state, timestamp, final=False) + if turn: + turn.metadata.update( + _compact_dict( + { + "active_execution_plan_id": workflow_id if workflow_id != self._session_id else None, + "active_loop_iteration_id": state.task_id, + "declared_loop_label": event.get("label"), + "declared_loop_summary": event.get("summary") or event.get("content"), + "declared_loop_started_at": started_at, + "declared_loop_seen": True, + } + ) + ) + for message in state.used_messages: + self._append_unique_message(turn.messages, message) + if workflow_id and workflow_id != self._session_id: + invocation.workflow_id = workflow_id + if not invocation.emitted: + self._reparent_invocation_to_loop(invocation, state) + if turn: + for turn_invocation in turn.invocations: + if turn_invocation is invocation: + continue + if not turn_invocation.emitted and turn_invocation.ended_at is None: + self._reparent_invocation_to_loop(turn_invocation, state) + + def _reparent_invocation_to_loop( + self, + invocation: ModelInvocationState, + state: TaggedTaskState, + ): + if invocation.emitted: + return + if state.workflow_id and state.workflow_id != self._session_id: + invocation.workflow_id = state.workflow_id + invocation.parent_task_id = state.task_id + if invocation.started_at is not None and state.started_at is not None: + invocation.started_at = max(invocation.started_at, state.started_at + 0.001) + for tool in invocation.tools.values(): + if tool.emitted: + continue + tool.workflow_id = invocation.workflow_id + tool.parent_task_id = state.task_id + if invocation.emitted_started: + self._emit_invocation_start_update(invocation) + self._move_buffered_task_before_descendants(state.task_id) + if state.parent_task_id: + self._move_buffered_task_before_descendants(state.parent_task_id) + + def _fallback_turn_children_started_at(self, turn: TurnState | None) -> float | None: + if not turn: + return None + started_at_values: list[float] = [] + for invocation in turn.invocations: + if invocation.parent_task_id != turn.task_id: + continue + if invocation.started_at is not None: + started_at_values.append(invocation.started_at) + started_at_values.extend( + tool.started_at + for tool in invocation.tools.values() + if tool.parent_task_id == turn.task_id and tool.started_at is not None + ) + return min(started_at_values, default=None) + + def _adopt_fallback_turn_children_into_loop( + self, + turn: TurnState, + state: TaggedTaskState, + ): + adopted = False + for invocation in turn.invocations: + if invocation.parent_task_id != turn.task_id: + continue + invocation.parent_task_id = state.task_id + if state.workflow_id and state.workflow_id != self._session_id: + invocation.workflow_id = state.workflow_id + self._update_buffered_task_context( + invocation.task_id, + workflow_id=invocation.workflow_id, + parent_task_id=state.task_id, + ) + for tool in invocation.tools.values(): + if tool.parent_task_id != turn.task_id: + continue + tool.parent_task_id = state.task_id + tool.workflow_id = invocation.workflow_id + self._update_buffered_task_context( + tool.task_id, + workflow_id=tool.workflow_id, + parent_task_id=state.task_id, + ) + adopted = True + if adopted: + turn.emitted = True + + def _emit_invocation_start_update(self, invocation: ModelInvocationState): + if invocation.emitted: + return + task = self._invocation_task(invocation, final=False) + self._intercept_task(task) + + def _emit_stale_invocation_before_new_loop( + self, + invocation: ModelInvocationState, + turn: TurnState | None, + ): + if not turn or invocation.emitted or invocation.ended_at is None: + return + active_loop_id = turn.metadata.get("active_loop_iteration_id") + if active_loop_id and invocation.parent_task_id == active_loop_id: + return + if invocation.parent_task_id: + self._emit_invocation(invocation) + if turn.current_invocation is invocation: + turn.current_invocation = None + + def _finish_observed_loop_context( + self, + invocation: ModelInvocationState, + event: JsonObject, + state: TaggedTaskState | None, + timestamp: float | None, + ): + turn = self._turns.get(invocation.turn_id) + if not turn: + return + if state is None: + return + state.generated_messages.extend(self._tagged_task_generated_messages(event, invocation, state.subtype)) + state.generated_entities.extend(self._tagged_task_generated_entities(event, invocation, state.subtype)) + state.metadata.update(self._tagged_task_metadata(event, invocation)) + self._emit_tagged_task(state, timestamp, final=True) + turn.metadata.pop("active_loop_iteration_id", None) + turn.metadata["last_loop_iteration_id"] = state.task_id + turn.metadata["last_loop_iteration_workflow_id"] = state.workflow_id + invocation_to_close = turn.current_invocation + if invocation_to_close is None and invocation.parent_task_id == state.task_id: + invocation_to_close = invocation + if invocation_to_close and not invocation_to_close.emitted: + invocation_to_close.ended_at = timestamp or invocation_to_close.ended_at + self._emit_invocation(invocation_to_close) + if turn.current_invocation is invocation_to_close: + turn.current_invocation = None + + def _clear_finished_tagged_context( + self, + invocation: ModelInvocationState, + state: TaggedTaskState, + timestamp: float | None = None, + ): + turn = self._turns.get(invocation.turn_id) + if not turn: + return + if state.subtype == PROV_AGENT_LOOP.LOOP_ITERATION.value: + turn.metadata.pop("active_loop_iteration_id", None) + turn.metadata["last_loop_iteration_id"] = state.task_id + turn.metadata["last_loop_iteration_workflow_id"] = state.workflow_id + elif state.subtype == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value: + if turn.metadata.get("active_plan_step_id") == state.task_id: + turn.metadata.pop("active_plan_step_id", None) + turn.metadata.pop("active_plan_step_label", None) + self._mark_execution_plan_step_finished(state, timestamp) + + def _mark_execution_plan_step_finished(self, state: TaggedTaskState, timestamp: float | None): + workflow_id = state.workflow_id + if not workflow_id: + return + expected_steps = self._execution_plan_steps.get(workflow_id) + if not expected_steps: + return + label = _canonical_plan_step_label(state.metadata.get("label") or state.activity_id) + if not label or label not in expected_steps: + return + finished_steps = self._execution_plan_finished_steps.setdefault(workflow_id, set()) + finished_steps.add(label) + if expected_steps.issubset(finished_steps): + self._finish_execution_plan_workflow(workflow_id, state, timestamp) + if self._active_execution_plan_workflow_id == workflow_id: + self._active_execution_plan_workflow_id = None + if self._pending_execution_plan_workflow_id == workflow_id: + self._pending_execution_plan_workflow_id = None + self._pending_execution_plan_turn_id = None + + def _finish_execution_plan_workflow( + self, + workflow_id: str, + state: TaggedTaskState, + timestamp: float | None, + ): + workflow = WorkflowObject(workflow_id=workflow_id, name="execution_plan") + workflow.subtype = PROV_AGENT_LOOP.EXECUTION_PLAN.value + workflow.campaign_id = self._campaign_id() + workflow.agent_id = state.agent_id + workflow.ended_at = timestamp or state.started_at + workflow.status = Status.FINISHED + self._send_workflow_message(workflow) + + def _loop_iteration_status(self, ended_at: float | None, metadata: JsonObject) -> Status: + declared_status = metadata.get("declared_loop_status") + if declared_status in {"finished", "passed"}: + return Status.FINISHED + if declared_status in {"failed", "error"}: + return Status.ERROR + return _task_status(ended_at, metadata) + + def _tagged_task_kind(self, event_type: str) -> tuple[str, str]: + if event_type.startswith("plan_step_"): + return PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value, "plan_step_execution" + if event_type.startswith("loop_iteration_"): + return PROV_AGENT_LOOP.LOOP_ITERATION.value, "loop_iteration" + if event_type.startswith("evaluation_"): + return PROV_AGENT_LOOP.EVALUATION.value, "evaluation" + return event_type, event_type + + def _tagged_task_key(self, event: JsonObject) -> tuple[str, str]: + event_type = event.get("type") + identity = ( + event.get("evaluation_id") + or event.get("loop_id") + or event.get("step_id") + or event.get("step") + or event.get("label") + or event.get("command") + or event_type + ) + return event_type, str(identity) + + def _next_tagged_task_id(self, invocation: ModelInvocationState, subtype: str) -> str: + count = self._tagged_task_counts.get(subtype, 0) + 1 + self._tagged_task_counts[subtype] = count + return f"{invocation.turn_id}:{subtype}:{count}" + + def _tagged_task_metadata(self, event: JsonObject, invocation: ModelInvocationState) -> JsonObject: + return _compact_dict( + { + key: value + for key, value in event.items() + if key not in {"content", "summary", "result", "criteria", "criteria_ids"} + } + | { + "turn_id": invocation.turn_id, + "ai_model_invocation_id": invocation.task_id, + } + ) + + def _tagged_task_used_messages( + self, + event: JsonObject, + invocation: ModelInvocationState, + subtype: str, + workflow_id: str | None, + ) -> list[JsonObject]: + return [] + + def _tagged_task_used_entities( + self, + event: JsonObject, + invocation: ModelInvocationState, + subtype: str, + workflow_id: str | None, + ) -> list[JsonObject]: + entities = [] + if subtype == PROV_AGENT_LOOP.EVALUATION.value: + entities.extend(self._criteria_entities_from_event(event, workflow_id, invocation.agent_id)) + return entities + + def _tagged_task_generated_messages( + self, + event: JsonObject, + invocation: ModelInvocationState, + subtype: str, + ) -> list[JsonObject]: + if subtype in { + PROV_AGENT_LOOP.EVALUATION.value, + PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value, + PROV_AGENT_LOOP.LOOP_ITERATION.value, + }: + return [] + content = event.get("summary") or event.get("result") or event.get("content") + message_type = "message" + return ( + [ + _entity( + message_type, + content=content, + attributed_to=invocation.agent_id, + status=event.get("status"), + ) + ] + if content + else [] + ) + + def _tagged_task_generated_entities( + self, + event: JsonObject, + invocation: ModelInvocationState, + subtype: str, + ) -> list[JsonObject]: + content = event.get("result") or event.get("summary") or event.get("content") + if subtype == PROV_AGENT_LOOP.EVALUATION.value: + return ( + [ + _entity( + "evaluation_result", + content=content, + status=event.get("status"), + decision=event.get("decision"), + reason=event.get("reason"), + attributed_to=invocation.agent_id, + ) + ] + if content or event.get("status") + else [] + ) + return [] + + def _criteria_entities_from_event( + self, + event: JsonObject, + workflow_id: str | None, + attributed_to: str | None, + ) -> list[JsonObject]: + entities = [] + for index, criterion in enumerate(event.get("criteria") or [], start=1): + entities.append(_entity("evaluation_criteria", content=criterion, attributed_to=attributed_to, index=index)) + for criterion_id in event.get("criteria_ids") or []: + entities.append(_entity("evaluation_criteria", criteria_id=criterion_id, attributed_to=attributed_to)) + if not entities and workflow_id and event.get("criteria_match") == "all_active_plan_criteria": + entities.extend(self._execution_plan_criteria.get(workflow_id, [])) + return entities + + def _event_attributed_to(self, event: JsonObject, invocation: ModelInvocationState) -> str | None: + attributed_to = event.get("attributed_to") + if attributed_to == "human": + return invocation.source_agent_id + if attributed_to in {"agent", "codex", "assistant"}: + return invocation.agent_id + return attributed_to or invocation.agent_id + + def _agent_response_from_events(self, text: str | None) -> str | None: + for event in reversed(_extract_flowcept_events(text)): + if event.get("type") == "agent_response" and event.get("content"): + return event["content"] + return None + + def _activate_pending_execution_plan(self, turn_id: str | None): + if turn_id is None or self._pending_execution_plan_turn_id != turn_id: + return + self._active_execution_plan_workflow_id = self._pending_execution_plan_workflow_id + self._pending_execution_plan_workflow_id = None + self._pending_execution_plan_turn_id = None + + def _is_test_command(self, arguments: Any) -> bool: + if isinstance(arguments, dict): + command = arguments.get("cmd") or arguments.get("command") + else: + command = arguments + if not isinstance(command, str): + return False + stripped = command.strip() + first_line = stripped.splitlines()[0].strip() if stripped else "" + if re.match(r"^(cat|tee|printf|echo|sed|awk|perl|apply_patch)\b", first_line): + return False + return bool( + re.search( + r"(^|[;&|]\s*)(" + r"test|pytest|unittest|ruff|mypy|pyright|eslint|vitest|jest|" + r"cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|go\s+test|" + r"python(?:3(?:\.\d+)?)?\s+-m\s+(?:pytest|unittest|py_compile)|" + r"uv\s+run(?:\s+--with\s+\S+)*\s+(?:pytest|python(?:3(?:\.\d+)?)?\s+-m\s+pytest)" + r")(\s|$)", + first_line, + ) + ) + + def _tool_used_messages( + self, + tool: ToolInvocationState, + invocation: ModelInvocationState, + workflow_id: str | None, + is_evaluation: bool, + ) -> list[JsonObject]: + return [] + + def _tool_used_entities( + self, + tool: ToolInvocationState, + workflow_id: str | None, + is_evaluation: bool, + ) -> list[JsonObject]: + entities = [] + entities.append(_entity("tool", name=tool.tool_name)) + if tool.arguments is not None: + entities.append(_entity(self._tool_data_entity_type(tool, tool.arguments), content=tool.arguments)) + if is_evaluation and workflow_id: + entities.extend(self._execution_plan_criteria.get(workflow_id, [])) + return entities + + def _tool_data_entity_type(self, tool: ToolInvocationState, value: Any) -> str: + text_parts = [str(tool.tool_name or "")] + if isinstance(tool.arguments, dict): + text_parts.extend(str(item) for pair in tool.arguments.items() for item in pair) + elif tool.arguments is not None: + text_parts.append(str(tool.arguments)) + if value is not None and value is not tool.arguments: + text_parts.append(str(value)) + haystack = "\n".join(text_parts) + if TOOL_DATA_TELEMETRY_PATTERNS.search(haystack): + return "telemetry_data" + if TOOL_DATA_SCHEDULING_PATTERNS.search(haystack): + return "scheduling_data" + if tool.tool_name: + return "domain_data" + return "entity" + + def _evaluation_criteria_match(self, workflow_id: str | None) -> str: + if workflow_id and self._execution_plan_criteria.get(workflow_id): + return "all_active_plan_criteria" + return "unresolved" + + def _find_invocation_for_call(self, call_id: str | None) -> ModelInvocationState | None: + if not call_id: + return None + for turn in self._turns.values(): + for invocation in turn.invocations: + if call_id in invocation.tools: + return invocation + return None + + def _workflow_provider(self) -> str | None: + return self._model_provider + + def _prompt_from_messages(self, messages: list[JsonObject]) -> str | None: + for message in reversed(messages): + if (message.get("role") == "user" or message.get("type") == "user_prompt") and message.get("content"): + return message["content"] + return None + + def _append_unique_message(self, messages: list[JsonObject], message: JsonObject): + for existing in messages: + if ( + existing.get("type") == message.get("type") + and existing.get("role") == message.get("role") + and existing.get("content") == message.get("content") + ): + return + messages.append(message) + + def _normalize_child_task_bounds(self, task: TaskObject): + if not hasattr(self, "_emitted_task_bounds"): + self._emitted_task_bounds = {} + if not task.parent_task_id: + return + parent_bounds = self._emitted_task_bounds.get(task.parent_task_id) + if not parent_bounds: + return + parent_started_at, parent_ended_at = parent_bounds + if parent_started_at is not None: + if task.started_at is None or task.started_at < parent_started_at: + task.started_at = parent_started_at + if parent_ended_at is not None: + if task.ended_at is None or task.ended_at > parent_ended_at: + task.ended_at = parent_ended_at + if task.started_at is None or task.started_at > parent_ended_at: + task.started_at = parent_ended_at + if task.started_at is not None and task.ended_at is not None and task.ended_at < task.started_at: + task.ended_at = task.started_at + task.utc_timestamp = task.started_at + + def _normalize_against_active_parent(self, task: TaskObject): + if not task.parent_task_id: + return + parent = next( + (state for state in self._active_tagged_tasks.values() if state.task_id == task.parent_task_id), + None, + ) + if parent is None or parent.started_at is None: + return + if task.started_at is None or task.started_at < parent.started_at: + task.started_at = parent.started_at + if task.ended_at is not None and task.ended_at < task.started_at: + task.ended_at = task.started_at + task.utc_timestamp = task.started_at + + def _intercept_task(self, task: TaskObject): + if not hasattr(self, "_emitted_task_bounds"): + self._emitted_task_bounds = {} + if task.campaign_id is None: + task.campaign_id = self._campaign_id() + self._emitted_task_bounds[task.task_id] = (task.started_at, task.ended_at) + task_msg = task.to_dict() + if self._replace_buffered_task(task_msg): + return + self.intercept(task_msg) + + def _send_workflow_message(self, workflow: WorkflowObject): + workflow.enrich(self.plugin_key) + workflow_msg = workflow.to_dict() + if self._replace_buffered_workflow(workflow_msg): + return + self.send_workflow_message(workflow) + + def _replace_buffered_workflow(self, workflow_msg: JsonObject) -> bool: + workflow_id = workflow_msg.get("workflow_id") + if not workflow_id: + return False + buffer = self._current_task_buffer() + if buffer is None: + return False + for index, existing in enumerate(buffer): + if existing.get("type") != "workflow" or existing.get("workflow_id") != workflow_id: + continue + updated = dict(existing) + for key, value in workflow_msg.items(): + if isinstance(value, dict) and isinstance(updated.get(key), dict): + updated[key] = {**updated[key], **value} + else: + updated[key] = value + buffer[index] = updated + return True + return False + + def _replace_buffered_task(self, task_msg: JsonObject) -> bool: + task_id = task_msg.get("task_id") + if not task_id: + return False + buffer = self._current_task_buffer() + if buffer is None: + return False + for index, existing in enumerate(buffer): + if existing.get("task_id") == task_id: + updated = dict(existing) + for key, value in task_msg.items(): + if isinstance(value, dict) and isinstance(updated.get(key), dict): + updated[key] = {**updated[key], **value} + else: + updated[key] = value + buffer[index] = updated + return True + return False + + def _update_buffered_task_context( + self, + task_id: str, + *, + workflow_id: str | None, + parent_task_id: str | None, + started_at: float | None = None, + ): + buffer = self._current_task_buffer() + if buffer is None: + return + for record in buffer: + if record.get("task_id") != task_id: + continue + record["workflow_id"] = workflow_id + record["parent_task_id"] = parent_task_id + if started_at is not None: + record["started_at"] = started_at + record["utc_timestamp"] = started_at + current_bounds = self._emitted_task_bounds.get(task_id, (None, None)) + self._emitted_task_bounds[task_id] = ( + started_at if started_at is not None else current_bounds[0], + current_bounds[1], + ) + return + + def _current_task_buffer(self) -> list[JsonObject] | None: + mq_dao = getattr(self, "_mq_dao", None) + if mq_dao is None: + return None + buffer = getattr(mq_dao, "buffer", None) + if hasattr(buffer, "current_buffer"): + buffer = buffer.current_buffer + return buffer if isinstance(buffer, list) else None + + def _move_buffered_task_before_descendants(self, task_id: str): + buffer = self._current_task_buffer() + if buffer is None: + return + parent_index = next( + (index for index, record in enumerate(buffer) if record.get("task_id") == task_id), + None, + ) + if parent_index is None: + return + descendant_index = next( + ( + index + for index, record in enumerate(buffer) + if record.get("parent_task_id") == task_id and index < parent_index + ), + None, + ) + if descendant_index is None: + return + record = buffer.pop(parent_index) + buffer.insert(descendant_index, record) + + def _discard_empty_started_tagged_tasks(self): + buffer = self._current_task_buffer() + if buffer is None: + return + changed = True + while changed: + changed = False + for key, state in list(self._active_tagged_tasks.items()): + if not state.emitted_started: + continue + if self._buffered_task_has_child(state.task_id): + continue + if self._remove_buffered_task(state.task_id): + self._active_tagged_tasks.pop(key, None) + self._emitted_task_bounds.pop(state.task_id, None) + changed = True + + def _buffered_task_has_child(self, task_id: str) -> bool: + buffer = self._current_task_buffer() + if buffer is None: + return False + return any(record.get("parent_task_id") == task_id for record in buffer) + + def _remove_buffered_task(self, task_id: str) -> bool: + buffer = self._current_task_buffer() + if buffer is None: + return False + for index, record in enumerate(buffer): + if record.get("task_id") == task_id: + buffer.pop(index) + return True + return False + + def _agent_obj(self, agent_id: str, name: str) -> AgentObject: + agent = AgentObject(agent_id=agent_id, name=name, workflow_id=self._session_id, campaign_id=self._campaign_id()) + agent.extra_metadata = {"source": "codex_jsonl"} + agent.enrich() + return agent + + def _campaign_id(self) -> str | None: + try: + from flowcept.flowcept_api.flowcept_controller import Flowcept + + return Flowcept.campaign_id + except Exception: + return None diff --git a/src/flowcept/instrumentation/flowcept_agent_task.py b/src/flowcept/instrumentation/flowcept_agent_task.py index 345a7b87..c0c55e76 100644 --- a/src/flowcept/instrumentation/flowcept_agent_task.py +++ b/src/flowcept/instrumentation/flowcept_agent_task.py @@ -64,7 +64,7 @@ def wrapper(*args, **kwargs): task_should_capture_telemetry = TELEMETRY_ENABLED if capture_telemetry is None else capture_telemetry task_obj = TaskObject() - task_obj.subtype = decorator_kwargs.get("subtype", PROV_AGENT.AGENT_TOOL) + task_obj.subtype = decorator_kwargs.get("subtype", PROV_AGENT.TOOL_INVOCATION) task_obj.activity_id = func.__name__ handled_args = args_handler(*args, **kwargs) task_obj.workflow_id = handled_args.pop("workflow_id", Flowcept.current_workflow_id) diff --git a/src/flowcept/version.py b/src/flowcept/version.py index bbabfd06..43071010 100644 --- a/src/flowcept/version.py +++ b/src/flowcept/version.py @@ -10,4 +10,4 @@ # ❗❗❗ Once again: DO NOT CHANGE THIS FILE ❗❗❗ # ✋⚠️⛔❗❗❗ STOP! DANGER!!ONEONEELEVEN! Did you carefully read the warning above?! :) -__version__ = "1.0.0" +__version__ = "1.0.3" diff --git a/tests/adapters/test_codex_interceptor.py b/tests/adapters/test_codex_interceptor.py new file mode 100644 index 00000000..90a98ada --- /dev/null +++ b/tests/adapters/test_codex_interceptor.py @@ -0,0 +1,1206 @@ +"""Replay tests for the Codex provenance adapter.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +from flowcept.commons.vocabulary import PROV_AGENT, PROV_AGENT_LOOP +from flowcept.flowceptor.adapters.code_assistants.codex.codex_interceptor import ( + CodexInterceptor, +) + + +ALLOWED_ENTITY_TYPES = { + "entity", + "objective", + "plan", + "checkpoint", + "message", + "evaluation_criteria", + "evaluation_result", + "memory", + "lesson_learned", + "observation", + "thought", + "decision", + "mandate", + "user_prompt", + "agent_response", + "belief", + "ai_model", + "tool", + "domain_data", + "scheduling_data", + "telemetry_data", +} + + +def _event(timestamp: str, event_type: str, payload: dict) -> dict: + return {"timestamp": timestamp, "type": event_type, "payload": payload} + + +def _message_payload(role: str, text: str, *, turn_id: str, phase: str | None = None) -> dict: + payload = { + "type": "message", + "role": role, + "content": [{"type": "output_text" if role == "assistant" else "input_text", "text": text}], + "internal_chat_message_metadata_passthrough": {"turn_id": turn_id}, + } + if phase: + payload["phase"] = phase + return payload + + +def _write_jsonl(path: Path, events: list[dict]): + path.write_text("\n".join(json.dumps(event) for event in events) + "\n", encoding="utf-8") + + +def _new_interceptor(*, declared_provenance_enabled: bool) -> tuple[CodexInterceptor, list[dict]]: + interceptor = CodexInterceptor.__new__(CodexInterceptor) + records: list[dict] = [] + interceptor.plugin_key = "codex" + interceptor.settings = SimpleNamespace( + declared_provenance_enabled=declared_provenance_enabled, + include_developer_messages=True, + include_reasoning=True, + watch_interval_sec=0, + file_path="", + recursive=False, + ) + interceptor._observer = None + interceptor._processed_lines = {} + interceptor._session_id = None + interceptor._codex_agent_id = None + interceptor._human_agent_id = None + interceptor._model_provider = None + interceptor._current_turn_id = None + interceptor._active_execution_plan_workflow_id = None + interceptor._pending_execution_plan_workflow_id = None + interceptor._pending_execution_plan_turn_id = None + interceptor._execution_plan_criteria = {} + interceptor._execution_plan_steps = {} + interceptor._execution_plan_finished_steps = {} + interceptor._tagged_task_counts = {} + interceptor._active_tagged_tasks = {} + interceptor._processed_declared_events = set() + interceptor._emitted_task_bounds = {} + interceptor._turns = {} + interceptor._emitted_records_count = 0 + interceptor._mq_dao = SimpleNamespace(buffer=SimpleNamespace(current_buffer=records)) + interceptor.intercept = lambda record: records.append(record) + interceptor.send_workflow_message = lambda obj: records.append(obj.to_dict()) + interceptor.send_agent_message = lambda obj: records.append(obj.to_dict()) + return interceptor, records + + +def _replay(path: Path, *, declared_provenance_enabled: bool = True) -> list[dict]: + interceptor, records = _new_interceptor(declared_provenance_enabled=declared_provenance_enabled) + interceptor._process_log(path) + interceptor._flush_open_records() + return records + + +def _tasks(records: list[dict]) -> list[dict]: + return [record for record in records if record.get("type") == "task"] + + +def _workflows(records: list[dict]) -> list[dict]: + return [record for record in records if record.get("type") == "workflow"] + + +def _assert_common_invariants(records: list[dict]): + tasks = _tasks(records) + by_id = {task["task_id"]: task for task in tasks} + for record in [*_workflows(records), *tasks]: + for side in ("used", "generated"): + payload = record.get(side) or {} + allowed_keys = {"entities"} + if record.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value and side == "used": + allowed_keys.add("prompt") + if record.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value and side == "generated": + allowed_keys.add("response") + assert set(payload).issubset(allowed_keys) + for entity in payload.get("entities") or []: + assert entity.get("type") in ALLOWED_ENTITY_TYPES + assert "entity_role" not in entity + for task in tasks: + if task.get("started_at"): + assert task.get("utc_timestamp") == task.get("started_at") + if task.get("subtype") in { + PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value, + PROV_AGENT_LOOP.LOOP_ITERATION.value, + }: + assert not task.get("generated") + parent = by_id.get(task.get("parent_task_id")) + if not parent: + continue + assert task.get("workflow_id") == parent.get("workflow_id") + if task.get("started_at") and parent.get("started_at"): + assert task["started_at"] >= parent["started_at"] + if task.get("ended_at") and parent.get("ended_at"): + assert task["ended_at"] <= parent["ended_at"] + execution_plan_workflows = { + workflow["workflow_id"] + for workflow in _workflows(records) + if workflow.get("subtype") == PROV_AGENT_LOOP.EXECUTION_PLAN.value + } + for task in tasks: + if task.get("workflow_id") not in execution_plan_workflows: + continue + if task.get("subtype") in { + PROV_AGENT.AI_MODEL_INVOCATION.value, + PROV_AGENT.TOOL_INVOCATION.value, + PROV_AGENT_LOOP.EVALUATION.value, + }: + assert task.get("parent_task_id") + + +def test_dpl_execution_plan_does_not_parent_plan_generation_turn(tmp_path: Path): + log = tmp_path / "codex.jsonl" + plan = """**Execution Plan** + +**Summary** +- Build a small package. + +**Steps** +- `Inspect tutorial` +- `Create package` + +**Evaluation criteria** +- YAML files parse. +""" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + _message_payload("assistant", f"\n{plan}\n", turn_id="turn-plan"), + ), + _event( + "2026-08-03T10:00:04Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 1, "output_tokens": 1}}}, + ), + _event( + "2026-08-03T10:00:05Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-plan", "last_agent_message": "plan ready"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + workflows = _workflows(records) + tasks = _tasks(records) + execution_plan = next(w for w in workflows if w.get("subtype") == PROV_AGENT_LOOP.EXECUTION_PLAN.value) + plan_invocation = next(t for t in tasks if t.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value) + + assert plan_invocation["workflow_id"] == "s1" + assert execution_plan["parent_workflow_id"] == "s1" + assert execution_plan.get("status") == "RUNNING" + assert execution_plan.get("ended_at") is None + assert [ + task + for task in tasks + if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value + and task.get("workflow_id") == "s1" + ] == [] + + +def test_dpl_tagged_step_loop_parents_observed_invocations_and_tools(tmp_path: Path): + log = tmp_path / "codex.jsonl" + start_tags = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Inspect tutorial"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Inspect tutorial loop"}' + ) + finish_tags = ( + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Inspect tutorial loop","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Inspect tutorial","status":"finished"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Inspect tutorial"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event( + "2026-08-03T10:00:04Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 1, "output_tokens": 1}}}, + ), + _event( + "2026-08-03T10:00:05Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-plan", "last_agent_message": "plan ready"}, + ), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-step"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event("2026-08-03T10:01:02Z", "response_item", _message_payload("assistant", start_tags, turn_id="turn-step")), + _event( + "2026-08-03T10:01:03Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "pwd"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-step"}, + }, + ), + _event( + "2026-08-03T10:01:04Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-step"}, + }, + ), + _event("2026-08-03T10:01:05Z", "response_item", _message_payload("assistant", finish_tags, turn_id="turn-step")), + _event( + "2026-08-03T10:01:06Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 1, "output_tokens": 1}}}, + ), + _event( + "2026-08-03T10:01:07Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-step", "last_agent_message": "done"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + counts = Counter(task.get("subtype") for task in tasks) + assert counts[PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value] == 1 + assert counts[PROV_AGENT_LOOP.LOOP_ITERATION.value] == 1 + + loop = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value) + step = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + assert step["started_at"] < loop["started_at"] + children = [task for task in tasks if task.get("parent_task_id") == loop["task_id"]] + assert all(loop["started_at"] <= task["started_at"] for task in children) + assert {task.get("subtype") for task in children} >= { + PROV_AGENT.AI_MODEL_INVOCATION.value, + PROV_AGENT.TOOL_INVOCATION.value, + } + tool = next(task for task in children if task.get("subtype") == PROV_AGENT.TOOL_INVOCATION.value) + assert (tool.get("used") or {})["entities"][1]["type"] == "domain_data" + assert (tool.get("generated") or {})["entities"][0]["type"] == "domain_data" + + +def test_dpl_does_not_create_second_execution_plan_workflow(tmp_path: Path): + log = tmp_path / "codex.jsonl" + plan_a = "\n**Plan A**\n\n**Steps**\n- `Create file`\n" + plan_b = "\n**Plan B**\n\n**Steps**\n- `Run tests`\n" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event("2026-08-03T10:00:03Z", "response_item", _message_payload("assistant", plan_a, turn_id="turn-plan")), + _event("2026-08-03T10:00:04Z", "event_msg", {"type": "task_complete", "turn_id": "turn-plan"}), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "continue"}), + _event("2026-08-03T10:01:02Z", "response_item", _message_payload("assistant", plan_b, turn_id="turn-work")), + _event("2026-08-03T10:01:03Z", "event_msg", {"type": "task_complete", "turn_id": "turn-work"}), + ] + _write_jsonl(log, events) + + records = _replay(log) + execution_plans = [ + workflow + for workflow in _workflows(records) + if workflow.get("subtype") == PROV_AGENT_LOOP.EXECUTION_PLAN.value + ] + + assert len(execution_plans) == 1 + + +def test_loop_start_reparents_existing_commentary_invocation_before_tool(tmp_path: Path): + log = tmp_path / "codex.jsonl" + tags = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Implement"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Implement loop"}' + ) + finish = ( + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Implement loop","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Implement","status":"finished"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-1"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + _message_payload("assistant", "Vou implementar agora.", turn_id="turn-1", phase="commentary"), + ), + _event("2026-08-03T10:00:04Z", "response_item", _message_payload("assistant", tags, turn_id="turn-1")), + _event( + "2026-08-03T10:00:05Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "pwd"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-1"}, + }, + ), + _event( + "2026-08-03T10:00:06Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-1"}, + }, + ), + _event("2026-08-03T10:00:07Z", "response_item", _message_payload("assistant", finish, turn_id="turn-1")), + _event("2026-08-03T10:00:08Z", "event_msg", {"type": "task_complete", "turn_id": "turn-1"}), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + loop = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value) + children = [task for task in tasks if task.get("parent_task_id") == loop["task_id"]] + + assert any(task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value for task in children) + assert any(task.get("subtype") == PROV_AGENT.TOOL_INVOCATION.value for task in children) + + +def test_dpl_without_step_tags_creates_execution_plan_loop_from_turn(tmp_path: Path): + log = tmp_path / "codex.jsonl" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Create package"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event( + "2026-08-03T10:00:04Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-plan", "last_agent_message": "plan ready"}, + ), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event( + "2026-08-03T10:01:02Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "pwd"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:03Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:04Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-work", "last_agent_message": "done"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + execution_plan_id = next( + workflow["workflow_id"] + for workflow in _workflows(records) + if workflow.get("subtype") == PROV_AGENT_LOOP.EXECUTION_PLAN.value + ) + execution_plan_loops = [ + task + for task in _tasks(records) + if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value + and task.get("workflow_id") == execution_plan_id + ] + assert len(execution_plan_loops) == 1 + loop = execution_plan_loops[0] + assert loop.get("parent_task_id") is None + children = [ + task + for task in _tasks(records) + if task.get("parent_task_id") == loop["task_id"] + ] + assert {task.get("subtype") for task in children} >= { + PROV_AGENT.AI_MODEL_INVOCATION.value, + PROV_AGENT.TOOL_INVOCATION.value, + } + + +def test_tool_data_entity_type_classifies_scheduling_and_telemetry(): + interceptor, _ = _new_interceptor(declared_provenance_enabled=True) + tool = SimpleNamespace(tool_name="exec_command", arguments={"cmd": "sacct -j 123 --format JobID,Elapsed"}) + assert interceptor._tool_data_entity_type(tool, tool.arguments) == "scheduling_data" + + tool = SimpleNamespace(tool_name="exec_command", arguments={"cmd": "nvidia-smi --query-gpu=utilization.gpu"}) + assert interceptor._tool_data_entity_type(tool, tool.arguments) == "telemetry_data" + + +def test_token_count_then_task_complete_populates_ai_model_response(tmp_path: Path): + log = tmp_path / "codex.jsonl" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-1"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "say done"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + _message_payload("assistant", "Vou responder no final.", turn_id="turn-1", phase="commentary"), + ), + _event( + "2026-08-03T10:00:04Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 3, "output_tokens": 5}}}, + ), + _event( + "2026-08-03T10:00:05Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-1", "last_agent_message": "done"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + invocation = next(task for task in _tasks(records) if task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value) + generated = invocation.get("generated") or {} + entities = generated.get("entities") or [] + + assert generated.get("response") == "done" + assert any(entity.get("type") == "agent_response" and entity.get("content") == "done" for entity in entities) + assert invocation.get("custom_metadata", {}).get("llm_usage", {}).get("input_tokens") == 3 + + +def test_dpl_semantic_entities_are_captured_from_declared_events(tmp_path: Path): + log = tmp_path / "codex.jsonl" + dpl = "\n".join( + [ + '{"layer":"DPL","class":"Observation","summary":"The validation command failed."}', + '{"layer":"DPL","class":"Belief","summary":"The config path is wrong."}', + '{"layer":"DPL","class":"Decision","decision":"retry","summary":"Patch the config path."}', + '{"layer":"DPL","class":"Memory","summary":"This repo stores configs under conf/."}', + '{"layer":"DPL","class":"lesson_learned","summary":"Validate config paths before launching training."}', + '{"layer":"DPL","class":"EvaluationCriteria","summary":"Validation loss stays below threshold."}', + '{"layer":"DPL","class":"evaluation_result","summary":"Validation failed before launch."}', + ] + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-1"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "validate"}), + _event("2026-08-03T10:00:03Z", "response_item", _message_payload("assistant", dpl, turn_id="turn-1")), + _event( + "2026-08-03T10:00:04Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-1", "last_agent_message": "done"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + invocation = next(task for task in _tasks(records) if task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value) + entity_types = { + entity.get("type") + for entity in (invocation.get("generated") or {}).get("entities") or [] + } + + assert { + "observation", + "belief", + "decision", + "memory", + "lesson_learned", + "evaluation_criteria", + "evaluation_result", + } <= entity_types + + +def test_opl_ignores_declared_event_only_messages(tmp_path: Path): + log = tmp_path / "codex.jsonl" + dpl_only = "\n".join( + [ + '{"layer":"DPL","class":"PlanStepExecution","event":"started","label":"Create file"}', + '{"layer":"DPL","class":"LoopIteration","event":"started","label":"Create file loop"}', + '{"layer":"DPL","class":"Observation","summary":"The file was created."}', + '{"layer":"DPL","class":"Belief","summary":"The file exists."}', + '{"layer":"DPL","class":"Memory","summary":"Use uv run for pytest."}', + '{"layer":"DPL","class":"LessonLearned","summary":"Validate after writing."}', + '{"layer":"DPL","class":"LoopIteration","event":"finished","label":"Create file loop","status":"finished"}', + '{"layer":"DPL","class":"PlanStepExecution","event":"finished","label":"Create file","status":"finished"}', + ] + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Create file"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event("2026-08-03T10:00:04Z", "event_msg", {"type": "task_complete", "turn_id": "turn-plan"}), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event("2026-08-03T10:01:02Z", "event_msg", {"type": "agent_message", "message": dpl_only}), + _event( + "2026-08-03T10:01:03Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "touch file.txt"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:04Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event("2026-08-03T10:01:05Z", "event_msg", {"type": "task_complete", "turn_id": "turn-work"}), + ] + _write_jsonl(log, events) + + records = _replay(log, declared_provenance_enabled=False) + _assert_common_invariants(records) + tasks = _tasks(records) + entity_types = { + entity.get("type") + for task in tasks + for side in ("used", "generated") + for entity in (task.get(side) or {}).get("entities") or [] + } + + assert PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value not in {task.get("subtype") for task in tasks} + assert PROV_AGENT_LOOP.EVALUATION.value not in {task.get("subtype") for task in tasks} + assert not {"observation", "belief", "memory", "lesson_learned"} & entity_types + + +def test_dpl_only_bridge_message_does_not_create_zero_duration_model_invocation(tmp_path: Path): + log = tmp_path / "codex.jsonl" + plan_tags = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Create file"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Create file loop"}' + ) + bridge_tags = ( + '{"layer":"DPL","class":"Observation","summary":"The file was created."}\n' + '{"layer":"DPL","class":"Decision","decision":"continue",' + '"summary":"Proceed to tests."}\n' + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Create file loop","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Create file","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Run tests"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Run tests loop"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Create file"}, {"step": "Run tests"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event( + "2026-08-03T10:00:04Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-plan", "last_agent_message": "plan ready"}, + ), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event("2026-08-03T10:01:02Z", "response_item", _message_payload("assistant", plan_tags, turn_id="turn-work")), + _event( + "2026-08-03T10:01:03Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "touch file.txt"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:04Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:05Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 3, "output_tokens": 4}}}, + ), + _event("2026-08-03T10:01:06Z", "event_msg", {"type": "agent_message", "message": bridge_tags}), + _event( + "2026-08-03T10:01:07Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-work", "last_agent_message": "done"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + ai_invocations = [task for task in tasks if task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value] + assert len([task for task in ai_invocations if task["task_id"].startswith("turn-work:")]) == 1 + assert Counter(task.get("subtype") for task in tasks)[PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value] == 1 + work_invocation = next( + task + for task in ai_invocations + if task["task_id"].startswith("turn-work:") + ) + generated_types = { + entity.get("type") + for entity in (work_invocation.get("generated") or {}).get("entities") or [] + } + assert {"observation", "decision"} <= generated_types + + +def test_tool_only_model_invocation_has_ui_response(tmp_path: Path): + log = tmp_path / "codex.jsonl" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-1"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "run pwd"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "pwd"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-1"}, + }, + ), + _event( + "2026-08-03T10:00:04Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-1"}, + }, + ), + _event( + "2026-08-03T10:00:05Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 3, "output_tokens": 4}}}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + invocation = next(task for task in _tasks(records) if task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value) + + assert (invocation.get("generated") or {}).get("response") == "Requested tool invocation: exec_command" + + +def test_dpl_multiple_retry_loops_can_share_one_plan_step(tmp_path: Path): + log = tmp_path / "codex.jsonl" + start = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Run focused tests"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Try pytest with python"}' + ) + retry = ( + '{"layer":"DPL","class":"Evaluation","event":"finished",' + '"label":"Try pytest with python","status":"failed","decision":"retry",' + '"result":"python is unavailable."}\n' + '{"layer":"DPL","class":"Observation","summary":"python is unavailable."}\n' + '{"layer":"DPL","class":"Decision","decision":"retry",' + '"summary":"Retry with python3."}\n' + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Try pytest with python","status":"failed"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Retry pytest with python3"}' + ) + finish = ( + '{"layer":"DPL","class":"Evaluation","event":"finished",' + '"label":"Retry pytest with python3","status":"passed","decision":"continue",' + '"result":"tests passed."}\n' + '{"layer":"DPL","class":"Decision","decision":"continue",' + '"summary":"Finish the step."}\n' + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Retry pytest with python3","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Run focused tests","status":"finished"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Run focused tests"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event("2026-08-03T10:00:04Z", "event_msg", {"type": "task_complete", "turn_id": "turn-plan"}), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event("2026-08-03T10:01:02Z", "response_item", _message_payload("assistant", start, turn_id="turn-work")), + _event("2026-08-03T10:01:03Z", "event_msg", {"type": "agent_message", "message": retry}), + _event("2026-08-03T10:01:04Z", "event_msg", {"type": "agent_message", "message": finish}), + _event("2026-08-03T10:01:05Z", "event_msg", {"type": "task_complete", "turn_id": "turn-work"}), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + step = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + loops = [task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value] + + assert len(loops) == 2 + assert all(loop.get("parent_task_id") == step["task_id"] for loop in loops) + assert {loop.get("activity_id") for loop in loops} == { + "Try pytest with python", + "Retry pytest with python3", + } + + +def test_file_write_containing_pytest_is_not_evaluation(tmp_path: Path): + log = tmp_path / "codex.jsonl" + start = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Create tests"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Create tests loop"}' + ) + finish = ( + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Create tests loop","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Create tests","status":"finished"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Create tests"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event("2026-08-03T10:00:04Z", "event_msg", {"type": "task_complete", "turn_id": "turn-plan"}), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event("2026-08-03T10:01:02Z", "response_item", _message_payload("assistant", start, turn_id="turn-work")), + _event( + "2026-08-03T10:01:03Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "cat > test_fibonacci.py <<'PY'\nimport pytest\nPY"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:04Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event("2026-08-03T10:01:05Z", "event_msg", {"type": "agent_message", "message": finish}), + _event("2026-08-03T10:01:06Z", "event_msg", {"type": "task_complete", "turn_id": "turn-work"}), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + loop = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value) + children = [task for task in tasks if task.get("parent_task_id") == loop["task_id"]] + + assert any(task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value for task in children) + assert any(task.get("subtype") == PROV_AGENT.TOOL_INVOCATION.value for task in children) + assert not any(task.get("subtype") == PROV_AGENT_LOOP.EVALUATION.value for task in children) + + +def test_late_loop_start_reparents_open_invocation_and_tool(tmp_path: Path): + log = tmp_path / "codex.jsonl" + tags = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Late step"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Late loop"}' + ) + finish = ( + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Late loop","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Late step","status":"finished"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-1"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "run pwd"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call", + "arguments": json.dumps({"cmd": "pwd"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-1"}, + }, + ), + _event("2026-08-03T10:00:04Z", "response_item", _message_payload("assistant", tags, turn_id="turn-1")), + _event( + "2026-08-03T10:00:05Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-1"}, + }, + ), + _event("2026-08-03T10:00:06Z", "response_item", _message_payload("assistant", finish, turn_id="turn-1")), + _event("2026-08-03T10:00:07Z", "event_msg", {"type": "task_complete", "turn_id": "turn-1"}), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + loop = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value) + invocation = next(task for task in tasks if task.get("subtype") == PROV_AGENT.AI_MODEL_INVOCATION.value) + tool = next(task for task in tasks if task.get("subtype") == PROV_AGENT.TOOL_INVOCATION.value) + + assert invocation["parent_task_id"] == loop["task_id"] + assert tool["parent_task_id"] == loop["task_id"] + + +def test_loop_start_adopts_emitted_fallback_invocation_and_tool(tmp_path: Path): + log = tmp_path / "codex.jsonl" + tags = ( + '{"layer":"DPL","class":"PlanStepExecution","event":"started",' + '"label":"Create package"}\n' + '{"layer":"DPL","class":"LoopIteration","event":"started",' + '"label":"Create package loop"}' + ) + finish = ( + '{"layer":"DPL","class":"LoopIteration","event":"finished",' + '"label":"Create package loop","status":"finished"}\n' + '{"layer":"DPL","class":"PlanStepExecution","event":"finished",' + '"label":"Create package","status":"finished"}' + ) + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Create package"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event("2026-08-03T10:00:04Z", "event_msg", {"type": "task_complete", "turn_id": "turn-plan"}), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-work"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "implement"}), + _event( + "2026-08-03T10:01:02Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call-1", + "arguments": json.dumps({"cmd": "mkdir src3"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:03Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call-1", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:04Z", + "event_msg", + {"type": "token_count", "info": {"last_token_usage": {"input_tokens": 1, "output_tokens": 1}}}, + ), + _event("2026-08-03T10:01:05Z", "response_item", _message_payload("assistant", tags, turn_id="turn-work")), + _event( + "2026-08-03T10:01:06Z", + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "tool-call-2", + "arguments": json.dumps({"cmd": "touch src3/__init__.py"}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event( + "2026-08-03T10:01:07Z", + "response_item", + { + "type": "function_call_output", + "call_id": "tool-call-2", + "output": "ok", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-work"}, + }, + ), + _event("2026-08-03T10:01:08Z", "response_item", _message_payload("assistant", finish, turn_id="turn-work")), + _event("2026-08-03T10:01:09Z", "event_msg", {"type": "task_complete", "turn_id": "turn-work"}), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + tasks = _tasks(records) + by_id = {task["task_id"]: task for task in tasks} + step = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value) + loop = next(task for task in tasks if task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value) + children = [task for task in tasks if task.get("parent_task_id") == loop["task_id"]] + + assert loop["parent_task_id"] == step["task_id"] + assert step["started_at"] < loop["started_at"] + assert {task.get("subtype") for task in children} >= { + PROV_AGENT.AI_MODEL_INVOCATION.value, + PROV_AGENT.TOOL_INVOCATION.value, + } + assert all(task.get("parent_task_id") in by_id for task in tasks if task.get("parent_task_id")) + + +def test_aborted_turn_without_invocation_does_not_emit_empty_fallback_loop(tmp_path: Path): + log = tmp_path / "codex.jsonl" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-plan"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "make a plan"}), + _event( + "2026-08-03T10:00:03Z", + "response_item", + { + "type": "function_call", + "name": "update_plan", + "call_id": "plan-call", + "arguments": json.dumps({"plan": [{"step": "Create package"}]}), + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-plan"}, + }, + ), + _event("2026-08-03T10:00:04Z", "event_msg", {"type": "task_complete", "turn_id": "turn-plan"}), + _event("2026-08-03T10:01:00Z", "event_msg", {"type": "task_started", "turn_id": "turn-abort"}), + _event("2026-08-03T10:01:01Z", "event_msg", {"type": "user_message", "message": "stop"}), + _event( + "2026-08-03T10:01:02Z", + "event_msg", + {"type": "turn_aborted", "turn_id": "turn-abort", "reason": "interrupted"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + _assert_common_invariants(records) + assert not any( + task.get("task_id") == "turn-abort" + and task.get("subtype") == PROV_AGENT_LOOP.LOOP_ITERATION.value + for task in _tasks(records) + ) + + +def test_codex_records_include_current_campaign_id(tmp_path: Path): + from flowcept.flowcept_api.flowcept_controller import Flowcept + + original_campaign_id = Flowcept.campaign_id + Flowcept.campaign_id = "campaign-test" + try: + log = tmp_path / "codex.jsonl" + events = [ + _event( + "2026-08-03T10:00:00Z", + "session_meta", + {"session_id": "s1", "model_provider": "openai", "base_instructions": {"text": "base"}}, + ), + _event("2026-08-03T10:00:01Z", "event_msg", {"type": "task_started", "turn_id": "turn-1"}), + _event("2026-08-03T10:00:02Z", "event_msg", {"type": "user_message", "message": "hello"}), + _event( + "2026-08-03T10:00:03Z", + "event_msg", + {"type": "task_complete", "turn_id": "turn-1", "last_agent_message": "hi"}, + ), + ] + _write_jsonl(log, events) + + records = _replay(log) + finally: + Flowcept.campaign_id = original_campaign_id + + assert all(record.get("campaign_id") == "campaign-test" for record in _workflows(records)) + assert all(record.get("campaign_id") == "campaign-test" for record in _tasks(records)) + assert all(record.get("campaign_id") == "campaign-test" for record in records if record.get("type") == "agent") diff --git a/tests/agent/agent_tests.py b/tests/agent/agent_tests.py index cb8ce8d5..74310b3c 100644 --- a/tests/agent/agent_tests.py +++ b/tests/agent/agent_tests.py @@ -749,10 +749,15 @@ class TestProvAgentInstrumentation(unittest.TestCase): """Structural tests for PROV-AGENT enum usage. No live services required.""" def test_prov_agent_enum_values(self): - from flowcept.commons.vocabulary import PROV_AGENT + from flowcept.commons.vocabulary import PROV_AGENT, PROV_AGENT_LOOP self.assertEqual(PROV_AGENT.AI_MODEL_INVOCATION.value, "ai_model_invocation") - self.assertEqual(PROV_AGENT.AGENT_TOOL.value, "agent_tool") + self.assertEqual(PROV_AGENT.TOOL_INVOCATION.value, "tool_invocation") + self.assertEqual(PROV_AGENT_LOOP.SESSION.value, "session") + self.assertEqual(PROV_AGENT_LOOP.EXECUTION_PLAN.value, "execution_plan") + self.assertEqual(PROV_AGENT_LOOP.PLAN_STEP_EXECUTION.value, "plan_step_execution") + self.assertEqual(PROV_AGENT_LOOP.LOOP_ITERATION.value, "loop_iteration") + self.assertEqual(PROV_AGENT_LOOP.EVALUATION.value, "evaluation") def test_flowcept_llm_uses_prov_agent_enum_not_bare_string(self): import inspect @@ -767,8 +772,12 @@ def test_agent_flowcept_task_default_uses_prov_agent_enum(self): import flowcept.instrumentation.flowcept_agent_task as m src = inspect.getsource(m.agent_flowcept_task) - self.assertNotIn('"agent_task"', src, "agent_flowcept_task must use PROV_AGENT.AGENT_TOOL, not bare string") - self.assertIn("PROV_AGENT.AGENT_TOOL", src) + self.assertNotIn( + '"agent_task"', + src, + "agent_flowcept_task must use PROV_AGENT.TOOL_INVOCATION, not bare string", + ) + self.assertIn("PROV_AGENT.TOOL_INVOCATION", src) def test_context_manager_comparisons_use_prov_agent_enum(self): import inspect diff --git a/tests/instrumentation_tests/ml_tests/single_layer_perceptron_test.py b/tests/instrumentation_tests/ml_tests/single_layer_perceptron_test.py index 1467d672..19321f4c 100644 --- a/tests/instrumentation_tests/ml_tests/single_layer_perceptron_test.py +++ b/tests/instrumentation_tests/ml_tests/single_layer_perceptron_test.py @@ -124,7 +124,7 @@ def call_hpc_agent(agent_id=None): n_configs = 5 return dataset_config, n_configs -@flowcept_task(output_names=["configs", "job_id"], subtype=PROV_AGENT.AGENT_TOOL) +@flowcept_task(output_names=["configs", "job_id"], subtype=PROV_AGENT.TOOL_INVOCATION) def submit_gridsearch_job( n_configs=5, agent_id=None, diff --git a/tests/webservice/test_webservice_integration.py b/tests/webservice/test_webservice_integration.py index 906ba005..39940771 100644 --- a/tests/webservice/test_webservice_integration.py +++ b/tests/webservice/test_webservice_integration.py @@ -1132,7 +1132,7 @@ def _new_chat_workflow_ids(): assert any(task["custom_metadata"]["llm_usage"]["output_chars"] > 0 for task in llm_tasks) tool_tasks = Flowcept.db.task_query( - filter={"workflow_id": chat_workflow_id, "subtype": PROV_AGENT.AGENT_TOOL.value}, + filter={"workflow_id": chat_workflow_id, "subtype": PROV_AGENT.TOOL_INVOCATION.value}, ) assert tool_tasks, "The forced DB query should record at least one agent tool task." diff --git a/ui/tests/aiUsage.test.ts b/ui/tests/aiUsage.test.ts index d1e39df8..793a1e1a 100644 --- a/ui/tests/aiUsage.test.ts +++ b/ui/tests/aiUsage.test.ts @@ -25,7 +25,7 @@ describe("getAiModelUsageRows", () => { used: { prompt: "What happened in this workflow?".repeat(10) }, generated: { response: "The workflow finished successfully.".repeat(10) }, }, - { task_id: "task-2", subtype: "agent_tool" }, + { task_id: "task-2", subtype: "tool_invocation" }, ]; const rows = getAiModelUsageRows(tasks);