diff --git a/interface/config.py b/interface/config.py index 048a719..785e9d2 100644 --- a/interface/config.py +++ b/interface/config.py @@ -8,7 +8,7 @@ class ExperimentConfig: """Selects one implementation along each experimental axis.""" - prompting: Literal["minimal", "standard", "verbose", "text_initial_maze"] = "standard" + prompting: Literal["minimal", "standard", "verbose"] = "standard" observation: Literal["text_only", "image_text", "image_only"] = "image_only" include_current_observation_description: bool = False observation_text_includes_facing: bool = False diff --git a/interface/prompt_strategies.py b/interface/prompt_strategies.py index 5240b06..e1a6dce 100644 --- a/interface/prompt_strategies.py +++ b/interface/prompt_strategies.py @@ -57,21 +57,6 @@ def build_system_prompt(self, querying_suffix: str = "") -> str: return "\n\n".join([std, MECHANISM_RULES]) -class TextInitialMazePromptStrategy(StandardPromptStrategy): - """Standard system prompt plus the initial maze section placeholder. - - This strategy returns the standard system prompt and appends the - `INITIAL_MAZE_SECTION` template (containing the `{maze_text}` placeholder). - The caller (for example `ExperimentRunner.build_prompt_message`) is - responsible for formatting `{maze_text}` with the rendered maze text. - """ - - def build_system_prompt(self, querying_suffix: str = "") -> str: - del querying_suffix - std = StandardPromptStrategy.build_system_prompt(self).rstrip() - return "\n\n".join([std, system_templates.INITIAL_MAZE_SECTION]) - - PromptStrategy = MinimalPromptStrategy diff --git a/interface/runner.py b/interface/runner.py index 790be65..efc6ee0 100644 --- a/interface/runner.py +++ b/interface/runner.py @@ -23,7 +23,6 @@ current_observation_text, history_content_blocks, history_text, - recent_history_steps, ) from interface.parser import ACTIONS_HINT from interface.prompt_strategies import ( @@ -32,12 +31,9 @@ StandardPromptStrategy, VerbosePromptStrategy, ) -from interface.prompt_strategies import TextInitialMazePromptStrategy from interface.querying import QueryingMode -from interface.renderer import render_initial_maze_text from prompting_experiments.prompt_templates import feedback as feedback_templates from prompting_experiments.prompt_templates import querying as querying_templates -from prompting_experiments.prompt_templates import system as system_templates from prompting_experiments.prompt_templates import user as user_templates logger = logging.getLogger(__name__) @@ -46,7 +42,6 @@ "minimal": MinimalPromptStrategy, "standard": StandardPromptStrategy, "verbose": VerbosePromptStrategy, - "text_initial_maze": TextInitialMazePromptStrategy, } @@ -64,18 +59,6 @@ def _trim_rolling_chat(messages: List[dict], max_pairs: int) -> None: del messages[1 : 1 + (tail_len - cap)] -def _reset_agent_usage(agent: Callable[[List[dict]], str]) -> None: - """Clear per-call telemetry so stale usage cannot leak into a later query.""" - reset_usage = getattr(agent, "reset_usage", None) - if callable(reset_usage): - reset_usage() - return - try: - setattr(agent, "last_usage", None) - except (AttributeError, TypeError): - pass - - def _replace_current_question(prompt_text: str, question: str) -> str: standard_question = user_templates.NEXT_ACTION_QUESTION before, match, after = prompt_text.rpartition(standard_question) @@ -147,21 +130,7 @@ def build_prompt_message( last_feedback: str, transcript: List[dict], ) -> tuple[str, dict]: - system_prompt = self.prompt.build_system_prompt() - # If the system prompt includes the `{maze_text}` placeholder, format - # it with the rendered maze. Otherwise, for text observations append - # the `INITIAL_MAZE_SECTION` so the maze is present in system-level - # context for text-only or image+text modes. - if "{maze_text}" in system_prompt: - system_prompt = system_prompt.format(maze_text=render_initial_maze_text(self.task_spec)) - elif self.config.observation in ("text_only", "image_text"): - maze_text = render_initial_maze_text(self.task_spec) - system_prompt = ( - system_prompt - + "\n\n" - + system_templates.INITIAL_MAZE_SECTION.format(maze_text=maze_text) - ) - return system_prompt, self._build_message( + return self.prompt.build_system_prompt(), self._build_message( state, last_feedback, transcript, @@ -177,9 +146,7 @@ def run( self.last_rgb, state, reset_info = self.backend.reset(seed=self.task_spec.seed) self.querying.reset() - # Build the initial system prompt (may include the initial maze for - # text-based observations) and the initial user message block. - system_prompt, _ = self.build_prompt_message(state, feedback_templates.INITIAL_FEEDBACK, []) + system_prompt = self.prompt.build_system_prompt() system_message = {"role": "system", "content": system_prompt} chat_history = self.config.chat_history messages: List[dict] = [system_message] if chat_history in ("rolling", "full") else [] @@ -199,9 +166,7 @@ def run( if logger.isEnabledFor(logging.INFO): logger.info( - "Episode start: task_id=%s seed=%s max_steps=%s querying=%s observation=%s context_window=%s chat_history=%s", - self.task_spec.task_id, - self.task_spec.seed, + "Episode start: max_steps=%s querying=%s observation=%s context_window=%s chat_history=%s", max_steps, self.config.querying, self.config.observation, @@ -233,14 +198,11 @@ def run( agent_messages = messages if logger.isEnabledFor(logging.INFO): logger.info( - "LLM query #%d: task_id=%s observation=%s messages_in_context=%d current_turn_has_image=%s", + "LLM query #%d: messages_in_context=%d current_turn_has_image=%s", query_count, - self.task_spec.task_id, - self.config.observation, len(agent_messages), has_image, ) - _reset_agent_usage(agent) t_llm = time.perf_counter() model_text = agent(agent_messages) llm_s = time.perf_counter() - t_llm @@ -251,50 +213,35 @@ def run( action_queue = self.querying.parse_actions(model_text) if logger.isEnabledFor(logging.INFO): logger.info( - "LLM query #%d finished: task_id=%s observation=%s elapsed=%.2fs reply_chars=%d actions_parsed=%d", + "LLM query #%d finished in %.2fs: reply_chars=%d actions_parsed=%d", query_count, - self.task_spec.task_id, - self.config.observation, llm_s, len(model_text), len(action_queue), ) if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "LLM query #%d reply: task_id=%s observation=%s\n%s", - query_count, - self.task_spec.task_id, - self.config.observation, - model_text, - ) - query_record = { - "kind": "query", - "query_index": query_count, - "env_step_count": state.step_count, - "agent_messages": copy.deepcopy(agent_messages), - "assistant_reply": model_text, - "parsed_actions": list(action_queue), - "parse_ok": bool(action_queue), - "has_image": has_image, - "llm_latency_s": llm_s, - "chat_history_mode": chat_history, - "agent_message_count": len(agent_messages), - "actions_remaining_before_step": len(action_queue), - } - usage = getattr(agent, "last_usage", None) - if isinstance(usage, dict): - query_record["usage"] = dict(usage) - transcript.append(query_record) - # check if we got any valid actions; - # if not, we'll count it as a parse failure and give feedback, - # but still allow retries until max_parse_retries is reached + logger.debug("LLM query #%d reply:\n%s", query_count, model_text) + transcript.append( + { + "kind": "query", + "query_index": query_count, + "env_step_count": state.step_count, + "agent_messages": copy.deepcopy(agent_messages), + "assistant_reply": model_text, + "parsed_actions": list(action_queue), + "parse_ok": bool(action_queue), + "has_image": has_image, + "llm_latency_s": llm_s, + "chat_history_mode": chat_history, + "agent_message_count": len(agent_messages), + "actions_remaining_before_step": len(action_queue), + } + ) if not action_queue: parse_failures += 1 logger.warning( - "LLM query #%d: task_id=%s observation=%s no valid actions parsed; parse failure %d/%d", + "LLM query #%d: no valid actions parsed; parse failure %d/%d", query_count, - self.task_spec.task_id, - self.config.observation, parse_failures, self.config.max_parse_retries, ) @@ -309,7 +256,6 @@ def run( continue parse_failures = 0 - # if action_queue is empty due to all actions having been executed, end the episode if not action_queue: end_reason = "exhausted" break