Skip to content

Remove stale text initial maze prompt strategy #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
helenlu66 wants to merge 1 commit into from
Closed

Remove stale text initial maze prompt strategy #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion interface/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
class ExperimentConfig:
"""Selects one implementation along each experimental axis."""

prompting: Literal["minimal", "standard", "verbose", "text_initial_maze"] = "standard"
prompting: Literal["minimal", "standard", "verbose"] = "standard"
observation: Literal["text_only", "image_text", "image_only"] = "image_only"
include_current_observation_description: bool = False
observation_text_includes_facing: bool = False
Expand Down
15 changes: 0 additions & 15 deletions interface/prompt_strategies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,6 @@ def build_system_prompt(self, querying_suffix: str = "") -> str:
return "\n\n".join([std, MECHANISM_RULES])


class TextInitialMazePromptStrategy(StandardPromptStrategy):
"""Standard system prompt plus the initial maze section placeholder.

This strategy returns the standard system prompt and appends the
`INITIAL_MAZE_SECTION` template (containing the `{maze_text}` placeholder).
The caller (for example `ExperimentRunner.build_prompt_message`) is
responsible for formatting `{maze_text}` with the rendered maze text.
"""

def build_system_prompt(self, querying_suffix: str = "") -> str:
del querying_suffix
std = StandardPromptStrategy.build_system_prompt(self).rstrip()
return "\n\n".join([std, system_templates.INITIAL_MAZE_SECTION])


PromptStrategy = MinimalPromptStrategy


Expand Down
100 changes: 23 additions & 77 deletions interface/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
current_observation_text,
history_content_blocks,
history_text,
recent_history_steps,
)
from interface.parser import ACTIONS_HINT
from interface.prompt_strategies import (
Expand All @@ -32,12 +31,9 @@
StandardPromptStrategy,
VerbosePromptStrategy,
)
from interface.prompt_strategies import TextInitialMazePromptStrategy
from interface.querying import QueryingMode
from interface.renderer import render_initial_maze_text
from prompting_experiments.prompt_templates import feedback as feedback_templates
from prompting_experiments.prompt_templates import querying as querying_templates
from prompting_experiments.prompt_templates import system as system_templates
from prompting_experiments.prompt_templates import user as user_templates

logger = logging.getLogger(__name__)
Expand All @@ -46,7 +42,6 @@
"minimal": MinimalPromptStrategy,
"standard": StandardPromptStrategy,
"verbose": VerbosePromptStrategy,
"text_initial_maze": TextInitialMazePromptStrategy,
}


Expand All @@ -64,18 +59,6 @@ def _trim_rolling_chat(messages: List[dict], max_pairs: int) -> None:
del messages[1 : 1 + (tail_len - cap)]


def _reset_agent_usage(agent: Callable[[List[dict]], str]) -> None:
"""Clear per-call telemetry so stale usage cannot leak into a later query."""
reset_usage = getattr(agent, "reset_usage", None)
if callable(reset_usage):
reset_usage()
return
try:
setattr(agent, "last_usage", None)
except (AttributeError, TypeError):
pass


def _replace_current_question(prompt_text: str, question: str) -> str:
standard_question = user_templates.NEXT_ACTION_QUESTION
before, match, after = prompt_text.rpartition(standard_question)
Expand Down Expand Up @@ -147,21 +130,7 @@ def build_prompt_message(
last_feedback: str,
transcript: List[dict],
) -> tuple[str, dict]:
system_prompt = self.prompt.build_system_prompt()
# If the system prompt includes the `{maze_text}` placeholder, format
# it with the rendered maze. Otherwise, for text observations append
# the `INITIAL_MAZE_SECTION` so the maze is present in system-level
# context for text-only or image+text modes.
if "{maze_text}" in system_prompt:
system_prompt = system_prompt.format(maze_text=render_initial_maze_text(self.task_spec))
elif self.config.observation in ("text_only", "image_text"):
maze_text = render_initial_maze_text(self.task_spec)
system_prompt = (
system_prompt
+ "\n\n"
+ system_templates.INITIAL_MAZE_SECTION.format(maze_text=maze_text)
)
return system_prompt, self._build_message(
return self.prompt.build_system_prompt(), self._build_message(
state,
last_feedback,
transcript,
Expand All @@ -177,9 +146,7 @@ def run(
self.last_rgb, state, reset_info = self.backend.reset(seed=self.task_spec.seed)
self.querying.reset()

# Build the initial system prompt (may include the initial maze for
# text-based observations) and the initial user message block.
system_prompt, _ = self.build_prompt_message(state, feedback_templates.INITIAL_FEEDBACK, [])
system_prompt = self.prompt.build_system_prompt()
system_message = {"role": "system", "content": system_prompt}
chat_history = self.config.chat_history
messages: List[dict] = [system_message] if chat_history in ("rolling", "full") else []
Expand All @@ -199,9 +166,7 @@ def run(

if logger.isEnabledFor(logging.INFO):
logger.info(
"Episode start: task_id=%s seed=%s max_steps=%s querying=%s observation=%s context_window=%s chat_history=%s",
self.task_spec.task_id,
self.task_spec.seed,
"Episode start: max_steps=%s querying=%s observation=%s context_window=%s chat_history=%s",
max_steps,
self.config.querying,
self.config.observation,
Expand Down Expand Up @@ -233,14 +198,11 @@ def run(
agent_messages = messages
if logger.isEnabledFor(logging.INFO):
logger.info(
"LLM query #%d: task_id=%s observation=%s messages_in_context=%d current_turn_has_image=%s",
"LLM query #%d: messages_in_context=%d current_turn_has_image=%s",
query_count,
self.task_spec.task_id,
self.config.observation,
len(agent_messages),
has_image,
)
_reset_agent_usage(agent)
t_llm = time.perf_counter()
model_text = agent(agent_messages)
Comment on lines 206 to 207
llm_s = time.perf_counter() - t_llm
Expand All @@ -251,50 +213,35 @@ def run(
action_queue = self.querying.parse_actions(model_text)
if logger.isEnabledFor(logging.INFO):
logger.info(
"LLM query #%d finished: task_id=%s observation=%s elapsed=%.2fs reply_chars=%d actions_parsed=%d",
"LLM query #%d finished in %.2fs: reply_chars=%d actions_parsed=%d",
query_count,
self.task_spec.task_id,
self.config.observation,
llm_s,
len(model_text),
len(action_queue),
)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"LLM query #%d reply: task_id=%s observation=%s\n%s",
query_count,
self.task_spec.task_id,
self.config.observation,
model_text,
)
query_record = {
"kind": "query",
"query_index": query_count,
"env_step_count": state.step_count,
"agent_messages": copy.deepcopy(agent_messages),
"assistant_reply": model_text,
"parsed_actions": list(action_queue),
"parse_ok": bool(action_queue),
"has_image": has_image,
"llm_latency_s": llm_s,
"chat_history_mode": chat_history,
"agent_message_count": len(agent_messages),
"actions_remaining_before_step": len(action_queue),
}
usage = getattr(agent, "last_usage", None)
if isinstance(usage, dict):
query_record["usage"] = dict(usage)
transcript.append(query_record)
# check if we got any valid actions;
# if not, we'll count it as a parse failure and give feedback,
# but still allow retries until max_parse_retries is reached
logger.debug("LLM query #%d reply:\n%s", query_count, model_text)
transcript.append(
{
"kind": "query",
"query_index": query_count,
"env_step_count": state.step_count,
"agent_messages": copy.deepcopy(agent_messages),
"assistant_reply": model_text,
"parsed_actions": list(action_queue),
"parse_ok": bool(action_queue),
"has_image": has_image,
"llm_latency_s": llm_s,
"chat_history_mode": chat_history,
"agent_message_count": len(agent_messages),
"actions_remaining_before_step": len(action_queue),
}
)
Comment on lines +224 to +239
if not action_queue:
parse_failures += 1
logger.warning(
"LLM query #%d: task_id=%s observation=%s no valid actions parsed; parse failure %d/%d",
"LLM query #%d: no valid actions parsed; parse failure %d/%d",
query_count,
self.task_spec.task_id,
self.config.observation,
parse_failures,
self.config.max_parse_retries,
)
Expand All @@ -309,7 +256,6 @@ def run(
continue
parse_failures = 0

# if action_queue is empty due to all actions having been executed, end the episode
if not action_queue:
end_reason = "exhausted"
break
Expand Down