Skip to content

Serialize code-derived values as raw strings #778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
eb8680 wants to merge 1 commit into from
Open

Serialize code-derived values as raw strings #778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions effectful/handlers/llm/harness/observability/rich.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,9 +240,9 @@ def _is_python(text: str, *, partial: bool = False) -> bool:
"""Whether `text` looks like a Python source snippet worth highlighting.

Detects code by *content* rather than schema/field name, so it covers every
`Encodable` type that serializes Python as a string -- the synthesis
`SynthesizedFunction.code` field, `exec_code`'s `types.CodeType`
argument, and any future code-carrying tool -- uniformly. Requires a
`Encodable` type that serializes Python as a string -- a synthesized
`Callable`, `exec_code`'s `types.CodeType` argument, and any future
code-carrying tool -- uniformly. Requires a
multi-line string that parses as a module with at least one real statement
(not a lone expression), which excludes prose and JSON-as-string.

Expand All @@ -261,8 +261,8 @@ def _is_python(text: str, *, partial: bool = False) -> bool:
def _extract_code(args: typing.Any, *, partial: bool = False) -> str | None:
"""Return an embedded Python source string from parsed tool-call arguments.

Walks nested dicts (a synthesized callable is ``{"implementation":
{"code": ...}}``; `exec_code` is a flat ``{"code": ...}``) and returns
Walks nested dicts (a synthesized callable is ``{"implementation": ...}``;
`exec_code` is ``{"code": ...}``) and returns
the first string value that :func:`_is_python` recognizes. ``partial`` is
forwarded, and is set while streaming, where the payload is source cut
mid-line and has to be judged as a prefix rather than as a module.
Expand Down
50 changes: 21 additions & 29 deletions effectful/handlers/llm/harness/serialization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,40 +712,33 @@ def _pydantic_type_image(ty: type[Image.Image]):
]


# The *serialization* view of a synthesized callable: the shape the model reads
# when a function is handed to it as a value (e.g. a tool's return) -- just the
# source, with none of the synthesis instructions the `SynthesizedFunction` subtype
# carries for the generation direction. Its JSON schema (docstring included, since
# pydantic renders it as the schema `description`) is what every synthesized-callable
# encoding serializes as, so keep the docstring model-facing.
class EncodedFunction(pydantic.BaseModel):
"""A function, encoded as a string of its complete Python source."""

code: str = pydantic.Field(..., description="Python source defining the function.")


def _serialize_callable(value: collections.abc.Callable) -> EncodedFunction:
"""Encode a callable back to its ``code`` form (source, or a stub).

Emits a plain `EncodedFunction` -- which is exactly what the serialization JSON
schema declares -- rather than the `SynthesizedFunction` subclass that governs
the *other* direction. The two directions carry different obligations, and
conflating them was a real bug: `SynthesizedFunction`'s constraints ("the last
statement must be a function definition", "every parameter must be annotated")
are demands on code a model is *writing*, and a value being serialized is under
no such obligation. It is any callable that reached this point -- a class handed
back by a lexical-scope read, or an inner function a Skill *body* returned,
which was never required to annotate anything -- and re-validating its recovered
source rejected those with a `ValidationError` raised from inside pydantic's
serializer, aborting the whole call rather than encoding the value.
# The *serialization* view of a synthesized callable: the shape the model reads when
# a function is handed to it as a value (e.g. a tool's return) -- the bare source,
# with none of the synthesis instructions the validation direction carries.
EncodedFunction = typing.Annotated[
str,
pydantic.Field(
description="A function, as a string of its complete Python source."
),
]


def _serialize_callable(value: collections.abc.Callable) -> str:
"""Encode a callable as its source, or as a stub when there is none.

The synthesis constraints ("the last statement must be a function definition",
"every parameter must be annotated") are demands on code a model is *writing*;
a value being serialized is under no such obligation -- it may be a class from a
lexical-scope read, or an inner function a Skill body returned -- so they are
deliberately not re-applied here.
"""
try:
source = inspect.getsource(value)
except (OSError, TypeError):
source = None

if source:
return EncodedFunction(code=textwrap.dedent(source))
return textwrap.dedent(source)

name = getattr(value, "__name__", None)
docstring = inspect.getdoc(value)
Expand All @@ -759,11 +752,10 @@ def _serialize_callable(value: collections.abc.Callable) -> EncodedFunction:
except (ValueError, TypeError):
sig_str = "(...)"

stub_code = f'''def {name}{sig_str}:
return f'''def {name}{sig_str}:
"""{docstring}"""
...
'''
return EncodedFunction(code=stub_code)


@TypeToPydanticType.register(collections.abc.Callable)
Expand Down
143 changes: 36 additions & 107 deletions effectful/handlers/llm/harness/synthesis/body.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import collections.abc
import functools
import inspect
import textwrap
import types
import typing
from collections.abc import Callable
Expand Down Expand Up @@ -69,9 +68,10 @@
)
from effectful.handlers.llm.harness.synthesis.function import (
SplicedRegion,
SynthesizedFunction,
_checked_source,
_def_nodes,
_recover_skill_def,
_synthesized_source_schema,
)
from effectful.handlers.llm.types import Encodable, Skill, Tool
from effectful.ops.semantics import fwd, handler
Expand Down Expand Up @@ -177,40 +177,20 @@ def __class_getitem__(cls, item):
return types.GenericAlias(cls, item)


class SynthesizedSkillBody(SynthesizedFunction):
"""Structured output for synthesizing a `Skill`'s body (`write_and_run_body`).
# A Skill body is type-checked against the Skill's own (already-annotated) signature
# by `_splice_body`, so the synthesized body's own annotations are optional.
_SKILL_BODY_CONSTRAINTS: list[str] = [
"Write the function with the Skill's signature; parameter and return "
"annotations are optional.",
"Do not include a docstring or doctests; the Skill's are supplied automatically.",
]

Decoded through `_pydantic_skill_body`: the function is type-checked against
the enclosing Skill's source and its doctests are run with self/recursive
calls routed to the synthesized implementation.

Unlike `SynthesizedFunction`, the parameter and return *annotations* are not
required: a Skill body is type-checked against the Skill's own signature
(see `splice_skill_body`), so the model may omit or vary them -- in
particular it need not annotate the ``self`` receiver of an instance-method
Skill.
"""

code: str = pydantic.Field(
...,
description=textwrap.dedent("""
The complete Python source implementing the Skill shown in its spec.
The code MUST satisfy the following constraints, or it will fail validation:

<constraints>
1. The code MUST be one complete syntactically valid Python module.
2. The code MUST NOT use star imports or ``__future__`` imports.
3. The function definition MUST be the LAST statement - do not add any code after it.
4. Write the function with the Skill's signature; parameter and return
annotations are optional.
5. Do not include a docstring or doctests; the Skill's are supplied automatically.
</constraints>
"""),
)

# A Skill body is checked against the Skill's own (already-annotated)
# signature, so the synthesized body's annotations are optional.
_require_annotations: typing.ClassVar[bool] = False
_METHOD_SKILL_BODY_CONSTRAINTS: list[str] = [
"Write the function with the Skill's signature: its FIRST parameter is the "
"instance receiver ``self`` (which you may leave unannotated); all other "
"parameter and return annotations are optional too.",
"Do not include a docstring or doctests; the Skill's are supplied automatically.",
]


@TypeToPydanticType.register(SkillBody)
Expand All @@ -223,30 +203,30 @@ def _pydantic_skill_body(ty: typing.Any) -> typing.Any:
implementation, so a doctest that calls the Skill (including for recursion)
exercises the freshly synthesized code rather than re-invoking the model.
"""
typed_enc = SynthesizedSkillBody._create_model_from_callable_type(
schema = _synthesized_source_schema(
ty if typing.get_args(ty) else Callable[..., typing.Any], # type: ignore[arg-type]
"the Skill shown in its spec",
_SKILL_BODY_CONSTRAINTS,
)

def _validate(
value: SynthesizedSkillBody | dict | str | Callable,
value: str | Callable,
info: pydantic.ValidationInfo,
) -> Callable:
if isinstance(value, str):
value = typed_enc.model_validate({"code": value})
elif isinstance(value, dict):
value = typed_enc.model_validate(value)
elif callable(value):
value = _checked_source(value, require_annotations=False)
if not isinstance(value, str):
return typing.cast(Callable, value)

ctx = info.context or {}
anchor = ctx.get(_TYPE_CHECK_ANCHOR_KEY)
if anchor is not None:
# skill bodies should not have access to call-local variables
assert isinstance(anchor, Skill)
ctx = anchor.__context__

filename = f"<synthesis:{id(value.code)}>"
filename = f"<synthesis:{id(value)}>"
module: ast.Module = effectful.handlers.llm.harness.execution.hooks.parse(
value.code, filename
value, filename
)

# `None` means the Skill's source can't be recovered (REPL/exec/notebook
Expand Down Expand Up @@ -282,7 +262,7 @@ def _validate(

return typing.Annotated[
pydantic.InstanceOf[ty_], # type: ignore
pydantic.BeforeValidator(_validate, json_schema_input_type=typed_enc),
pydantic.BeforeValidator(_validate, json_schema_input_type=schema),
pydantic.PlainSerializer(_serialize_callable, return_type=EncodedFunction),
]

Expand All @@ -299,58 +279,6 @@ class MethodSkillBody(SkillBody):
"""


class SynthesizedMethodSkillBody(SynthesizedSkillBody):
"""Structured output for synthesizing an *instance-method* `Skill`'s body.

Decoded through `_pydantic_skill_body`: the function is type-checked against
the enclosing Skill's source and its doctests are run with self/recursive
calls routed to the synthesized implementation.

Unlike `SynthesizedFunction`, the parameter and return *annotations* are not
required: a Skill body is type-checked against the Skill's own signature
(see `splice_skill_body`), so the model may omit or vary them -- in
particular it need not annotate the ``self`` receiver of an instance-method
Skill.
"""

code: str = pydantic.Field(
...,
description=textwrap.dedent("""
The complete Python source implementing the instance-method Skill shown in
its spec. The code MUST satisfy the following constraints, or it will fail
validation:

<constraints>
1. The code MUST be one complete syntactically valid Python module.
2. The code MUST NOT use star imports or ``__future__`` imports.
3. The function definition MUST be the LAST statement - do not add any code after it.
4. Write the function with the Skill's signature: its FIRST parameter is the
instance receiver ``self`` (which you may leave unannotated); all other parameter
and return annotations are optional too.
5. Do not include a docstring or doctests; the Skill's are supplied automatically.
</constraints>
"""),
)

@classmethod
def _param_names(cls, param_types: typing.Iterable[typing.Any]) -> list[str]:
# The method's callable type already carries the receiver as its first
# parameter (with an uninformative Agent-class type); relabel it ``self`` so
# the model reproduces it rather than inventing one -- do NOT prepend a receiver.
names = super()._param_names(param_types)
if names:
names[0] = "self"
return names

@classmethod
def _extra_instructions(cls) -> str:
return (
"\n\nThis implements an instance method: the first parameter is the "
"instance receiver `self`. Include it as the first parameter; you may "
"leave it unannotated."
)


def _class_skill_of(op: typing.Any) -> typing.Any | None:
"""The class-level `Skill` underlying an Agent-method Skill ``op``.

Expand Down Expand Up @@ -382,30 +310,31 @@ def _pydantic_method_skill_body(ty: typing.Any) -> typing.Any:
their own instances -- route ``agent.method(...)`` on *any* instance to the
synthesized implementation.
"""
typed_enc = SynthesizedMethodSkillBody._create_model_from_callable_type(
schema = _synthesized_source_schema(
ty if typing.get_args(ty) else Callable[..., typing.Any], # type: ignore[arg-type]
"the instance-method Skill shown in its spec",
_METHOD_SKILL_BODY_CONSTRAINTS,
receiver=True,
)

def _validate(
value: SynthesizedMethodSkillBody | dict | str | Callable,
value: str | Callable,
info: pydantic.ValidationInfo,
) -> Callable:
if isinstance(value, str):
value = typed_enc.model_validate({"code": value})
elif isinstance(value, dict):
value = typed_enc.model_validate(value)
elif callable(value):
value = _checked_source(value, require_annotations=False)
if not isinstance(value, str):
return typing.cast(Callable, value)

ctx = info.context or {}
anchor = ctx.get(_TYPE_CHECK_ANCHOR_KEY)
if anchor is not None:
# skill bodies should not have access to call-local variables
assert isinstance(anchor, Skill)
ctx = anchor.__context__

filename = f"<synthesis:{id(value.code)}>"
filename = f"<synthesis:{id(value)}>"
module: ast.Module = effectful.handlers.llm.harness.execution.hooks.parse(
value.code, filename
value, filename
)
anchor_asts = _recover_skill_def(anchor) if anchor is not None else None
if anchor_asts is not None:
Expand Down Expand Up @@ -445,7 +374,7 @@ def _doctest_apply(op, *args, **kwargs):

return typing.Annotated[
pydantic.InstanceOf[ty_], # type: ignore
pydantic.BeforeValidator(_validate, json_schema_input_type=typed_enc),
pydantic.BeforeValidator(_validate, json_schema_input_type=schema),
pydantic.PlainSerializer(_serialize_callable, return_type=EncodedFunction),
]

Expand Down
Loading
Loading