From 1b059ebf3f6cc67894c2770f909959e1816742c2 Mon Sep 17 00:00:00 2001 From: datvo06 Date: Thu, 13 Aug 2026 12:22:03 -0400 Subject: [PATCH 1/2] Validate a submit_solution answer against its declared return type A Template answered by the direct final message has its result validated through Encodable[return_type], so a return type carrying an AfterValidator (a compile gate, a probe) is enforced. A Template answered via submit_solution did not: it returned implementation(*args, **kwargs) unchecked, so the same validator was silently skipped depending only on how the model chose to reply. Run the declared return type's value-validators on the submitted answer. The answer is an already-built value, not the source the encoding's decode expects, so apply only the AfterValidators the Encodable annotation carries -- letting pydantic dispatch (value)/(value, info) and thread the decode context (the Template's lexical scope plus its arguments, as the direct path builds it). Return types without a value-validator are unaffected. --- .../handlers/llm/harness/synthesis/body.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/effectful/handlers/llm/harness/synthesis/body.py b/effectful/handlers/llm/harness/synthesis/body.py index 4c33799c..021fdcf9 100644 --- a/effectful/handlers/llm/harness/synthesis/body.py +++ b/effectful/handlers/llm/harness/synthesis/body.py @@ -555,7 +555,28 @@ def submit_solution(implementation: body_type) -> return_type: # type: ignore it (see the "Code synthesis" section); its return value on the original arguments becomes the answer. """ - return implementation(*args, **kwargs) # type: ignore + answer = implementation(*args, **kwargs) # type: ignore + # The direct final-message path validates the answer through + # `Encodable[return_type]`, so a return type carrying a validator + # is enforced there. A `submit_solution` answer must be checked the + # same way, or that validator is silently skipped. The answer is an + # already-built value, not the source the encoding's decode expects, + # so apply only the declared type's value-validators (its + # `AfterValidator`s), letting pydantic dispatch (value)/(value, info) + # and thread the decode context (the Template's lexical scope plus + # its arguments, matching how the direct path builds it). + encodable = TypeToPydanticType().evaluate(return_type) + afters = [ + m + for m in getattr(encodable, "__metadata__", ()) + if isinstance(m, pydantic.AfterValidator) + ] + if afters: + context = {**dict(template.__context__), **bound_args.arguments} + answer = pydantic.TypeAdapter( + typing.Annotated[typing.Any, *afters] + ).validate_python(answer, context=context) + return answer # type: ignore return super().define(submit_solution, name=cls.__toolname__) From 50c6bdb4b487c90e4d314fd2a1db7f91269f718e Mon Sep 17 00:00:00 2001 From: datvo06 Date: Thu, 13 Aug 2026 13:35:18 -0400 Subject: [PATCH 2/2] Complete the answer validation: full default check unless decode-style Extracting only AfterValidators was incomplete: it skipped types with no Annotated metadata entirely (int, BaseModel, list), and Before/Wrap validators and nested element validators never ran. The complete check is pydantic's own: validate the answer through the full evaluated encoding. The exception is a decode-style encoding, one carrying a PlainValidator anywhere (a synthesized callable's source, an image's data URL): its core validation is a decode from the wire form, so an already-built value cannot go through it (demonstrated: a built callable raises AttributeError on module_code, a built PIL image fails validation). For those, run only the top-level post-decode AfterValidators, which is the post-decode set by construction. str returns and missing annotations are skipped, matching the direct path's special case. Known parity gap, upstream and shared by both paths: TypeToPydanticType wraps an Annotated return's metadata in a tuple (Annotated[int, (AfterValidator(...),)]), so pydantic ignores user-authored constraints on the direct path and here alike; that is a separate fix. --- .../handlers/llm/harness/synthesis/body.py | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/effectful/handlers/llm/harness/synthesis/body.py b/effectful/handlers/llm/harness/synthesis/body.py index 021fdcf9..65f8fa38 100644 --- a/effectful/handlers/llm/harness/synthesis/body.py +++ b/effectful/handlers/llm/harness/synthesis/body.py @@ -445,6 +445,21 @@ def _callable_type_from_signature( return collections.abc.Callable[param_types, return_type] # type: ignore +def _has_plain_validator(ty: typing.Any) -> bool: + """Whether an `Encodable` annotation decodes from a wire form anywhere. + + A `PlainValidator` in the encoding (a synthesized callable's source, an + image's data URL) replaces core validation with a decode step, so an + already-built value of the type cannot be re-validated through it. + """ + if any( + isinstance(m, pydantic.PlainValidator) + for m in getattr(ty, "__metadata__", ()) + ): + return True + return any(_has_plain_validator(arg) for arg in typing.get_args(ty)) + + class FinalBodySynthesizer(ObjectInterpretation): """You may "answer" a Template by writing code instead of producing the value directly. The `submit_solution` tool accepts a single argument: a Python @@ -556,26 +571,27 @@ def submit_solution(implementation: body_type) -> return_type: # type: ignore original arguments becomes the answer. """ answer = implementation(*args, **kwargs) # type: ignore - # The direct final-message path validates the answer through - # `Encodable[return_type]`, so a return type carrying a validator - # is enforced there. A `submit_solution` answer must be checked the - # same way, or that validator is silently skipped. The answer is an - # already-built value, not the source the encoding's decode expects, - # so apply only the declared type's value-validators (its - # `AfterValidator`s), letting pydantic dispatch (value)/(value, info) - # and thread the decode context (the Template's lexical scope plus - # its arguments, matching how the direct path builds it). + if return_type is inspect.Signature.empty or return_type is str: + return answer # type: ignore encodable = TypeToPydanticType().evaluate(return_type) - afters = [ - m - for m in getattr(encodable, "__metadata__", ()) - if isinstance(m, pydantic.AfterValidator) - ] - if afters: - context = {**dict(template.__context__), **bound_args.arguments} - answer = pydantic.TypeAdapter( - typing.Annotated[typing.Any, *afters] - ).validate_python(answer, context=context) + context = {**dict(template.__context__), **bound_args.arguments} + if _has_plain_validator(encodable): + # The answer is already in decoded form; a `PlainValidator` + # encoding expects the wire form, so run only the + # post-decode checks. + afters = [ + m + for m in getattr(encodable, "__metadata__", ()) + if isinstance(m, pydantic.AfterValidator) + ] + if afters: + answer = pydantic.TypeAdapter( + typing.Annotated[typing.Any, *afters] + ).validate_python(answer, context=context) + else: + answer = pydantic.TypeAdapter(encodable).validate_python( + answer, context=context + ) return answer # type: ignore return super().define(submit_solution, name=cls.__toolname__)