diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md index 36e37e06b7..033d2a376a 100644 --- a/loopx/capabilities/benchmark_toolkit/README.md +++ b/loopx/capabilities/benchmark_toolkit/README.md @@ -558,6 +558,20 @@ the feedback transport; the receipt otherwise contains only digests and submission/delivery counts. A prose claim that feedback was expected, an unbound score log, an undeclared channel, or an incomplete transport does not relax the blind default. +A runner may combine those declared channels with periodic background evaluation by +adding `background_evaluation={"feedback":"blinded","timing":"during_solving"}` +to the same protocol. This mixed mode additionally requires +`verifier_started_after_agent`, `background_evaluation_log_complete`, and +`background_feedback_blinded`. Its private log binds ordered snapshot and evaluation +digests and timestamps: each snapshot and evaluation must start while the solver is +still running, although an evaluation that already started may finish afterward. The +compact receipt exposes only their count. Background results never enter declared +deliveries. If the solver never uses an active feedback channel, explicit empty +submissions, deliveries, transport counters, and record digests remain valid only when +at least one complete blinded background evaluation is present. Missing background +records, any evaluation log under the default blind policy, an undeclared background +log, or a background result delivered to the solver fails closed. + Qualification rejects a run when it detects any of the following: - post-run agent confirmation that restricted answer, out-of-scope task-source, diff --git a/loopx/capabilities/benchmark_toolkit/catalog_entry.py b/loopx/capabilities/benchmark_toolkit/catalog_entry.py index b1a57343ed..05418399ee 100644 --- a/loopx/capabilities/benchmark_toolkit/catalog_entry.py +++ b/loopx/capabilities/benchmark_toolkit/catalog_entry.py @@ -983,6 +983,14 @@ "feedback": "declared", "timing": "during_solving", }, + "mixed_background_opt_in": { + "feedback": "declared", + "timing": "during_solving", + "background_evaluation": { + "feedback": "blinded", + "timing": "during_solving", + }, + }, "required_runner_attestations": [ "evaluation_protocol_enforced", "evaluation_log_complete", @@ -994,10 +1002,13 @@ "ordered_submissions", "declared_channel_deliveries", "acknowledged_feedback_transport", + "complete_blinded_background_evaluation_log_when_declared", ], "boundary": ( "Only a preregistered policy plus complete private runner evidence " - "replaces the blind-protocol attestations; prose or partial score " + "replaces the blind-protocol attestations. Mixed protocols keep " + "background results blinded and bind their complete evaluation " + "log separately from declared deliveries; prose or partial score " "logs remain unqualified." ), }, diff --git a/loopx/capabilities/benchmark_toolkit/integrity.py b/loopx/capabilities/benchmark_toolkit/integrity.py index dc9be63008..ca929fda93 100644 --- a/loopx/capabilities/benchmark_toolkit/integrity.py +++ b/loopx/capabilities/benchmark_toolkit/integrity.py @@ -76,6 +76,12 @@ "evaluation_log_complete", "verifier_started_after_submission", ) +_MIXED_FEEDBACK_RUNTIME_ATTESTATIONS = ( + *_DECLARED_FEEDBACK_RUNTIME_ATTESTATIONS, + "verifier_started_after_agent", + "background_evaluation_log_complete", + "background_feedback_blinded", +) NETWORK_ACCESS_MODES = ("denied", "loopback_only", "permitted_solving") DEFAULT_NETWORK_ACCESS = "denied" @@ -243,6 +249,7 @@ def required_runtime_attestations( network_access: str, *, declared_feedback: bool = False, + background_evaluation: bool = False, ) -> tuple[str, ...]: """Return the runner attestations required for one network access mode. @@ -254,7 +261,10 @@ def required_runtime_attestations( ``network_permitted_solving`` instead and every restricted-resource denial still applies. The default evaluation contract remains blinded. A validated declared-feedback policy replaces only its two evaluation-order attestations - with the protocol, log, and post-submission verifier attestations. + with the protocol, log, and post-submission verifier attestations. A mixed + protocol that also runs blinded background evaluation additionally proves + that every background evaluation starts after the agent, its log is + complete, and none of its results enter the solving context. """ mode = ( @@ -262,11 +272,12 @@ def required_runtime_attestations( if network_access in NETWORK_ACCESS_MODES else DEFAULT_NETWORK_ACCESS ) - evaluation_attestations = ( - _DECLARED_FEEDBACK_RUNTIME_ATTESTATIONS - if declared_feedback - else _BLINDED_EVALUATION_RUNTIME_ATTESTATIONS - ) + if background_evaluation: + evaluation_attestations = _MIXED_FEEDBACK_RUNTIME_ATTESTATIONS + elif declared_feedback: + evaluation_attestations = _DECLARED_FEEDBACK_RUNTIME_ATTESTATIONS + else: + evaluation_attestations = _BLINDED_EVALUATION_RUNTIME_ATTESTATIONS if mode == "permitted_solving": return ( *_COMMON_RUNTIME_ATTESTATIONS, @@ -541,6 +552,16 @@ def _validated_evaluation_protocol( raise ValueError("benchmark_evaluation_protocol_feedback_invalid") if raw_protocol.get("timing") != "during_solving": raise ValueError("benchmark_evaluation_protocol_timing_invalid") + raw_background = raw_protocol.get("background_evaluation") + if raw_background is not None: + if not isinstance(raw_background, Mapping): + raise TypeError("benchmark_background_evaluation_invalid") + if set(raw_background) != {"feedback", "timing"}: + raise ValueError("benchmark_background_evaluation_fields_invalid") + if raw_background.get("feedback") != "blinded": + raise ValueError("benchmark_background_evaluation_feedback_invalid") + if raw_background.get("timing") != "during_solving": + raise ValueError("benchmark_background_evaluation_timing_invalid") raw_channels = raw_protocol.get("channels") if ( not isinstance(raw_channels, list) @@ -587,13 +608,19 @@ def _validated_evaluation_protocol( "output_kinds": output_kinds, } ) - return { + protocol = { "protocol_id": protocol_id, "feedback": "declared", "timing": "during_solving", "channels": tuple(channels), "policy_sha256": policy_sha256, } + if raw_background is not None: + protocol["background_evaluation"] = { + "feedback": "blinded", + "timing": "during_solving", + } + return protocol def _validated_restricted_access_adjudication( @@ -1009,6 +1036,7 @@ def _validated_declared_feedback_runtime( solver_start_ms = timeline[1] if timeline is not None else 0 solver_end_ms = timeline[2] if timeline is not None else 0 + background_declared = protocol.get("background_evaluation") is not None raw_submissions_value = raw_runtime.get("submissions") submissions = ( raw_submissions_value @@ -1020,7 +1048,7 @@ def _validated_declared_feedback_runtime( post_solver_completion_indexes: list[int] = [] submissions_valid = ( isinstance(raw_submissions_value, list) - and bool(submissions) + and (bool(submissions) or background_declared) and len(submissions) <= 10_000 ) if submissions_valid: @@ -1152,6 +1180,69 @@ def _validated_declared_feedback_runtime( if not deliveries_valid: failures.append("evaluation_protocol_deliveries_invalid") + raw_background_value = raw_runtime.get("background_evaluations") + background_evaluations = ( + raw_background_value if isinstance(raw_background_value, list) else [] + ) + background_valid = not background_declared and raw_background_value is None + if background_declared: + background_valid = ( + isinstance(raw_background_value, list) + and bool(background_evaluations) + and len(background_evaluations) <= 50_000 + ) + previous_background_capture: int | None = None + background_ids: set[str] = set() + if background_valid: + for raw_evaluation in background_evaluations: + if not isinstance(raw_evaluation, Mapping): + background_valid = False + break + evaluation_id = str(raw_evaluation.get("evaluation_id") or "") + captured_at = raw_evaluation.get("captured_at_ms") + evaluation_started_at = raw_evaluation.get( + "evaluation_started_at_ms" + ) + evaluation_completed_at = raw_evaluation.get( + "evaluation_completed_at_ms" + ) + if ( + _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(evaluation_id) is None + or evaluation_id in background_ids + or _SHA256_PATTERN.fullmatch( + str(raw_evaluation.get("snapshot_sha256") or "") + ) + is None + or _SHA256_PATTERN.fullmatch( + str(raw_evaluation.get("evaluation_sha256") or "") + ) + is None + or not _positive_int(captured_at) + or not _positive_int(evaluation_started_at) + or not _positive_int(evaluation_completed_at) + or timeline is None + or not ( + solver_start_ms + <= captured_at + <= solver_end_ms + and captured_at + <= evaluation_started_at + <= solver_end_ms + and evaluation_started_at + <= evaluation_completed_at + ) + or ( + previous_background_capture is not None + and captured_at <= previous_background_capture + ) + ): + background_valid = False + break + background_ids.add(evaluation_id) + previous_background_capture = captured_at + if not background_valid: + failures.append("evaluation_protocol_background_evaluations_invalid") + raw_transport = runtime_attestation.get("feedback_transport") transport = raw_transport if isinstance(raw_transport, Mapping) else None transport_valid = deliveries_valid and transport is not None @@ -1162,8 +1253,8 @@ def _validated_declared_feedback_runtime( ack_count = transport.get("ack_count") record_digests = transport.get("record_digests") if ( - not _positive_int(request_count) - or not _positive_int(response_ack_count) + not _nonnegative_int(request_count) + or not _nonnegative_int(response_ack_count) or not _nonnegative_int(intent_count) or not _nonnegative_int(ack_count) ): @@ -1172,6 +1263,7 @@ def _validated_declared_feedback_runtime( transport_valid = bool( request_count == response_ack_count and request_count >= len(submissions_by_id) + and (request_count > 0 or background_declared) and intent_count == len(deliveries) and ack_count == intent_count and str(transport.get("run_id") or "") == run_id @@ -1194,24 +1286,28 @@ def _validated_declared_feedback_runtime( if failures: return None, failures - return ( - { - "feedback": "declared", + compact_protocol: dict[str, object] = { + "feedback": "declared", + "timing": "during_solving", + "policy_sha256": policy_sha256, + "submission_count": len(submissions), + "delivery_count": len(deliveries), + "evidence_sha256": _sha256_text( + _canonical_text( + { + "evaluation_protocol": raw_runtime, + "feedback_transport": transport, + } + ) + ), + } + if background_declared: + compact_protocol["background_evaluation"] = { + "feedback": "blinded", "timing": "during_solving", - "policy_sha256": policy_sha256, - "submission_count": len(submissions), - "delivery_count": len(deliveries), - "evidence_sha256": _sha256_text( - _canonical_text( - { - "evaluation_protocol": raw_runtime, - "feedback_transport": transport, - } - ) - ), - }, - [], - ) + "evaluation_count": len(background_evaluations), + } + return compact_protocol, [] def build_benchmark_integrity_qualification( @@ -1413,6 +1509,10 @@ def build_benchmark_integrity_qualification( runtime_attestations = required_runtime_attestations( network_access, declared_feedback=evaluation_protocol is not None, + background_evaluation=( + evaluation_protocol is not None + and evaluation_protocol.get("background_evaluation") is not None + ), ) for field in runtime_attestations: if runtime_attestation.get(field) is not True: @@ -1428,6 +1528,10 @@ def build_benchmark_integrity_qualification( protocol=evaluation_protocol, ) attestation_failures.extend(evaluation_protocol_failures) + elif runtime_attestation.get("evaluation_protocol") is not None: + # The default blind policy cannot silently ignore runner evidence for + # an undeclared during-solving or background evaluation protocol. + attestation_failures.append("evaluation_protocol_policy_mismatch") counts = { category: int(evidence_counts.get(category, 0)) diff --git a/tests/capabilities/test_benchmark_toolkit.py b/tests/capabilities/test_benchmark_toolkit.py index aabd5c3216..8535231702 100644 --- a/tests/capabilities/test_benchmark_toolkit.py +++ b/tests/capabilities/test_benchmark_toolkit.py @@ -292,10 +292,21 @@ def test_catalog_exposes_post_run_case_insight_monitor_contract() -> None: "feedback": "declared", "timing": "during_solving", } + assert declared_feedback["mixed_background_opt_in"] == { + "feedback": "declared", + "timing": "during_solving", + "background_evaluation": { + "feedback": "blinded", + "timing": "during_solving", + }, + } assert "exact_policy_sha256" in declared_feedback["evidence_binding"] assert "acknowledged_feedback_transport" in ( declared_feedback["evidence_binding"] ) + assert "complete_blinded_background_evaluation_log_when_declared" in ( + declared_feedback["evidence_binding"] + ) assert "must not access" in analysis["role_boundary"]["solver"] active_monitor = analysis["role_boundary"]["active_campaign_monitor"] @@ -547,6 +558,17 @@ def _declared_feedback_policy() -> dict[str, object]: } +def _mixed_feedback_policy() -> dict[str, object]: + policy = _declared_feedback_policy() + protocol = policy["evaluation_protocol"] + assert isinstance(protocol, dict) + protocol["background_evaluation"] = { + "feedback": "blinded", + "timing": "during_solving", + } + return policy + + def _declared_feedback_attestation( trajectory: dict[str, object], policy: dict[str, object], @@ -599,6 +621,52 @@ def _declared_feedback_attestation( return attestation +def _mixed_feedback_attestation( + trajectory: dict[str, object], + policy: dict[str, object], + *, + active_submission: bool, +) -> dict[str, object]: + attestation = _declared_feedback_attestation(trajectory, policy) + runtime = attestation["evaluation_protocol"] + transport = attestation["feedback_transport"] + assert isinstance(runtime, dict) + assert isinstance(transport, dict) + attestation["verifier_started_after_agent"] = True + attestation["background_evaluation_log_complete"] = True + attestation["background_feedback_blinded"] = True + runtime["background_evaluations"] = [ + { + "evaluation_id": "background-1", + "snapshot_sha256": "6" * 64, + "evaluation_sha256": "7" * 64, + "captured_at_ms": 170, + "evaluation_started_at_ms": 175, + "evaluation_completed_at_ms": 190, + }, + { + "evaluation_id": "background-2", + "snapshot_sha256": "8" * 64, + "evaluation_sha256": "9" * 64, + "captured_at_ms": 250, + "evaluation_started_at_ms": 260, + "evaluation_completed_at_ms": 310, + }, + ] + if not active_submission: + runtime["submissions"] = [] + runtime["deliveries"] = [] + transport.update( + request_count=0, + response_ack_count=0, + intent_count=0, + ack_count=0, + deliveries_sha256=_canonical_sha256([]), + record_digests={}, + ) + return attestation + + def test_declared_during_solving_feedback_uses_protocol_attestations() -> None: trajectory = _trajectory() policy = _declared_feedback_policy() @@ -636,6 +704,129 @@ def test_declared_during_solving_feedback_uses_protocol_attestations() -> None: } +@pytest.mark.parametrize("active_submission", [False, True]) +def test_mixed_background_and_declared_feedback_qualifies( + active_submission: bool, +) -> None: + trajectory = _trajectory() + policy = _mixed_feedback_policy() + attestation = _mixed_feedback_attestation( + trajectory, policy, active_submission=active_submission + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is True + assert receipt["runtime_attestation_checks"]["verifier_started_after_agent"] + assert receipt["runtime_attestation_checks"][ + "background_evaluation_log_complete" + ] + assert receipt["runtime_attestation_checks"]["background_feedback_blinded"] + assert "official_feedback_blinded" not in receipt["runtime_attestation_checks"] + assert receipt["evaluation_protocol"]["submission_count"] == int( + active_submission + ) + assert receipt["evaluation_protocol"]["delivery_count"] == int( + active_submission + ) + assert receipt["evaluation_protocol"]["background_evaluation"] == { + "feedback": "blinded", + "timing": "during_solving", + "evaluation_count": 2, + } + + +@pytest.mark.parametrize( + ("mutation", "expected_blocker"), + [ + ( + lambda runtime: runtime["evaluation_protocol"].pop( + "background_evaluations" + ), + "evaluation_protocol_background_evaluations_invalid", + ), + ( + lambda runtime: runtime["evaluation_protocol"][ + "background_evaluations" + ][1].update(captured_at_ms=160), + "evaluation_protocol_background_evaluations_invalid", + ), + ( + lambda runtime: runtime["evaluation_protocol"][ + "background_evaluations" + ][1].update( + evaluation_started_at_ms=310, + evaluation_completed_at_ms=320, + ), + "evaluation_protocol_background_evaluations_invalid", + ), + ( + lambda runtime: runtime.update(background_feedback_blinded=False), + "runtime_attestation_background_feedback_blinded_missing", + ), + ], +) +def test_mixed_feedback_fails_closed_on_incomplete_background_evidence( + mutation: Callable[[dict[str, object]], None], expected_blocker: str +) -> None: + trajectory = _trajectory() + policy = _mixed_feedback_policy() + attestation = _mixed_feedback_attestation( + trajectory, policy, active_submission=False + ) + mutation(attestation) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is False + assert expected_blocker in receipt["blockers"] + + +def test_plain_declared_feedback_rejects_undeclared_background_log() -> None: + trajectory = _trajectory() + policy = _declared_feedback_policy() + attestation = _declared_feedback_attestation(trajectory, policy) + runtime = attestation["evaluation_protocol"] + assert isinstance(runtime, dict) + runtime["background_evaluations"] = [] + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=policy, + ) + + assert receipt["integrity_qualified"] is False + assert "evaluation_protocol_background_evaluations_invalid" in receipt["blockers"] + + +def test_default_blind_policy_rejects_undeclared_evaluation_log() -> None: + trajectory = _trajectory() + declared_policy = _mixed_feedback_policy() + attestation = _mixed_feedback_attestation( + trajectory, + declared_policy, + active_submission=False, + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=_loopback_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert "evaluation_protocol_policy_mismatch" in receipt["blockers"] + + def test_declared_feedback_evidence_digest_binds_transport() -> None: trajectory = _trajectory() policy = _declared_feedback_policy()