Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/agent_env/env/gateway/triggers.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@
_DROP = object() # an unresolved `${x?}` used as a whole value: drop the key / element
# Real seconds between re-reads of the clock while waiting out virtual time, so a rate change, re-arm or clear is followed.
_VIRTUAL_RECHECK_S = 5.0
# Seconds an executor call waits to connect, and for a task poll's answer.
_EXECUTOR_HTTP_TIMEOUT_S = 30.0
# An allowlist, not a size bound: identifier/metadata keys only, since bodies and addresses are PII.
_ECHO_KEYS = frozenset({
"action", "createdTime", "fileName", "kind", "mimeType", "modifiedTime", "name", "role",
Expand Down Expand Up @@ -1564,8 +1566,10 @@ async def _call_executor(self, executor: dict | None, instruction: str, context_
return None
url = f"{executor['a2a_url']}/a2a"
deadline = asyncio.get_running_loop().time() + executor["timeout_seconds"]
async with httpx.AsyncClient(timeout=30) as client:
send = await client.post(url, json={
async with httpx.AsyncClient(timeout=_EXECUTOR_HTTP_TIMEOUT_S) as client:
# an executor holds message/send open until its task ends, so the read waits as long as the executor may take
read_s = max(_EXECUTOR_HTTP_TIMEOUT_S, executor["timeout_seconds"])
send = await client.post(url, timeout=httpx.Timeout(_EXECUTOR_HTTP_TIMEOUT_S, read=read_s), json={
"jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "message/send",
"params": {"message": {"role": "user", "messageId": uuid.uuid4().hex,
"contextId": context_id,
Expand Down
86 changes: 86 additions & 0 deletions tst/unit/env/gateway/triggers_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
from __future__ import annotations

import asyncio
import contextlib
import json
import random
import re
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo

Expand Down Expand Up @@ -1111,6 +1113,90 @@ def do_POST(self):
server.shutdown()


@contextlib.asynccontextmanager
async def _executor_answering_after(seconds: float):
"""A loopback executor whose message/send, like a deployed one's, answers only once its task ends `seconds` later;
its task polls answer at once. Yields its URL."""
from pytest_socket import enable_socket

enable_socket() # re-disabled by the autouse fixture's next setup
handlers: set[asyncio.Task] = set()

async def serve(reader, writer):
handlers.add(asyncio.current_task())
try:
while True: # one connection carries the send and then the polls
head = await reader.readuntil(b"\r\n\r\n")
req = json.loads(await reader.readexactly(int(re.search(rb"(?i)content-length: *(\d+)", head)[1])))
if req["method"] == "message/send":
await asyncio.sleep(seconds)
body = json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": {"id": "task-1", "status": {
"state": "completed", "message": {"parts": [{"kind": "text", "text": "replied"}]}}}}).encode()
writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s"
% (len(body), body))
await writer.drain()
except (asyncio.IncompleteReadError, ConnectionError):
pass # the caller hung up
finally:
writer.close()

server = await asyncio.start_server(serve, "127.0.0.1", 0)
try:
yield f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}"
finally:
server.close()
for task in handlers:
task.cancel()
await asyncio.gather(*handlers, return_exceptions=True)


def _reply_then_post(url: str, timeout_seconds: float) -> dict:
return {"executor": {"a2a_url": url, "role": "executor", "timeout_seconds": timeout_seconds},
"triggers": [{"id": "x", "when": {"type": "action", "tool": "t"},
"actions": [{"type": "nl", "instruction": "reply to the client"},
{"type": "tool", "tool": "slack_send_message",
"args": {"channel": "#c", "text": "the client replied"}}]}]}


async def _settled(engine, tid: str, within: float = 10) -> None:
for _ in range(int(within / 0.05)):
if engine._triggers[tid]["status"] not in ("firing", "queued"):
return
await asyncio.sleep(0.05)
raise AssertionError(f"trigger {tid!r} still {engine._triggers[tid]['status']} after {within}s")


@pytest.mark.asyncio
async def test_an_executor_slower_than_the_http_timeout_is_waited_out_and_the_later_actions_run(engine, monkeypatch):
"""FD-5333: a contact slower than 30 s used to fail its fire on a read timeout, though its reply still landed."""
monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) # stands in for the 30 s
async with _executor_answering_after(1.0) as url:
engine.register(_reply_then_post(url, timeout_seconds=10))
engine.on_tool_call("default", "t", {}, _result())
await asyncio.sleep(0.4)
assert engine._triggers["x"]["status"] == "firing" # past the HTTP timeout, still in flight
await _settled(engine, "x")

assert engine._triggers["x"]["status"] == "fired", engine.state()["events"][-3:]
assert [e["action_index"] for e in engine._events if e["kind"] == "action_ok"] == [0, 1]
assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})]


@pytest.mark.asyncio
async def test_an_executor_slower_than_its_own_timeout_still_fails_the_fire(engine, monkeypatch):
monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2)
async with _executor_answering_after(3.0) as url:
engine.register(_reply_then_post(url, timeout_seconds=1.0))
engine.on_tool_call("default", "t", {}, _result())
await asyncio.sleep(0.4)
assert engine._triggers["x"]["status"] == "firing"
await _settled(engine, "x", within=2.5)

[failure] = [e for e in engine._events if e["kind"] == "action_failed"]
assert (failure["action_index"], failure["detail"]) == (0, "ReadTimeout")
assert engine._triggers["x"]["status"] == "armed" and engine._test_gw.calls == []


@pytest.mark.asyncio
async def test_broken_check_is_fail_open(engine):
engine.register({"triggers": [
Expand Down
Loading