From f6cde72cbd1de026064354906b3fa842684c71bf Mon Sep 17 00:00:00 2001 From: Angelo Mantellini Date: Wed, 11 Nov 2026 07:12:57 +0100 Subject: [PATCH 1/2] feat(rt-norm-worker): add realtime normalization Signed-off-by: Angelo Mantellini --- backend/norm/docs/events.md | 236 +++++ backend/norm/src/norm/__init__.py | 10 +- backend/norm/src/norm/ioa_observe/build.py | 55 +- backend/norm/src/norm/ioa_observe/events.py | 78 ++ backend/norm/src/norm/ioa_observe/fields.py | 61 +- .../src/norm/ioa_observe/handlers/__init__.py | 32 + .../norm/src/norm/ioa_observe/handlers/a2a.py | 166 +++ .../src/norm/ioa_observe/handlers/agent.py | 142 ++- .../src/norm/ioa_observe/handlers/chat.py | 24 +- .../src/norm/ioa_observe/handlers/graph.py | 4 +- .../norm/src/norm/ioa_observe/handlers/llm.py | 115 +++ .../norm/src/norm/ioa_observe/handlers/mcp.py | 179 ++++ .../src/norm/ioa_observe/handlers/session.py | 24 +- .../src/norm/ioa_observe/handlers/tool.py | 24 +- .../norm/ioa_observe/handlers/tool_events.py | 143 +++ .../src/norm/ioa_observe/handlers/topology.py | 249 +++++ .../norm/ioa_observe/heuristics/handoff.py | 6 +- backend/norm/src/norm/ioa_observe/otel_io.py | 6 +- backend/norm/src/norm/ioa_observe/registry.py | 8 + backend/norm/src/norm/ioa_observe/stream.py | 349 +++++++ .../norm/src/norm/ioa_observe/trajectory.py | 73 +- backend/norm/src/norm/normalizer.py | 22 +- backend/norm/tests/test_normalizer.py | 949 +++++++++++++++++- backend/norm/tests/test_otel_io.py | 19 +- .../norm/tests/test_pipeline_functional.py | 4 +- backend/pyproject.toml | 2 + backend/uv.lock | 105 +- backend/workers/rt-norm-worker/.gitignore | 138 +++ backend/workers/rt-norm-worker/README.md | 189 ++++ backend/workers/rt-norm-worker/pyproject.toml | 35 + .../workers/rt-norm-worker/rules.example.yaml | 28 + backend/workers/rt-norm-worker/rules.yaml | 24 + .../src/rt_norm_worker/__init__.py | 2 + .../src/rt_norm_worker/callbacks.py | 75 ++ .../rt-norm-worker/src/rt_norm_worker/cli.py | 76 ++ .../rt-norm-worker/src/rt_norm_worker/otlp.py | 117 +++ .../src/rt_norm_worker/rules.py | 170 ++++ .../src/rt_norm_worker/server.py | 134 +++ .../src/rt_norm_worker/store.py | 141 +++ .../src/rt_norm_worker/worker.py | 72 ++ .../rt-norm-worker/tests/test_rules.py | 141 +++ .../rt-norm-worker/tests/test_server.py | 119 +++ .../rt-norm-worker/tests/test_worker.py | 63 ++ 43 files changed, 4443 insertions(+), 166 deletions(-) create mode 100644 backend/norm/docs/events.md create mode 100644 backend/norm/src/norm/ioa_observe/events.py create mode 100644 backend/norm/src/norm/ioa_observe/handlers/a2a.py create mode 100644 backend/norm/src/norm/ioa_observe/handlers/llm.py create mode 100644 backend/norm/src/norm/ioa_observe/handlers/mcp.py create mode 100644 backend/norm/src/norm/ioa_observe/handlers/tool_events.py create mode 100644 backend/norm/src/norm/ioa_observe/handlers/topology.py create mode 100644 backend/norm/src/norm/ioa_observe/stream.py create mode 100644 backend/workers/rt-norm-worker/.gitignore create mode 100644 backend/workers/rt-norm-worker/README.md create mode 100644 backend/workers/rt-norm-worker/pyproject.toml create mode 100644 backend/workers/rt-norm-worker/rules.example.yaml create mode 100644 backend/workers/rt-norm-worker/rules.yaml create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/__init__.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/callbacks.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/cli.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/otlp.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/rules.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/server.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/store.py create mode 100644 backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py create mode 100644 backend/workers/rt-norm-worker/tests/test_rules.py create mode 100644 backend/workers/rt-norm-worker/tests/test_server.py create mode 100644 backend/workers/rt-norm-worker/tests/test_worker.py diff --git a/backend/norm/docs/events.md b/backend/norm/docs/events.md new file mode 100644 index 0000000..c3cef5a --- /dev/null +++ b/backend/norm/docs/events.md @@ -0,0 +1,236 @@ +# Event normalization + +How `norm` turns OTel **log records** (events) into knowledge-graph nodes and edges. It covers +the streaming path, `StreamNormalizer.process(record, kind="event")`, used by the +[rt-norm-worker](../../workers/rt-norm-worker/README.md). Spans take a different path +(`dispatch_span`, see [normalization.md](normalization.md)). + +Code: [`events.py`](../src/norm/ioa_observe/events.py) routes on `LogAttributes["event.name"]`; +the handlers are in [`handlers/`](../src/norm/ioa_observe/handlers/). + +## How an event is processed + +1. A dry run on an empty registry finds which node ids (and agent names) the event touches. +2. The reader loads them from the graph store, together with the event's session subgraph. +3. The handler runs against the loaded graph. +4. Only the difference is returned as a `StreamDelta` and written back. + +Nothing is kept between events: the stored graph is the state. Consequences: + +- Handlers are **idempotent**. Replaying an event changes nothing. +- Handlers are **order-tolerant**. Most of them create whatever is missing above the node they + need (MAS, Session, MASCall, Agent, AgentCall), so a `completed` event arriving alone builds + what its `started` would have. +- A node that already exists is **reused, not overwritten**. `startTime` never changes; + `endTime` only moves forward. Initial and final States are set once, by whichever event + arrives first. +- An event with no `session.id` (or no agent / tool name where one is needed) is ignored. + +## Identity + +| Node | Id | +|---|---| +| `Session` | `session.id` without its `_` prefix | +| `MAS` | `application_id` (falls back to `ServiceName`) | +| `MASCall` | `mascall:` | +| `Agent` | the agent's **name** (`agent.name`, `source.agent`, `target.agent`, `tool.name` of an a2a tool, the card's `name`). The same name in another MAS is another Agent, with id `:`. The span id is the AgentCall's, not the Agent's. | +| `AgentCall` | hash of the Agent's id and the span id (`topology.node.*` `span.id`, or `ioa_observe.agent.span_id` of the other events) | +| `LLM` | `llm--` (from `llm.name`, `/`) | +| `LLMCall` | hash of provider, model and `llm.call.id` (or the event's span) | +| `Tool` | the tool name | +| `ToolCall` | `tool.call.id` (= the `message.id` of the MCP messages) when there is one, else a hash of tool name and span | +| `State`, `Transition` | hashes of the owner's span and the content | + +Events that carry only an `ioa_observe.agent.span_id` (`tool.*`, `mcp.tools.discovered`) find the +Agent through the AgentCall of that span. + +## States + +Every execution (`Session`, `AgentCall`, `LLMCall`, `ToolCall`) gets an initial `State` +(`hasInitialState` + `hasState`) when it starts and a final `State` plus a `Transition` +(`hasFinalState`, `inputTo`, `leadsTo`, `correspondsToTransition`, `representsExecution`) when it +completes. The content of a State is the event's input or output text, or +`(no content recorded)` when there is none. + +## Events + +### `topology.session.started` + +Reads `session.id`, `application_id`, `event.time`. + +- Creates `Session` and `MAS`, linked by `executesSession`. +- Creates the Session's initial State. + +### `topology.session.completed` + +Same nodes as `session.started`. + +- Moves the Session's `endTime` forward and sets `duration`. +- Creates the initial State if there is none yet. +- Adds a final State and Transition for **each** completion. Only the **latest** one, by + `event.time` and not by arrival, holds `hasFinalState`; a later event moves it, an earlier or + repeated one leaves it where it is. The other completions keep `hasState` only. + +### `topology.node.started` + +Reads `agent.name`, `agent.input`, plus the common ids. The event's own `span.id` is the AgentCall's. + +- Creates `Agent` (keyed on its name; one created earlier by a card or a handoff is reused) and + `belongsToMAS` to the application. +- Creates `AgentCall` with `executesAgent`, and `MASCall` with `hasAgentCall` (plus Session and + MAS if missing). +- Attaches ToolCalls that arrived before it (`hasToolCall`, `usesTool`). +- Creates the AgentCall's initial State from `agent.input`. + +### `topology.node.completed` + +Same nodes as `node.started`. + +- Moves the AgentCall's `endTime` forward and sets `duration`. +- Creates the initial State if there is none (from `agent.input`). +- Creates the final State from `agent.output` and the Transition, once. + +### `topology.edge.updated` + +Reads `source.agent.span.id`, `target.agent`. Creates nothing. + +- Adds `executesAgent` from the **AgentCall** of the source span to the **Agent** named + `target.agent`. +- Ignored when either does not exist: a target that is not an agent (an MCP server such as + `debug`) matches nothing. + +`executesAgent` otherwise means "this call is an instance of that agent"; here it also records +the handoff. A call's own agent is therefore the one it was created for, not any `executesAgent` +target (`_own_agent_id`). + +### `llm.started` + +Reads `llm.name`, `llm.input`, `llm.call.id`, `agent.name`, `ioa_observe.agent.span_id`. + +- Ensures the chain down to the LLM call: MAS, Session, MASCall, Agent (`agent.name`), AgentCall (of + `ioa_observe.agent.span_id`), `LLM`, `LLMCall`. +- Adds `usesLLM` (Agent to LLM), `executesLLM`, `hasLLMCall` (AgentCall to LLMCall). +- Creates the LLMCall's initial State from `llm.input`. + +### `llm.completed` + +Same nodes as `llm.started`. + +- Moves the LLMCall's `endTime` forward and sets `duration`. +- Creates the final State from `llm.output` and the Transition, once. + +### `tool.started` + +Reads `tool.name`, `tool.input`, `tool.type`, `tool.call.id`, `ioa_observe.agent.span_id`. + +For a regular tool: + +- Creates `Tool` and `ToolCall` with `executesTool`, plus MAS, Session and MASCall. +- If the owning AgentCall (found by `ioa_observe.agent.span_id`) exists: `hasToolCall` and + `usesTool` from its Agent. An AgentCall is not created here; `topology.node.*` links it later. +- Creates the ToolCall's initial State from `tool.input`. + +For `tool.type = a2a` (a call to another agent): no Tool or ToolCall. Only makes sure an Agent +named `tool.name` exists, unattached if it has to create it. + +### `tool.completed` + +Same nodes as `tool.started`. + +- Moves the ToolCall's `endTime` forward and sets `duration`. +- Sets `success` from `tool.status`. +- Creates the final State from `tool.output` and the Transition, once. +- An `a2a` tool does nothing. + +### `a2a.agent_card` + +Reads `session.id`, `capability.content` (JSON): `name`, `description`. + +- Ignored without a `session.id`. +- Creates `MAS` (the first part of `session.id`, `_`) and `Session` (the + rest), linked by `executesSession`. +- Adds the `Agent` named by the card, keyed on its **name** (the card has no span id of the + agent it describes), with `belongsToMAS`. An Agent of that name already in the MAS is reused + and only gets its description. The ontology has no + Agent-to-Session edge, so the Session is not linked to the Agent. +- Ignores the card's own `ioa_observe.agent.span_id`: it is the span of the agent that fetched + the card, not of the one it describes. +- The rest of the card (version, interfaces, skills, ...) is not stored yet. + +Every other event that needs the Agent (`topology.node.*`, `llm.*`, `a2a.message.*`, +`mcp.message.*`) uses the same node, since it is keyed on the same name. Only if an Agent of that +name exists in a different MAS does it create a separate one (`:`). + +### `a2a.message.sent` + +Reads `source.agent`, `source.agent.span.id`, `target.agent`, `message.id`, `message.input`. + +- Ensures the sender's chain: MAS, Session, MASCall, Agent, AgentCall. +- Ensures an Agent named `target.agent` (unattached if new). +- Keeps `message.input` as a State of the sender's AgentCall (`hasState` only, so its initial + and final States are untouched). + +### `a2a.message.received` + +Same nodes as `a2a.message.sent`. + +- Keeps `message.input` as a State of the **recipient's** latest AgentCall, only if that call + already exists. The event does not carry the recipient's span id, and its call is normally + created a few milliseconds later by its own `topology.node.started`, so usually nothing of + the message is kept. + +### `mcp.message.sent` / `mcp.message.received` + +Reads `message.input` (JSON), `message.id`, `source.agent`, `source.agent.span.id`. + +- Ignored when `message.input` is empty or has no `name` (a `tools/list` request). +- Ensures the sender's chain: MAS (from `application_id`, else the first part of `session.id`), + Session, MASCall, Agent, AgentCall. +- Creates `Tool` (`message.input.name`) and `ToolCall` keyed on `message.id`, with + `executesTool`, `hasToolCall` and `usesTool`. +- Creates the ToolCall's initial State from `message.input`. + +`sent` and `received` of one message, and the `tool.*` events of the same call, share the +`message.id` / `tool.call.id`, so they land on one ToolCall. + +### `mcp.message.completed` + +Same nodes as `mcp.message.sent`. + +- Moves the ToolCall's `endTime` forward and sets `duration`. +- Sets `success` to `not isError` when `message.output` says so. +- Creates the final State from `message.output` and the Transition, once. + +### `mcp.tools.discovered` + +Reads `ioa_observe.agent.span_id`, `session.id`, `capability.content` (JSON with a `tools` list). + +- Ignored without `ioa_observe.agent.span_id` or a session. +- Creates `MAS` (named by the first part of `session.id`, `_`) and `Session`. +- Creates a `Tool` for every entry in `tools` with its `description`, and fills the description + of a Tool that already exists without one. +- The event names the agent only by span id, so the agent is the one of the AgentCall of that + span, and every tool is linked to it with `usesTool`. If that call does not exist yet (the + event arrives before the agent's `topology.node.started`), the tools are created **without** + the link: nothing is remembered to link them later. + +## Not handled + +These are ignored; `dispatch_event` returns `False` for names it does not know. + +| Event | Status | +|---|---| +| `slim.message.sent`, `slim.message.received` | `TODO` | +| `mcp.tools.published` | `TODO` | +| `a2a.message.completed` | no case | +| `llm.tools.presented` | no case | + +## Known gaps + +- The handoff between agents is only the `executesAgent` edge from `topology.edge.updated`. The + ontology has no Agent-to-Agent relationship. +- `tool.*` events without `tool.call.id` and without `ioa_observe.agent.span_id` create a + ToolCall that nothing links to an agent. +- When an agent runs twice it has one `Agent` and two `AgentCall` nodes. The same agent name in + another MAS is a different Agent. +- `mcp.tools.discovered` arriving before the agent's `topology.node.started` does not link its tools. diff --git a/backend/norm/src/norm/__init__.py b/backend/norm/src/norm/__init__.py index 5e407a0..3192edf 100644 --- a/backend/norm/src/norm/__init__.py +++ b/backend/norm/src/norm/__init__.py @@ -2,8 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 from .compare import KGCompareResult, compare_kg, read_kg_json, write_kg_json from .ioa_observe import build_kg +from .ioa_observe.stream import GraphReader, InMemoryGraph, Kind, StreamDelta, StreamNormalizer from .ioa_observe.otel_io import infer_run_id, load_otel_export -from .normalizer import Normalizer, dump_jsonld, normalize +from .normalizer import Normalizer, OnNormalizedCallback, configure, dump_jsonld, normalize from .verifier import ( KGCheckSkipped, check_edge_domain_range, @@ -20,12 +21,19 @@ "KGCheckSkipped", "KGCompareResult", "Normalizer", + "OnNormalizedCallback", + "GraphReader", + "InMemoryGraph", + "Kind", + "StreamDelta", + "StreamNormalizer", "build_kg", "check_edge_domain_range", "check_orphaned_edges", "check_unknown_edge_types", "check_unknown_node_types", "compare_kg", + "configure", "dump_jsonld", "infer_run_id", "load_otel_export", diff --git a/backend/norm/src/norm/ioa_observe/build.py b/backend/norm/src/norm/ioa_observe/build.py index be6ca5f..3038ff7 100644 --- a/backend/norm/src/norm/ioa_observe/build.py +++ b/backend/norm/src/norm/ioa_observe/build.py @@ -14,7 +14,7 @@ from typing import Any -from .fields import attrs +from .fields import get_span_attrs from .handlers import ( handle_agent, handle_chat, @@ -37,29 +37,26 @@ } -def build_kg( - spans: list[dict[str, Any]], -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Build (nodes, edges) JSON dicts from raw ClickHouse-shaped OTel spans.""" - registry = Registry() +def dispatch_span(span: dict[str, Any], registry: Registry) -> None: + """Run the one per-span handler matching ``span``'s ``SpanName`` (if any).""" + span_name = str(span.get("SpanName") or "") + span_attrs = get_span_attrs(span) - for span in spans: - span_name = str(span.get("SpanName") or "") - span_attrs = attrs(span) + handler = _HANDLERS.get(span_name) + if handler is not None: + handler(span, span_attrs, registry) + elif span_name.endswith(".graph"): + handle_graph(span, span_attrs, registry) + elif span_name.endswith(".agent"): + handle_agent(span, span_attrs, registry) + elif span_name.endswith(".chat"): + handle_chat(span, span_attrs, registry) + elif span_name.endswith(".tool"): + handle_tool(span, span_attrs, registry) - handler = _HANDLERS.get(span_name) - if handler is not None: - handler(span, span_attrs, registry) - elif span_name.endswith(".graph"): - handle_graph(span, span_attrs, registry) - elif span_name.endswith(".agent"): - handle_agent(span, span_attrs, registry) - elif span_name.endswith(".chat"): - handle_chat(span, span_attrs, registry) - elif span_name.endswith(".tool"): - handle_tool(span, span_attrs, registry) - # Clean up trajectory once all spans are processed +def apply_heuristics(spans: list[dict[str, Any]], registry: Registry) -> None: + """Cross-span clean-up of the trajectory, once ``spans`` are all handled.""" # Agent handoffs explicit_handoff_targets = link_agent_handoffs(spans, registry) # Fallback: timestamp-order sibling AgentCalls the above left unlinked @@ -69,6 +66,11 @@ def build_kg( # Session and MAS call i/o assign_container_boundary_states(registry) + +def serialize_registry( + registry: Registry, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Registry -> (nodes, edges) JSON dicts.""" nodes: list[dict[str, Any]] = [] edges: list[dict[str, Any]] = [] for item in registry.all_items(): @@ -86,3 +88,14 @@ def build_kg( payload[attr] = value nodes.append(payload) return nodes, edges + + +def build_kg( + spans: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Build (nodes, edges) JSON dicts from raw ClickHouse-shaped OTel spans.""" + registry = Registry() + for span in spans: + dispatch_span(span, registry) + apply_heuristics(spans, registry) + return serialize_registry(registry) diff --git a/backend/norm/src/norm/ioa_observe/events.py b/backend/norm/src/norm/ioa_observe/events.py new file mode 100644 index 0000000..08c7c1e --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/events.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Event (OTel log record) dispatch, keyed on the ``event.name`` attribute.""" + +from __future__ import annotations + +from typing import Any + +from .handlers import ( + handle_a2a_agent_card, + handle_a2a_message_received, + handle_a2a_message_sent, + handle_llm_completed, + handle_llm_started, + handle_mcp_message_completed, + handle_mcp_message_received, + handle_mcp_message_sent, + handle_mcp_tools_discovered, + handle_tool_completed, + handle_tool_started, + handle_topology_edge_updated, + handle_topology_node_completed, + handle_topology_node_started, + handle_topology_session_completed, + handle_topology_session_started, +) +from .registry import Registry + + +def dispatch_event(event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry) -> bool: + """Run the case for the event's ``event.name``. False if the name is unknown.""" + match str(event_attrs.get("event.name") or ""): + case "topology.session.started": + handle_topology_session_started(event, event_attrs, registry) + case "topology.session.completed": + handle_topology_session_completed(event, event_attrs, registry) + case "topology.node.started": + handle_topology_node_started(event, event_attrs, registry) + case "topology.node.completed": + handle_topology_node_completed(event, event_attrs, registry) + case "topology.edge.updated": + # TODO: handle topology.edge.updated + #handle_topology_edge_updated(event, event_attrs, registry) + pass + case "llm.started": + handle_llm_started(event, event_attrs, registry) + case "llm.completed": + handle_llm_completed(event, event_attrs, registry) + case "tool.started": + handle_tool_started(event, event_attrs, registry) + case "tool.completed": + handle_tool_completed(event, event_attrs, registry) + case "a2a.message.sent": + handle_a2a_message_sent(event, event_attrs, registry) + case "a2a.message.received": + handle_a2a_message_received(event, event_attrs, registry) + case "a2a.agent_card": + handle_a2a_agent_card(event, event_attrs, registry) + case "slim.message.sent": + # TODO: handle slim.message.sent + pass + case "slim.message.received": + # TODO: handle slim.message.received + pass + case "mcp.message.sent": + handle_mcp_message_sent(event, event_attrs, registry) + case "mcp.message.received": + handle_mcp_message_received(event, event_attrs, registry) + case "mcp.message.completed": + handle_mcp_message_completed(event, event_attrs, registry) + case "mcp.tools.discovered": + handle_mcp_tools_discovered(event, event_attrs, registry) + case "mcp.tools.published": + # TODO: handle mcp.tools.published + pass + case _: + return False + return True diff --git a/backend/norm/src/norm/ioa_observe/fields.py b/backend/norm/src/norm/ioa_observe/fields.py index e2abfcb..12b32c8 100644 --- a/backend/norm/src/norm/ioa_observe/fields.py +++ b/backend/norm/src/norm/ioa_observe/fields.py @@ -8,35 +8,36 @@ from datetime import datetime, timezone from typing import Any +# Span field derivation functions -def attrs(span: dict[str, Any]) -> dict[str, Any]: +def get_span_attrs(span: dict[str, Any]) -> dict[str, Any]: """Return a span's SpanAttributes map, or {} if absent.""" return span.get("SpanAttributes") or {} -def get_application_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: +def get_span_application_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: return str(span_attrs.get("application_id") or span.get("ServiceName") or "") -def get_session_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: +def get_span_session_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: """Strip the ``_`` prefix ioa-observe puts on session.id. Strips the exact application_id prefix rather than splitting on the first "_", since application_id itself may contain one. """ raw = str(span.get("SessionId") or span_attrs.get("session.id") or "") - application_id = get_application_id(span, span_attrs) + application_id = get_span_application_id(span, span_attrs) prefix = f"{application_id}_" if application_id and raw.startswith(prefix): return raw[len(prefix) :] return raw.split("_")[-1] if "_" in raw else raw -def get_agent_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: +def get_span_agent_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: return str(span.get("AgentId") or span_attrs.get("agent_id") or "") -def get_success(span: dict[str, Any], span_attrs: dict[str, Any]) -> bool: +def get_span_success(span: dict[str, Any], span_attrs: dict[str, Any]) -> bool: """Read the span's own ``execution.success`` attribute (string-valued, e.g. "true"/"false"), falling back to the OTel ``StatusCode`` column when that attribute is absent: only an explicit "Error" status counts @@ -54,7 +55,7 @@ def _parse_timestamp_ns(value: str) -> int: return int(dt.timestamp()) * 1_000_000_000 + int(fractional or 0) -def get_start_time(span: dict[str, Any], span_attrs: dict[str, Any]) -> float: +def get_span_start_time(span: dict[str, Any], span_attrs: dict[str, Any]) -> float: """Prefer the SDK's own ``ioa_start_time`` capture; fall back to the collector-recorded ``Timestamp`` column if it's absent.""" raw = span_attrs.get("ioa_start_time") @@ -69,16 +70,16 @@ def get_start_time(span: dict[str, Any], span_attrs: dict[str, Any]) -> float: return _parse_timestamp_ns(timestamp) / 1e9 -def get_duration_seconds(span: dict[str, Any]) -> float: +def get_span_duration_seconds(span: dict[str, Any]) -> float: return float(span.get("Duration") or 0) / 1e9 -def get_duration_ms(span: dict[str, Any]) -> float: +def get_span_duration_ms(span: dict[str, Any]) -> float: return float(span.get("Duration") or 0) / 1e6 -def get_end_time(start_time: float, span: dict[str, Any]) -> float: - return start_time + get_duration_seconds(span) +def get_span_end_time(start_time: float, span: dict[str, Any]) -> float: + return start_time + get_span_duration_seconds(span) def _to_float(value: Any) -> float: @@ -107,3 +108,41 @@ def _hash_id(prefix: str, *parts: str) -> str: same id, so edges can reference a node before or after it's built.""" digest = hashlib.sha1("\x1f".join(str(p) for p in parts).encode("utf-8")).hexdigest()[:12] return f"{prefix}:{digest}" + +# Event field derivation functions + +def get_event_attrs(event: dict[str, Any]) -> dict[str, Any]: + """Return an event's (OTel log record's) LogAttributes map, or {} if absent.""" + return event.get("LogAttributes") or {} + +def get_event_application_id(event: dict[str, Any], event_attrs: dict[str, Any]) -> str: + return str(event_attrs.get("application_id") or event.get("ServiceName") or "") + + +def get_event_session_id(event: dict[str, Any], event_attrs: dict[str, Any]) -> str: + """Strip the ``_`` prefix from an event's ``session.id`` (in LogAttributes). + + Same rule as ``get_span_session_id``: strip the exact application_id prefix rather + than splitting on the first "_", since application_id itself may contain one. + """ + raw = str(event_attrs.get("session.id") or "") + application_id = get_event_application_id(event, event_attrs) + prefix = f"{application_id}_" + if application_id and raw.startswith(prefix): + return raw[len(prefix) :] + return raw.split("_")[-1] if "_" in raw else raw + + +def get_event_start_time(event: dict[str, Any], event_attrs: dict[str, Any]) -> float: + """``event.time`` (ISO 8601) in epoch seconds; falls back to the ``Timestamp`` column.""" + raw = str(event_attrs.get("event.time") or "") + if raw: + try: + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.timestamp() + except ValueError: + pass + timestamp = str(event.get("Timestamp") or "") + return _parse_timestamp_ns(timestamp) / 1e9 if timestamp else 0.0 diff --git a/backend/norm/src/norm/ioa_observe/handlers/__init__.py b/backend/norm/src/norm/ioa_observe/handlers/__init__.py index e4f42e5..e7185c9 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/__init__.py +++ b/backend/norm/src/norm/ioa_observe/handlers/__init__.py @@ -2,17 +2,49 @@ # SPDX-License-Identifier: Apache-2.0 """One handler module per ``SpanName`` shape -- see build.py for dispatch.""" +from .a2a import handle_a2a_agent_card, handle_a2a_message_received, handle_a2a_message_sent from .agent import handle_agent from .chat import handle_chat from .graph import handle_graph +from .mcp import ( + handle_mcp_message_completed, + handle_mcp_message_received, + handle_mcp_message_sent, + handle_mcp_tools_discovered, +) +from .llm import handle_llm_completed, handle_llm_started from .session import handle_session_end, handle_session_start from .tool import handle_tool +from .tool_events import handle_tool_completed, handle_tool_started +from .topology import ( + handle_topology_edge_updated, + handle_topology_node_completed, + handle_topology_node_started, + handle_topology_session_completed, + handle_topology_session_started, +) __all__ = [ + "handle_a2a_agent_card", + "handle_a2a_message_received", + "handle_a2a_message_sent", "handle_agent", "handle_chat", "handle_graph", + "handle_llm_completed", + "handle_llm_started", + "handle_mcp_message_completed", + "handle_mcp_message_received", + "handle_mcp_message_sent", + "handle_mcp_tools_discovered", "handle_session_end", "handle_session_start", "handle_tool", + "handle_tool_completed", + "handle_tool_started", + "handle_topology_edge_updated", + "handle_topology_node_completed", + "handle_topology_node_started", + "handle_topology_session_completed", + "handle_topology_session_started", ] diff --git a/backend/norm/src/norm/ioa_observe/handlers/a2a.py b/backend/norm/src/norm/ioa_observe/handlers/a2a.py new file mode 100644 index 0000000..2a2519c --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/handlers/a2a.py @@ -0,0 +1,166 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Handlers for the "a2a.*" events (OTel log records).""" + +from __future__ import annotations + +import json +from typing import Any + +from scope_ontology.models.edges import belongsToMAS, hasState +from scope_ontology.models.nodes import Agent, AgentCall, State + +from ..fields import _hash_id +from ..registry import Registry +from .agent import _agent_id_for, _ensure_agent, _latest_call_of +from .topology import _ensure_agent_call, _ensure_session_and_mas + + +def _ensure_message_parties( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> tuple[AgentCall, str, str, str, str, str] | None: + """What ``a2a.message.sent`` and ``a2a.message.received`` have in common: the sender's + AgentCall (found by ``source.agent.span.id``) with everything above it -- MAS, Session, + MASCall, Agent -- and the recipient Agent (``target.agent``), each created if missing. + + Returns ``(sender_call, session_id, source_span_id, target_name, message_id, content)``, + or None if the event names no sender, recipient, sender span or session.""" + source_name = str(event_attrs.get("source.agent") or "") + target_name = str(event_attrs.get("target.agent") or "") + source_span_id = str( + event_attrs.get("source.agent.span.id") + or event_attrs.get("ioa_observe.agent.span_id") + or "" + ) + if not source_name or not target_name or not source_span_id: + return None + ensured = _ensure_agent_call( + event, {**event_attrs, "agent.name": source_name}, registry, span_id=source_span_id + ) + if ensured is None: + return None + agent_call, session_id, _, _ = ensured + if registry.find(Agent, lambda agent: agent.name == target_name) is None: + registry.add(Agent(id=target_name, name=target_name, description="")) + message_id = str(event_attrs.get("message.id") or event_attrs.get("event.id") or "") + content = str(event_attrs.get("message.input") or "") + return agent_call, session_id, source_span_id, target_name, message_id, content + + +def handle_a2a_message_sent( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the sender's call and recipient Agent, and retain the outgoing message + as a State of the sender without changing its initial or final State.""" + parties = _ensure_message_parties(event, event_attrs, registry) + if parties is None: + return + agent_call, session_id, source_span_id, target_name, message_id, content = parties + if not content: + return + state = registry.add( + State( + id=_hash_id( + "state", "a2a.message.sent", session_id, source_span_id, + target_name, message_id, content, + ), + sessionId=session_id, + content=content, + ) + ) + registry.add_edge(hasState(source_id=agent_call.id, target_id=state.id)) + + +def handle_a2a_message_received( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the same nodes as ``a2a.message.sent``, and retain the incoming message as a + State of the recipient's AgentCall without changing its initial or final State. + + The event names the recipient only by ``target.agent`` -- its span is not in it, and + its AgentCall is normally created a few milliseconds later by its own + ``topology.node.started`` -- so the State is added only if that call is already there; + otherwise nothing of the message is kept.""" + parties = _ensure_message_parties(event, event_attrs, registry) + if parties is None: + return + _, session_id, source_span_id, target_name, message_id, content = parties + receiver_call = _latest_call_of(registry, target_name, session_id) + if receiver_call is None or not content: + return + state = registry.add( + State( + id=_hash_id( + "state", "a2a.message.received", session_id, source_span_id, + target_name, message_id, content, + ), + sessionId=session_id, + content=content, + ) + ) + registry.add_edge(hasState(source_id=receiver_call.id, target_id=state.id)) + + +def _agent_card(event_attrs: dict[str, Any]) -> dict[str, Any]: + """The agent card carried in ``capability.content`` (a JSON object), or {} if absent or malformed.""" + content = event_attrs.get("capability.content") + if isinstance(content, str): + try: + content = json.loads(content) + except ValueError: + return {} + return content if isinstance(content, dict) else {} + + +def handle_a2a_agent_card( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Add the Agent the card describes -- ``capability.content``'s ``name`` and + ``description`` -- to the MAS and Session named by ``session.id``. + + Skipped without a ``session.id``. The MAS is the first part of ``session.id`` + (``_``) and the Session the rest; both are created if missing. The + card has no span id of the described agent (its own ``ioa_observe.agent.span_id`` is the + span of the agent that fetched it). An Agent is keyed on its name, so the card's Agent is + the one the agent's own events later reuse. It is attached to the MAS with ``belongsToMAS``; + one of that name already there is reused and only gets its description.""" + card = _agent_card(event_attrs) + name = str(card.get("name") or "") + mas_name = str(event_attrs.get("session.id") or "").partition("_")[0] + if not name or not mas_name: + return + if _ensure_session_and_mas(event, {**event_attrs, "application_id": mas_name}, registry) is None: + return + description = str(card.get("description") or "") + agent = _ensure_agent(registry, _agent_id_for(registry, name, (mas_name,)), name, (mas_name,)) + if description and agent.description != description: + agent.description = description # type: ignore[attr-defined] + registry.add_edge(belongsToMAS(source_id=agent.id, target_id=mas_name)) + # TODO: version (card["version"]) + # TODO: supportedInterfaces (how to reach the agent: url, protocolBinding, protocolVersion) + # TODO: provider (card["provider"], capability.provider.id) + # TODO: capabilities (card["capabilities"]: streaming, pushNotifications, ...) + # TODO: defaultInputModes / defaultOutputModes + # TODO: skills (card["skills"]) + # TODO: securitySchemes / security + + +def handle_a2a_tool_started( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """A ``tool.started`` of type ``a2a``: the agent whose span is + ``ioa_observe.agent.span_id`` hands off to the agent named ``tool.name``. + + The target Agent is made sure to exist: an Agent of that name, or else an unattached + one keyed on the name, which the target's own events later reuse (an Agent is keyed on + its name).""" + source_span_id = str(event_attrs.get("ioa_observe.agent.span_id") or "") + target_name = str(event_attrs.get("tool.name") or "") + if not source_span_id or not target_name: + return + if not any(agent.name == target_name for agent in registry.all_of(Agent)): + registry.add(Agent(id=target_name, name=target_name, description="")) + # TODO: record the handoff itself, from the Agent of the AgentCall of ``source_span_id`` to the + # target. The ontology has no Agent -> Agent relationship; a handoff is otherwise a + # bridge ProcessingCall between the two AgentCalls (heuristics/handoff.py), and the + # target's AgentCall does not exist yet when this event arrives. diff --git a/backend/norm/src/norm/ioa_observe/handlers/agent.py b/backend/norm/src/norm/ioa_observe/handlers/agent.py index 9c20a97..a6e6b54 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/agent.py +++ b/backend/norm/src/norm/ioa_observe/handlers/agent.py @@ -18,18 +18,112 @@ from ..fields import ( _hash_id, - get_agent_id, - get_application_id, - get_duration_ms, - get_end_time, - get_session_id, - get_start_time, - get_success, + get_span_agent_id, + get_span_application_id, + get_span_duration_ms, + get_span_end_time, + get_span_session_id, + get_span_start_time, + get_span_success, ) from ..registry import Registry from ..trajectory import add_state_pair_transition +def _is_unattached(registry: Registry, node_id: str) -> bool: + return not any(node_id in (source, target) for _, source, target in registry.edges) + + +def _in_mas(registry: Registry, agent: Agent, mas_ids: tuple[str, ...]) -> bool: + """True if the Agent is attached to nothing or belongs to one of ``mas_ids``.""" + if _is_unattached(registry, agent.id): + return True + return any(m in registry.edges_from(belongsToMAS, agent.id) for m in mas_ids if m) or not any( + True for _ in registry.edges_from(belongsToMAS, agent.id) + ) + + +def _find_nameless_agent(registry: Registry, name: str, mas_ids: tuple[str, ...]) -> Agent | None: + """An Agent of exactly this ``name`` that has no span id of its own yet, i.e. one left by + an ``a2a.agent_card`` (or by a handoff target): keyed on its name and either attached to + one of ``mas_ids`` (what the card does) or attached to nothing.""" + for agent in registry.all_of(Agent): + if agent.name != name: + continue + if _is_unattached(registry, agent.id): + return agent + if agent.id == name and any( + mas_id in registry.edges_from(belongsToMAS, agent.id) for mas_id in mas_ids + ): + return agent + return None + + +def _ensure_agent( + registry: Registry, agent_id: str, name: str, mas_ids: tuple[str, ...] = () +) -> Agent: + """The Agent with this id; created if absent. + + Before creating one, an Agent of exactly the same name that is still keyed on its name + -- what an ``a2a.agent_card`` leaves behind, attached to the MAS (``mas_ids``) -- or an + unattached one, is adopted: it is replaced by the new node, which keeps its description.""" + agent = registry.get(Agent, agent_id) + if agent is not None: + if name and not agent.name: # created by an event that did not know the name + agent.name = name # type: ignore[attr-defined] + return agent + description = "" + orphan = _find_nameless_agent(registry, name, tuple(m for m in mas_ids if m)) if name else None + if orphan is not None and orphan.id != agent_id: + description = orphan.description + registry.remove_node(Agent, orphan.id) + registry.remove_edges_where(lambda _, source, target: orphan.id in (source, target)) + return registry.add(Agent(id=agent_id, name=name, description=description)) + + +def _latest_call_of( + registry: Registry, agent_name: str, session_id: str, not_after: float | None = None +) -> AgentCall | None: + """The most recently started AgentCall, in the session, of the Agent called ``agent_name`` + (agents are found by name: their ids are span ids), optionally one started no later than + ``not_after``.""" + latest: AgentCall | None = None + for edge_type, call_id, agent_id in registry.edges: + if edge_type is not executesAgent: + continue + agent = registry.get(Agent, agent_id) + call = registry.get(AgentCall, call_id) + if agent is None or call is None or agent.name != agent_name or call.sessionId != session_id: + continue + if not_after is not None and call.startTime > not_after: + continue + if latest is None or call.startTime > latest.startTime: + latest = call + return latest + + +def _own_agent_id(registry: Registry, call: AgentCall) -> str | None: + """The id of the Agent this AgentCall is an instance of. + + A call can have more than one ``executesAgent`` edge: ``topology.edge.updated`` adds + one to the agent it hands off to. The call's own Agent is the one it was created for + (``_ensure_agent_call`` names the call ``" call "``); failing that, the + first.""" + agent_ids = registry.edges_from(executesAgent, call.id) + own = call.name.removesuffix(f" call {call.spanId}") + return next((i for i in agent_ids if i == own), agent_ids[0] if agent_ids else None) + + +def _agent_id_for(registry: Registry, name: str, mas_ids: tuple[str, ...]) -> str: + """The id of the Agent called ``name`` in the MAS: the name itself, unless an Agent with + that id already belongs to another MAS, in which case it is ``:``.""" + existing = registry.get(Agent, name) + if existing is None or _in_mas(registry, existing, mas_ids): + return name + mas_id = next((m for m in mas_ids if m), "") + return f"{mas_id}:{name}" if mas_id else name + + def _ensure_session( registry: Registry, session_id: str, application_id: str, start_time: float ) -> Session: @@ -90,24 +184,34 @@ def handle_agent( span_attrs: dict[str, Any], registry: Registry, ) -> None: - agent_id = get_agent_id(span, span_attrs) + # Some "*.agent" spans carry no agent id: the agent is then known by its name, which is + # also the id of the Agent an a2a.agent_card creates. + entity_name = str(span_attrs.get("ioa_observe.entity.name") or "") + agent_id = get_span_agent_id(span, span_attrs) or entity_name if not agent_id: return - application_id = get_application_id(span, span_attrs) - session_id = get_session_id(span, span_attrs) + application_id = get_span_application_id(span, span_attrs) + session_id = get_span_session_id(span, span_attrs) span_id = str(span.get("SpanId") or "") - name = str(span_attrs.get("ioa_observe.entity.name") or agent_id) + name = entity_name or agent_id description = str(span_attrs.get("ioa_observe.entity.description") or "") - agent, created = registry.add_if_not_exists( - Agent(id=agent_id, name=name, description=description) - ) - if created and application_id: + # An Agent may already be there from its a2a.agent_card (keyed on its name, attached to + # the MAS of the card's session): reuse it only if it is the same name in the same MAS, + # fill in what it lacks, and connect it. The same name in another MAS is another Agent. + mas_ids = (application_id,) + existing = registry.get(Agent, agent_id) + if existing is not None and not _in_mas(registry, existing, mas_ids): + agent_id = f"{application_id}:{agent_id}" + agent = _ensure_agent(registry, agent_id, name, mas_ids) + if description and not agent.description: + agent.description = description # type: ignore[attr-defined] + if application_id: registry.add_edge(belongsToMAS(source_id=agent.id, target_id=application_id)) - start_time = get_start_time(span, span_attrs) - end_time = get_end_time(start_time, span) - duration_ms = get_duration_ms(span) + start_time = get_span_start_time(span, span_attrs) + end_time = get_span_end_time(start_time, span) + duration_ms = get_span_duration_ms(span) call_id = _hash_id("agentcall", agent_id, span_id) agent_call = registry.add( @@ -120,7 +224,7 @@ def handle_agent( duration=duration_ms, spanId=span_id, parentSpanId=str(span.get("ParentSpanId") or ""), - success=get_success(span, span_attrs), + success=get_span_success(span, span_attrs), ) ) registry.add_edge(executesAgent(source_id=agent_call.id, target_id=agent.id)) diff --git a/backend/norm/src/norm/ioa_observe/handlers/chat.py b/backend/norm/src/norm/ioa_observe/handlers/chat.py index a6ebccc..1a8c625 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/chat.py +++ b/backend/norm/src/norm/ioa_observe/handlers/chat.py @@ -14,12 +14,12 @@ _hash_id, _to_float, _to_int, - get_agent_id, - get_duration_ms, - get_end_time, - get_session_id, - get_start_time, - get_success, + get_span_agent_id, + get_span_duration_ms, + get_span_end_time, + get_span_session_id, + get_span_start_time, + get_span_success, ) from ..registry import Registry from ..trajectory import add_state_pair_transition @@ -59,8 +59,8 @@ def handle_chat( span_attrs: dict[str, Any], registry: Registry, ) -> None: - session_id = get_session_id(span, span_attrs) - agent_id = get_agent_id(span, span_attrs) + session_id = get_span_session_id(span, span_attrs) + agent_id = get_span_agent_id(span, span_attrs) span_id = str(span.get("SpanId") or "") parent_span_id = str(span.get("ParentSpanId") or "") @@ -78,9 +78,9 @@ def handle_chat( ) registry.add_edge(usesLLM(source_id=agent_id, target_id=llm.id)) - start_time = get_start_time(span, span_attrs) - end_time = get_end_time(start_time, span) - duration_ms = get_duration_ms(span) + start_time = get_span_start_time(span, span_attrs) + end_time = get_span_end_time(start_time, span) + duration_ms = get_span_duration_ms(span) call_id = _hash_id("llmcall", provider, model, span_id) llm_call = registry.add( @@ -100,7 +100,7 @@ def handle_chat( finishReason=_first_str(span_attrs.get("gen_ai.response.finish_reasons")), responseId=str(span_attrs.get("gen_ai.response.id") or ""), cacheReadTokenCount=_to_int(span_attrs.get("gen_ai.usage.cache_read.input_tokens")), - success=get_success(span, span_attrs), + success=get_span_success(span, span_attrs), ) ) registry.add_edge(executesLLM(source_id=llm_call.id, target_id=llm.id)) diff --git a/backend/norm/src/norm/ioa_observe/handlers/graph.py b/backend/norm/src/norm/ioa_observe/handlers/graph.py index 978840a..cbba519 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/graph.py +++ b/backend/norm/src/norm/ioa_observe/handlers/graph.py @@ -19,7 +19,7 @@ from scope_ontology.models.edges import belongsToMAS from scope_ontology.models.nodes import MAS, Agent, StructuralElement -from ..fields import get_application_id +from ..fields import get_span_application_id from ..registry import Registry log = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def handle_graph( span_attrs: dict[str, Any], registry: Registry, ) -> None: - application_id = get_application_id(span, span_attrs) + application_id = get_span_application_id(span, span_attrs) if not application_id: return name = str(span_attrs.get("ioa_observe.entity.name") or application_id) diff --git a/backend/norm/src/norm/ioa_observe/handlers/llm.py b/backend/norm/src/norm/ioa_observe/handlers/llm.py new file mode 100644 index 0000000..a2ce4b8 --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/handlers/llm.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Handlers for the "llm.*" events (OTel log records): the event-side of handle_chat.""" + +from __future__ import annotations + +from typing import Any + +from scope_ontology.models.edges import ( + executesLLM, + hasFinalState, + hasInitialState, + hasLLMCall, + usesLLM, +) +from scope_ontology.models.nodes import LLM, LLMCall + +from ..fields import _hash_id +from ..registry import Registry +from ..trajectory import add_final_state_transition, add_initial_state +from .agent import _own_agent_id +from .topology import _ensure_agent_call + + +def _ensure_llm_call( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> tuple[LLMCall, str, str, float] | None: + """Ensure the whole chain down to the LLMCall exists -- MAS, Session, MASCall, Agent, + AgentCall (the owning ``ioa_observe.agent.span_id``), LLM and LLMCall -- creating + whatever is missing. + + Returns ``(llm_call, session_id, span_id, event_time)``, or None if the event names + no agent, session or LLM. + """ + owning_span_id = str(event_attrs.get("ioa_observe.agent.span_id") or "") + ensured = _ensure_agent_call(event, event_attrs, registry, span_id=owning_span_id) + llm_name = str(event_attrs.get("llm.name") or "") + if ensured is None or not llm_name: + return None + agent_call, session_id, _, event_time = ensured + agent_id = _own_agent_id(registry, agent_call) or str(event_attrs.get("agent.name") or "") + span_id = str(event_attrs.get("llm.call.id") or event.get("SpanId") or event_attrs.get("span.id") or "") + + # llm.name is "/" (as in "azure/gpt-4o"); handle_chat gets them separately. + provider, _, model = llm_name.partition("/") + if not model: + provider, model = "", llm_name + llm, _ = registry.add_if_not_exists( + LLM(id=f"llm-{provider}-{model}", name=model, provider=provider) + ) + registry.add_edge(usesLLM(source_id=agent_id, target_id=llm.id)) + + llm_call, _ = registry.add_if_not_exists( + LLMCall( + id=_hash_id("llmcall", provider, model, span_id), + name=f"LLM call ({provider}/{model}) {span_id}", + sessionId=session_id, + startTime=event_time, + endTime=event_time, + duration=0.0, + spanId=span_id, + parentSpanId=owning_span_id, + ) + ) + registry.add_edge(executesLLM(source_id=llm_call.id, target_id=llm.id)) + registry.add_edge(hasLLMCall(source_id=agent_call.id, target_id=llm_call.id)) + return llm_call, session_id, span_id, event_time + + +def handle_llm_started( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the chain down to the LLMCall and give it its initial State (from + ``llm.input``) once. Its final State and Transition need ``llm.output``, so they + belong to ``llm.completed``.""" + ensured = _ensure_llm_call(event, event_attrs, registry) + if ensured is None: + return + llm_call, session_id, span_id, _ = ensured + if registry.edge_target(hasInitialState, llm_call.id) is None: + add_initial_state( + registry, llm_call.id, session_id, span_id, str(event_attrs.get("llm.input") or "") + ) + + +def handle_llm_completed( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the chain down to the LLMCall, close it, and give it its final State + (from ``llm.output``) and Transition. The initial State is whatever ``llm.started`` + stored; only if there is none is one created, from ``llm.input``.""" + ensured = _ensure_llm_call(event, event_attrs, registry) + if ensured is None: + return + llm_call, session_id, span_id, event_time = ensured + + if event_time > llm_call.endTime: + llm_call.endTime = event_time # type: ignore[attr-defined] + llm_call.duration = (event_time - llm_call.startTime) * 1000.0 # type: ignore[attr-defined] + + input_text = str(event_attrs.get("llm.input") or "") + initial_id = registry.edge_target(hasInitialState, llm_call.id) + if initial_id is None: + initial_id = add_initial_state(registry, llm_call.id, session_id, span_id, input_text).id + if registry.edge_target(hasFinalState, llm_call.id) is not None: + return + add_final_state_transition( + registry, + llm_call.id, + session_id, + span_id, + initial_id, + input_text, + str(event_attrs.get("llm.output") or ""), + ) diff --git a/backend/norm/src/norm/ioa_observe/handlers/mcp.py b/backend/norm/src/norm/ioa_observe/handlers/mcp.py new file mode 100644 index 0000000..861f3c0 --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/handlers/mcp.py @@ -0,0 +1,179 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Handlers for the "mcp.*" events (OTel log records).""" + +from __future__ import annotations + +import json +from typing import Any + +from scope_ontology.models.edges import hasFinalState, hasInitialState, usesTool +from scope_ontology.models.nodes import AgentCall, Tool, ToolCall + +from ..registry import Registry +from ..trajectory import add_final_state_transition, add_initial_state +from .agent import _own_agent_id +from .tool_events import _ensure_tool_call +from .topology import _ensure_agent_call, _ensure_session_and_mas + + +def _discovered_tools(event_attrs: dict[str, Any]) -> list[dict[str, Any]]: + """The tools listed in ``capability.content`` (a JSON object with a ``tools`` array), + or [] if absent or malformed.""" + content = event_attrs.get("capability.content") + if isinstance(content, str): + try: + content = json.loads(content) + except ValueError: + return [] + tools = content.get("tools") if isinstance(content, dict) else None + return [tool for tool in tools if isinstance(tool, dict)] if isinstance(tools, list) else [] + + +def handle_mcp_tools_discovered( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the MAS and Session, then create every tool in ``capability.content`` and link + each to the agent that discovered them (``usesTool``). + + Skipped without an ``ioa_observe.agent.span_id`` or a session. The MAS is named by the + first part of ``session.id`` (``_``). The event names the agent only by + its span id, but an Agent is keyed on its name, so the agent is the one of the AgentCall of + that span. If that call does not exist yet, the tools are created and not linked.""" + agent_span_id = str(event_attrs.get("ioa_observe.agent.span_id") or "") + mas_name = str(event_attrs.get("session.id") or "").partition("_")[0] + if not agent_span_id or not mas_name: + return + session = _ensure_session_and_mas(event, {**event_attrs, "application_id": mas_name}, registry) + if session is None: + return + agent_call = registry.find( + AgentCall, lambda call: call.spanId == agent_span_id and call.sessionId == session.id + ) + agent_id = _own_agent_id(registry, agent_call) if agent_call is not None else None + + for item in _discovered_tools(event_attrs): + name = str(item.get("name") or "") + if not name: + continue + description = str(item.get("description") or "") + tool, _ = registry.add_if_not_exists(Tool(id=name, name=name, description=description)) + if description and not tool.description: # created earlier by a tool event, without one + tool.description = description # type: ignore[attr-defined] + if agent_id: + registry.add_edge(usesTool(source_id=agent_id, target_id=tool.id)) + + +def _message_tool_name(message_input: Any) -> str: + """``name`` of the JSON object in ``message.input`` (the tool of a ``tools/call``), or "" + if the input is empty, not a JSON object, or has no name.""" + if isinstance(message_input, str): + try: + message_input = json.loads(message_input) + except ValueError: + return "" + return str(message_input.get("name") or "") if isinstance(message_input, dict) else "" + + +def _ensure_mcp_tool_call( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> tuple[ToolCall, str, str, float, str] | None: + """A tool call sent by an agent: ensure the MAS, Session, MASCall, Agent and AgentCall of + the sender, then the Tool (``message.input.name``) and the ToolCall, and its initial State + from ``message.input`` -- all created if missing. + + The sender is ``source.agent`` (name) and ``source.agent.span.id`` (the Agent's id and + the AgentCall's span). The ToolCall is keyed on ``message.id``, which + ``mcp.message.sent`` and ``mcp.message.completed`` share. + + Returns ``(tool_call, session_id, span_id, event_time, message_input)``, or None if + ``message.input`` is empty or names no tool, as in a ``tools/list`` request, or the event + names no sender or session.""" + message_input = str(event_attrs.get("message.input") or "") + tool_name = _message_tool_name(message_input) + source_name = str(event_attrs.get("source.agent") or "") + source_span_id = str( + event_attrs.get("source.agent.span.id") or event_attrs.get("ioa_observe.agent.span_id") or "" + ) + if not tool_name or not source_name or not source_span_id: + return None + mas_name = str(event_attrs.get("session.id") or "").partition("_")[0] + agent_attrs = { + **event_attrs, + "application_id": event_attrs.get("application_id") or mas_name, + "agent.name": source_name, + } + if _ensure_agent_call(event, agent_attrs, registry, span_id=source_span_id) is None: + return None + ensured = _ensure_tool_call( + event, + { + **agent_attrs, + "tool.name": tool_name, + "ioa_observe.agent.span_id": source_span_id, + # the message id is the id of the tool call: tool.started/completed carry it as tool.call.id + "tool.call.id": event_attrs.get("message.id"), + }, + registry, + ) + if ensured is None: + return None + tool_call, session_id, span_id, event_time = ensured + if registry.edge_target(hasInitialState, tool_call.id) is None: + add_initial_state(registry, tool_call.id, session_id, span_id, message_input) + return tool_call, session_id, span_id, event_time, message_input + + +def handle_mcp_message_sent( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Create the Tool and ToolCall of an agent's tool call, with everything above them + and the ToolCall's initial State (see ``_ensure_mcp_tool_call``). Nothing here closes it.""" + _ensure_mcp_tool_call(event, event_attrs, registry) + + +def handle_mcp_message_received( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """The same as ``mcp.message.sent``: the message reaching the server carries the same + input, sender and ``message.id``, so it lands on the same ToolCall (on another span).""" + _ensure_mcp_tool_call(event, event_attrs, registry) + + +def handle_mcp_message_completed( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """The same nodes as ``mcp.message.sent``, then close the ToolCall and give it its final + State (from ``message.output``) and Transition. ``success`` is ``not isError`` when the + output says so. A repeated event adds nothing.""" + ensured = _ensure_mcp_tool_call(event, event_attrs, registry) + if ensured is None: + return + tool_call, session_id, span_id, event_time, message_input = ensured + + if event_time > tool_call.endTime: + tool_call.endTime = event_time # type: ignore[attr-defined] + tool_call.duration = (event_time - tool_call.startTime) * 1000.0 # type: ignore[attr-defined] + output = str(event_attrs.get("message.output") or "") + is_error = _output_is_error(output) + if is_error is not None: + tool_call.success = not is_error # type: ignore[attr-defined] + + if registry.edge_target(hasFinalState, tool_call.id) is not None: + return + initial_id = registry.edge_target(hasInitialState, tool_call.id) + if initial_id is None: # _ensure_mcp_tool_call just made it + return + add_final_state_transition( + registry, tool_call.id, session_id, span_id, initial_id, message_input, output + ) + + +def _output_is_error(message_output: str) -> bool | None: + """The ``isError`` flag of a ``tools/call`` result in ``message.output``, if it has one.""" + try: + parsed = json.loads(message_output) + except ValueError: + return None + flag = parsed.get("isError") if isinstance(parsed, dict) else None + return flag if isinstance(flag, bool) else None diff --git a/backend/norm/src/norm/ioa_observe/handlers/session.py b/backend/norm/src/norm/ioa_observe/handlers/session.py index 507b3b9..7d17ab1 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/session.py +++ b/backend/norm/src/norm/ioa_observe/handlers/session.py @@ -11,10 +11,10 @@ from ..fields import ( _to_float, - get_application_id, - get_session_id, - get_start_time, - get_success, + get_span_application_id, + get_span_session_id, + get_span_start_time, + get_span_success, ) from ..registry import Registry @@ -22,11 +22,11 @@ def handle_session_start( span: dict[str, Any], span_attrs: dict[str, Any], registry: Registry ) -> None: - session_id = get_session_id(span, span_attrs) - application_id = get_application_id(span, span_attrs) + session_id = get_span_session_id(span, span_attrs) + application_id = get_span_application_id(span, span_attrs) if not session_id or registry.get(Session, session_id) is not None: return - start_time = get_start_time(span, span_attrs) + start_time = get_span_start_time(span, span_attrs) session = registry.add( Session( id=session_id, @@ -37,7 +37,7 @@ def handle_session_start( duration=0.0, spanId=str(span.get("SpanId") or ""), parentSpanId=str(span.get("ParentSpanId") or ""), - success=get_success(span, span_attrs), + success=get_span_success(span, span_attrs), ) ) if application_id: @@ -59,18 +59,18 @@ def handle_session_start( def handle_session_end( span: dict[str, Any], span_attrs: dict[str, Any], registry: Registry ) -> None: - session_id = get_session_id(span, span_attrs) - application_id = get_application_id(span, span_attrs) + session_id = get_span_session_id(span, span_attrs) + application_id = get_span_application_id(span, span_attrs) session: Session = registry.get(Session, session_id) if session is None: return # no matching session seen yet -- nothing to update - ended_at = _to_float(span_attrs.get("session.ended_at")) or get_start_time( + ended_at = _to_float(span_attrs.get("session.ended_at")) or get_span_start_time( span, span_attrs, ) session.endTime = ended_at # type: ignore[attr-defined] session.duration = max(0.0, (ended_at - session.startTime) * 1000.0) # type: ignore[attr-defined] - end_success = get_success(span, span_attrs) + end_success = get_span_success(span, span_attrs) if end_success is not None: session.success = end_success # type: ignore[attr-defined] if application_id: diff --git a/backend/norm/src/norm/ioa_observe/handlers/tool.py b/backend/norm/src/norm/ioa_observe/handlers/tool.py index 5875853..8e1f21d 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/tool.py +++ b/backend/norm/src/norm/ioa_observe/handlers/tool.py @@ -11,12 +11,12 @@ from ..fields import ( _hash_id, - get_agent_id, - get_duration_ms, - get_end_time, - get_session_id, - get_start_time, - get_success, + get_span_agent_id, + get_span_duration_ms, + get_span_end_time, + get_span_session_id, + get_span_start_time, + get_span_success, ) from ..registry import Registry from ..trajectory import add_state_pair_transition @@ -27,8 +27,8 @@ def handle_tool( span_attrs: dict[str, Any], registry: Registry, ) -> None: - session_id = get_session_id(span, span_attrs) - agent_id = get_agent_id(span, span_attrs) + session_id = get_span_session_id(span, span_attrs) + agent_id = get_span_agent_id(span, span_attrs) span_id = str(span.get("SpanId") or "") parent_span_id = str(span.get("ParentSpanId") or "") entity_name = str(span_attrs.get("ioa_observe.entity.name") or "") @@ -45,9 +45,9 @@ def handle_tool( ) registry.add_edge(usesTool(source_id=agent_id, target_id=tool.id)) - start_time = get_start_time(span, span_attrs) - end_time = get_end_time(start_time, span) - duration_ms = get_duration_ms(span) + start_time = get_span_start_time(span, span_attrs) + end_time = get_span_end_time(start_time, span) + duration_ms = get_span_duration_ms(span) call_id = _hash_id("toolcall", entity_name, span_id) tool_call = registry.add( @@ -60,7 +60,7 @@ def handle_tool( duration=duration_ms, spanId=span_id, parentSpanId=parent_span_id, - success=get_success(span, span_attrs), + success=get_span_success(span, span_attrs), ) ) registry.add_edge(executesTool(source_id=tool_call.id, target_id=tool.id)) diff --git a/backend/norm/src/norm/ioa_observe/handlers/tool_events.py b/backend/norm/src/norm/ioa_observe/handlers/tool_events.py new file mode 100644 index 0000000..9f7f31c --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/handlers/tool_events.py @@ -0,0 +1,143 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Handlers for the "tool.*" events (OTel log records): the event-side of handle_tool.""" + +from __future__ import annotations + +from typing import Any + +from scope_ontology.models.edges import ( + executesTool, + hasFinalState, + hasInitialState, + hasToolCall, + usesTool, +) +from scope_ontology.models.nodes import AgentCall, Tool, ToolCall + +from ..fields import ( + _hash_id, + get_event_application_id, + get_event_session_id, + get_event_start_time, +) +from ..registry import Registry +from ..trajectory import add_final_state_transition, add_initial_state +from .a2a import handle_a2a_tool_started +from .agent import _ensure_mas_call, _own_agent_id + + +def _ensure_tool_call( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> tuple[ToolCall, str, str, float] | None: + """Ensure MAS, Session, MASCall, Tool and ToolCall exist, creating whatever is missing, + and link the ToolCall to its AgentCall. + + Tool events carry no ``agent.name``, only the owning ``ioa_observe.agent.span_id``, so + the AgentCall (and Agent) can be found but not created here. If it isn't there yet, + ``topology.node.*`` links this ToolCall when it creates the AgentCall. + + Returns ``(tool_call, session_id, span_id, event_time)``, or None if the event names + no session or tool. + """ + tool_name = str(event_attrs.get("tool.name") or "") + session_id = get_event_session_id(event, event_attrs) + if not tool_name or not session_id: + return None + application_id = get_event_application_id(event, event_attrs) + owning_span_id = str(event_attrs.get("ioa_observe.agent.span_id") or "") + span_id = str(event.get("SpanId") or event_attrs.get("span.id") or "") + event_time = get_event_start_time(event, event_attrs) + + if application_id: + _ensure_mas_call(registry, session_id, application_id, event_time) + tool, _ = registry.add_if_not_exists(Tool(id=tool_name, name=tool_name, description="")) + # ``tool.call.id`` is the id of the call itself, shared by every event about it: the + # tool.* events and, for an MCP tool, the mcp.message.* events (as their message.id). + # Without it the call is told apart by the span of the event. + call_id = str(event_attrs.get("tool.call.id") or "") or _hash_id("toolcall", tool_name, span_id) + tool_call, _ = registry.add_if_not_exists( + ToolCall( + id=call_id, + name=f"Tool call ({tool_name}) {span_id}", + sessionId=session_id, + startTime=event_time, + endTime=event_time, + duration=0.0, + spanId=span_id, + parentSpanId=owning_span_id, + ) + ) + registry.add_edge(executesTool(source_id=tool_call.id, target_id=tool.id)) + + agent_call = ( + registry.find(AgentCall, lambda a: a.spanId == owning_span_id) if owning_span_id else None + ) + if agent_call is not None: + registry.add_edge(hasToolCall(source_id=agent_call.id, target_id=tool_call.id)) + agent_id = _own_agent_id(registry, agent_call) + if agent_id: + registry.add_edge(usesTool(source_id=agent_id, target_id=tool.id)) + return tool_call, session_id, span_id, event_time + + +def handle_tool_started( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the ToolCall's nodes and edges and give it its initial State (from + ``tool.input``) once. Its final State and Transition need ``tool.output``, so they + belong to ``tool.completed``. + + A tool of type ``a2a`` is a call to another agent, not a tool: it is a handoff (see + ``handle_a2a_tool_started``) and gets no Tool or ToolCall.""" + if str(event_attrs.get("tool.type") or "") == "a2a": + handle_a2a_tool_started(event, event_attrs, registry) + return + ensured = _ensure_tool_call(event, event_attrs, registry) + if ensured is None: + return + tool_call, session_id, span_id, _ = ensured + if registry.edge_target(hasInitialState, tool_call.id) is None: + add_initial_state( + registry, tool_call.id, session_id, span_id, str(event_attrs.get("tool.input") or "") + ) + + +def handle_tool_completed( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the ToolCall's nodes and edges, close it, and give it its final State + (from ``tool.output``) and Transition. The initial State is whatever ``tool.started`` + stored; only if there is none is one created, from ``tool.input``. + + A tool of type ``a2a`` is a handoff, recorded by ``tool.started``; nothing to close.""" + if str(event_attrs.get("tool.type") or "") == "a2a": + # TODO: the handoff's result (tool.output) is not recorded yet. + return + ensured = _ensure_tool_call(event, event_attrs, registry) + if ensured is None: + return + tool_call, session_id, span_id, event_time = ensured + + if event_time > tool_call.endTime: + tool_call.endTime = event_time # type: ignore[attr-defined] + tool_call.duration = (event_time - tool_call.startTime) * 1000.0 # type: ignore[attr-defined] + status = str(event_attrs.get("tool.status") or "") + if status: + tool_call.success = status == "success" # type: ignore[attr-defined] + + input_text = str(event_attrs.get("tool.input") or "") + initial_id = registry.edge_target(hasInitialState, tool_call.id) + if initial_id is None: + initial_id = add_initial_state(registry, tool_call.id, session_id, span_id, input_text).id + if registry.edge_target(hasFinalState, tool_call.id) is not None: + return + add_final_state_transition( + registry, + tool_call.id, + session_id, + span_id, + initial_id, + input_text, + str(event_attrs.get("tool.output") or ""), + ) diff --git a/backend/norm/src/norm/ioa_observe/handlers/topology.py b/backend/norm/src/norm/ioa_observe/handlers/topology.py new file mode 100644 index 0000000..79f466f --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/handlers/topology.py @@ -0,0 +1,249 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Handlers for the "topology.*" events (OTel log records).""" + +from __future__ import annotations + +from typing import Any + +from scope_ontology.models.edges import ( + belongsToMAS, + executesAgent, + executesSession, + executesTool, + hasAgentCall, + hasFinalState, + hasInitialState, + hasToolCall, + usesTool, +) +from scope_ontology.models.nodes import MAS, Agent, AgentCall, Session, State, ToolCall + +from ..fields import ( + _hash_id, + get_event_application_id, + get_event_session_id, + get_event_start_time, +) +from ..registry import Registry +from ..trajectory import add_final_state_transition, add_initial_state +from .agent import _agent_id_for, _ensure_agent, _ensure_mas_call, _own_agent_id + + +def handle_topology_edge_updated( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """The handoff from one agent to another: an ``executesAgent`` edge from the source's + AgentCall to the target Agent. + + The source is the AgentCall of ``source.agent.span.id``; the target is named only + (``target.agent``), so it is the Agent of that name. Nothing is created: the agents' + own events do that, and an edge to something that is not an agent (an MCP server, say) + matches nothing.""" + source_span_id = str(event_attrs.get("source.agent.span.id") or "") + target_name = str(event_attrs.get("target.agent") or "") + session_id = get_event_session_id(event, event_attrs) + if not source_span_id or not target_name or not session_id: + return + source = registry.find( + AgentCall, lambda call: call.spanId == source_span_id and call.sessionId == session_id + ) + target = registry.find(Agent, lambda agent: agent.name == target_name) + if source is not None and target is not None and target.id != _own_agent_id(registry, source): + registry.add_edge(executesAgent(source_id=source.id, target_id=target.id)) + + +def _ensure_session_and_mas( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> Session | None: + """Ensure the Session and its MAS exist, linked by ``executesSession``, creating + whichever is missing. None if the event has no session id.""" + session_id = get_event_session_id(event, event_attrs) + application_id = get_event_application_id(event, event_attrs) + if not session_id: + return None + start_time = get_event_start_time(event, event_attrs) + + session, _ = registry.add_if_not_exists( + Session( + id=session_id, + name=f"Session {session_id}", + sessionId=session_id, + startTime=start_time, + endTime=start_time, + duration=0.0, + spanId=str(event.get("SpanId") or event_attrs.get("span.id") or ""), + ) + ) + if application_id: + mas, _ = registry.add_if_not_exists( + MAS(id=application_id, name=application_id, description=application_id) + ) + registry.add_edge(executesSession(source_id=session.id, target_id=mas.id)) + return session + + +def _ensure_session_initial_state(registry: Registry, session: Session) -> str: + """The Session's first State, with ``hasInitialState``, created once. Keyed on the + session id: the session events carry no span id of their own, nor any content.""" + initial_id = registry.edge_target(hasInitialState, session.id) + if initial_id is None: + initial_id = add_initial_state(registry, session.id, session.sessionId, session.id, "").id + return initial_id + + +def handle_topology_session_started( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the Session and its MAS, and give the Session its initial State.""" + session = _ensure_session_and_mas(event, event_attrs, registry) + if session is not None: + _ensure_session_initial_state(registry, session) + + +def handle_topology_session_completed( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Same nodes as session.started, then close the Session and give it a final State + and Transition. The event carries no input/output, so the States have no content. + + Every ``session.completed`` adds a State (and Transition), but only the latest one + -- by ``event.time``, not by arrival -- holds ``hasFinalState``: a later event moves + it, an earlier or repeated one leaves it where it is.""" + session = _ensure_session_and_mas(event, event_attrs, registry) + if session is None: + return + end_time = get_event_start_time(event, event_attrs) + initial_id = _ensure_session_initial_state(registry, session) + + # Session.endTime only ever moves forward, so it is the latest completion seen so far + # (it starts at startTime, hence the check for no final State yet). + is_latest = registry.edge_target(hasFinalState, session.id) is None or end_time > session.endTime + if end_time > session.endTime: + session.endTime = end_time # type: ignore[attr-defined] + session.duration = (end_time - session.startTime) * 1000.0 # type: ignore[attr-defined] + if is_latest: + registry.remove_edges_where(lambda cls, src, _: cls is hasFinalState and src == session.id) + add_final_state_transition( + registry, + session.id, + session.sessionId, + f"{session.id}@{end_time}", # one final State per completion time + initial_id, + "", + "", + mark_final=is_latest, + ) + + +def _ensure_agent_call( + event: dict[str, Any], + event_attrs: dict[str, Any], + registry: Registry, + span_id: str | None = None, +) -> tuple[AgentCall, str, str, float] | None: + """Agent-side counterpart of handle_agent: make sure the MAS, Session, MASCall, + Agent and AgentCall for this event's node exist, creating whichever are missing. + + ``span_id`` is the AgentCall's span (default: the event's own, as for ``node.*`` + events; ``llm.*``/``tool.*`` events pass their owning agent span). + + Returns ``(agent_call, session_id, span_id, event_time)``, or None if the event + names no agent or session. + """ + agent_id = str(event_attrs.get("agent.name") or "") + session_id = get_event_session_id(event, event_attrs) + if not agent_id or not session_id: + return None + application_id = get_event_application_id(event, event_attrs) + if span_id is None: + span_id = str(event.get("SpanId") or event_attrs.get("span.id") or "") + event_time = get_event_start_time(event, event_attrs) + + # The Agent is keyed on the agent's name, so the Agent that an a2a.agent_card or a handoff + # created under that name is the one reused. (The same name in another MAS is another + # Agent: see _agent_id_for.) The AgentCall is what carries the span id. + mas_prefix = str(event_attrs.get("session.id") or "").partition("_")[0] + mas_ids = (application_id, mas_prefix) + agent = _ensure_agent(registry, _agent_id_for(registry, agent_id, mas_ids), agent_id, mas_ids) + if application_id: + registry.add_edge(belongsToMAS(source_id=agent.id, target_id=application_id)) + + agent_call, _ = registry.add_if_not_exists( + AgentCall( + id=_hash_id("agentcall", agent.id, span_id), + name=f"{agent.id} call {span_id}", + sessionId=session_id, + startTime=event_time, + endTime=event_time, + duration=0.0, + spanId=span_id, + ) + ) + registry.add_edge(executesAgent(source_id=agent_call.id, target_id=agent.id)) + if application_id: + mas_call = _ensure_mas_call(registry, session_id, application_id, agent_call.startTime) + registry.add_edge(hasAgentCall(source_id=mas_call.id, target_id=agent_call.id)) + # Tool events name only their owning agent *span*, so a ToolCall can arrive before its + # AgentCall exists: attach the ones already waiting for this span. + for tool_call in registry.all_of(ToolCall): + if span_id and tool_call.parentSpanId == span_id: + registry.add_edge(hasToolCall(source_id=agent_call.id, target_id=tool_call.id)) + tool_id = registry.edge_target(executesTool, tool_call.id) + if tool_id: + registry.add_edge(usesTool(source_id=agent.id, target_id=tool_id)) + return agent_call, session_id, span_id, event_time + + +def handle_topology_node_started( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the agent's nodes exist and give the AgentCall its initial State + (from ``agent.input``) -- once; if it already has one, it is left alone.""" + ensured = _ensure_agent_call(event, event_attrs, registry) + if ensured is None: + return + agent_call, session_id, span_id, _ = ensured + if registry.edge_target(hasInitialState, agent_call.id) is None: + add_initial_state( + registry, agent_call.id, session_id, span_id, str(event_attrs.get("agent.input") or "") + ) + + +def handle_topology_node_completed( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + """Ensure the agent's nodes exist, close the AgentCall and give it its final + State (from ``agent.output``) and Transition. The initial State is whatever + ``node.started`` stored; only if there is none is one created, from ``agent.input``.""" + ensured = _ensure_agent_call(event, event_attrs, registry) + if ensured is None: + return + agent_call, session_id, span_id, event_time = ensured + + if event_time > agent_call.endTime: + agent_call.endTime = event_time # type: ignore[attr-defined] + agent_call.duration = (event_time - agent_call.startTime) * 1000.0 # type: ignore[attr-defined] + + # node.completed carries agent.input as well as agent.output. If it is ever missing, + # the stored initial State (from node.started) supplies the input below. + input_text = str(event_attrs.get("agent.input") or "") + + initial_id = registry.edge_target(hasInitialState, agent_call.id) + if initial_id is None: + initial_id = add_initial_state(registry, agent_call.id, session_id, span_id, input_text).id + if registry.edge_target(hasFinalState, agent_call.id) is not None: + return + if not input_text: + initial = registry.get(State, initial_id) + stored = initial.content if initial is not None else "" + input_text = "" if stored == "(no content recorded)" else stored + add_final_state_transition( + registry, + agent_call.id, + session_id, + span_id, + initial_id, + input_text, + str(event_attrs.get("agent.output") or ""), + ) diff --git a/backend/norm/src/norm/ioa_observe/heuristics/handoff.py b/backend/norm/src/norm/ioa_observe/heuristics/handoff.py index eff2792..3c631b9 100644 --- a/backend/norm/src/norm/ioa_observe/heuristics/handoff.py +++ b/backend/norm/src/norm/ioa_observe/heuristics/handoff.py @@ -37,7 +37,7 @@ from scope_ontology.models.edges import hasAgentCall, hasFinalState, hasInitialState from scope_ontology.models.nodes import AgentCall, MASCall -from ..fields import _hash_id, attrs, get_agent_id +from ..fields import _hash_id, get_span_agent_id, get_span_attrs from ..registry import Registry from ..trajectory import bridge_states_if_mismatched @@ -87,12 +87,12 @@ def link_agent_handoffs(spans: list[dict[str, Any]], registry: Registry) -> set[ span_name = str(span.get("SpanName") or "") if not span_name.endswith(".agent"): continue - span_attrs = attrs(span) + span_attrs = get_span_attrs(span) source_span_ids = _source_span_ids(span_attrs) if not source_span_ids: continue - agent_id = get_agent_id(span, span_attrs) + agent_id = get_span_agent_id(span, span_attrs) span_id = str(span.get("SpanId") or "") target_call = registry.get(AgentCall, _hash_id("agentcall", agent_id, span_id)) if target_call is None: diff --git a/backend/norm/src/norm/ioa_observe/otel_io.py b/backend/norm/src/norm/ioa_observe/otel_io.py index 663632f..da0e434 100644 --- a/backend/norm/src/norm/ioa_observe/otel_io.py +++ b/backend/norm/src/norm/ioa_observe/otel_io.py @@ -5,7 +5,7 @@ No transformation into an intermediate span shape: ``normalize()`` and the ``ioa_observe`` handlers consume these dicts directly (``SpanId``, ``ParentSpanId``, ``SpanName``, ``SpanAttributes``, ``Timestamp``, -``Duration``, ...) via ``ioa_observe.fields.attrs()``/``get_session_id()``/etc. +``Duration``, ...) via ``ioa_observe.fields.get_span_attrs()``/``get_span_session_id()``/etc. """ from __future__ import annotations @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -from .fields import attrs, get_session_id +from .fields import get_span_attrs, get_span_session_id def load_otel_export(path: str | Path) -> list[dict[str, Any]]: @@ -30,7 +30,7 @@ def load_otel_export(path: str | Path) -> list[dict[str, Any]]: def infer_run_id(spans: list[dict[str, Any]], default: str = "unknown-run") -> str: """Return the (stripped) session id spans were recorded under.""" for span in spans: - session_id = get_session_id(span, attrs(span)) + session_id = get_span_session_id(span, get_span_attrs(span)) if session_id: return session_id return default diff --git a/backend/norm/src/norm/ioa_observe/registry.py b/backend/norm/src/norm/ioa_observe/registry.py index 33ec9d3..c2e2f52 100644 --- a/backend/norm/src/norm/ioa_observe/registry.py +++ b/backend/norm/src/norm/ioa_observe/registry.py @@ -53,6 +53,14 @@ def add_edge(self, edge: KGBase) -> None: key = (type(edge), edge.source_id, edge.target_id) self.edges.setdefault(key, edge) + def remove_node(self, cls: type[KGBase], node_id: str) -> None: + self.nodes.pop((cls.__name__, node_id), None) + + def remove_edges_where(self, predicate: Callable[[type, str, str], bool]) -> None: + """Drop every edge for which predicate(edge_class, source_id, target_id) holds.""" + for key in [k for k in self.edges if predicate(*k)]: + del self.edges[key] + def find(self, cls: type[T], predicate: Callable[[T], bool]) -> T | None: """Return the first node of this class satisfying predicate.""" for (type_name, _node_id), node in self.nodes.items(): diff --git a/backend/norm/src/norm/ioa_observe/stream.py b/backend/norm/src/norm/ioa_observe/stream.py new file mode 100644 index 0000000..2f8b612 --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/stream.py @@ -0,0 +1,349 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Stateless, span-at-a-time normalization against a graph database. + +``build_kg`` needs the whole span array. ``StreamNormalizer.process`` takes a +single span and keeps nothing between calls: the graph already stored in the +database *is* the state. For each span it + +1. loads what it needs from the database through a ``GraphReader`` (the + span's session subgraph plus the structural nodes the span touches), +2. adds the span to that graph with the usual per-span handler, +3. recomputes the cross-span heuristics (handoffs, capability chains, + container boundary states) over the loaded graph, and +4. returns only the difference against what was loaded, as a ``StreamDelta``, + for the caller to write back. + +The heuristics' derived items (bridge ProcessingCalls, container-boundary +Transitions and States) are dropped from the loaded graph and rebuilt every +time, so a late-arriving span replaces stale ones instead of piling up. + +The one thing a node cannot rebuild is a span's handoff signal +(``ioa_observe.handoff.source.span_ids``): ontology models forbid extra +fields, so it is carried on the AgentCall payload as the non-ontology +``handoffSourceSpanIds`` property. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Any, Literal, Protocol + +from pydantic import ValidationError +from scope_ontology.models import edges as edge_models +from scope_ontology.models import nodes as node_models +from scope_ontology.models.base import KGBase +from scope_ontology.models.edges import ( + correspondsToTransition, + executesAgent, + hasFinalState, + hasInitialState, + hasState, +) +from scope_ontology.models.nodes import Agent, AgentCall, MASCall, ProcessingCall, Session, Transition + +from .build import apply_heuristics, dispatch_span, serialize_registry +from .events import dispatch_event +from .fields import get_event_attrs, get_event_session_id, get_span_attrs, get_span_session_id +from .heuristics.handoff import _source_span_ids +from .registry import Registry +from ..verifier import PROPERTY_DOMAIN_RANGE + +Kind = Literal["span", "event"] +KINDS = ("span", "event") + +HANDOFF_SOURCES_PROP = "handoffSourceSpanIds" + + +@dataclass +class StreamDelta: + """What a single ``StreamNormalizer.process`` call changed in the database. + + ``nodes``/``edges`` are new or modified items; ``removed_*`` are items the + database holds that the recomputed graph no longer contains. Edges carry + ``from_type``/``to_type`` so they can be written without their endpoint + nodes being part of the same delta. + """ + + nodes: list[dict[str, Any]] = field(default_factory=list) + edges: list[dict[str, Any]] = field(default_factory=list) + removed_edges: list[dict[str, Any]] = field(default_factory=list) + removed_nodes: list[dict[str, Any]] = field(default_factory=list) + + +class GraphReader(Protocol): + def load( + self, session_id: str, node_ids: Sequence[str] + ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Return ``(nodes, edges)`` as stored. + + ``nodes``: every node with ``sessionId == session_id`` or ``id`` in + ``node_ids``, every Agent whose ``name`` is in ``node_ids`` (so ``node_ids`` + may also carry agent names), plus the endpoints of the edges below, each a + dict with ``node_type`` and its properties. ``edges``: every edge touching a + node of that session or one of those Agents, plus those among ``node_ids``, + each a dict with ``edge_type``, ``from_id`` and ``to_id``. + """ + ... + + +def _node_key(node: dict[str, Any]) -> tuple[str, str, str]: + return ("node", str(node["node_type"]), str(node["id"])) + + +def _edge_key(edge: dict[str, Any]) -> tuple[str, str, str, str]: + return ("edge", str(edge["edge_type"]), str(edge["from_id"]), str(edge["to_id"])) + + +def _rehydrate( + nodes: list[dict[str, Any]], edges: list[dict[str, Any]] +) -> tuple[Registry, dict[str, list[str]]]: + registry = Registry() + signals: dict[str, list[str]] = {} + for raw in nodes: + cls = getattr(node_models, str(raw["node_type"]), None) + if cls is None: + continue + data = {k: v for k, v in raw.items() if k in cls.model_fields} + try: + node = cls.model_validate(data) + except ValidationError: + continue # partial node (only an id) written ahead of its own span; that span will fill it in + registry.upsert(node) + sources = raw.get(HANDOFF_SOURCES_PROP) + if sources: + signals[node.id] = [str(item) for item in sources] + for raw in edges: + cls = getattr(edge_models, str(raw["edge_type"]), None) + if cls is None: + continue + registry.add_edge(cls(source_id=str(raw["from_id"]), target_id=str(raw["to_id"]))) + return registry, signals + + +def _strip_derived(registry: Registry) -> None: + """Drop everything ``apply_heuristics`` derives, so it can be rebuilt.""" + containers = {n.id for n in registry.all_of(Session)} | {n.id for n in registry.all_of(MASCall)} + bridges = {n.id for n in registry.all_of(ProcessingCall)} + owners = containers | bridges + transitions = { + target + for etype, source, target in registry.edges + if etype is correspondsToTransition and source in owners + } + for bridge_id in bridges: + registry.remove_node(ProcessingCall, bridge_id) + for transition_id in transitions: + registry.remove_node(Transition, transition_id) + gone = bridges | transitions + boundary_edges = (hasInitialState, hasFinalState, hasState) + registry.remove_edges_where( + lambda etype, source, target: ( + source in gone or target in gone or (etype in boundary_edges and source in containers) + ) + ) + + +def _handoff_spans(registry: Registry, signals: dict[str, list[str]]) -> list[dict[str, Any]]: + """Minimal ``*.agent`` spans carrying only the handoff signal, rebuilt from stored state.""" + spans: list[dict[str, Any]] = [] + for call in registry.all_of(AgentCall): + sources = signals.get(call.id) + agent_id = registry.edge_target(executesAgent, call.id) + if not sources or agent_id is None: + continue + spans.append( + { + "SpanName": "handoff.agent", + "SpanId": call.spanId, + "SpanAttributes": { + "agent_id": agent_id, + "ioa_observe.handoff.source.span_ids": json.dumps(sources), + }, + } + ) + return spans + + +def _endpoint_type(edge_type: str, side: int, node_id: str, node_types: dict[str, str]) -> str: + """Type of an edge endpoint: its node's own type, else the one class the + ontology allows there (``side`` 0 = domain, 1 = range). "" if ambiguous. + + An endpoint node can legitimately not exist yet -- e.g. ``usesLLM`` is + emitted by a chat span before the agent span that creates the Agent -- + in which case the database holds a partial (id-only) node until then. + """ + if node_id in node_types: + return node_types[node_id] + candidates = PROPERTY_DOMAIN_RANGE.get(edge_type, ([], []))[side] + return str(candidates[0]) if len(candidates) == 1 else "" + + +def _snapshot(registry: Registry, signals: dict[str, list[str]]) -> dict[tuple[str, ...], dict[str, Any]]: + nodes, edges = serialize_registry(registry) + node_types = {str(n["id"]): str(n["node_type"]) for n in nodes} + items: dict[tuple[str, ...], dict[str, Any]] = {} + for node in nodes: + if node["node_type"] == "AgentCall" and signals.get(str(node["id"])): + node[HANDOFF_SOURCES_PROP] = signals[str(node["id"])] + items[_node_key(node)] = node + for edge in edges: + edge["from_type"] = _endpoint_type(edge["edge_type"], 0, str(edge["from_id"]), node_types) + edge["to_type"] = _endpoint_type(edge["edge_type"], 1, str(edge["to_id"]), node_types) + items[_edge_key(edge)] = edge + return items + + +def _diff(before: dict[tuple[str, ...], dict[str, Any]], after: dict[tuple[str, ...], dict[str, Any]]) -> StreamDelta: + delta = StreamDelta() + for key, item in after.items(): + if before.get(key) != item: + (delta.nodes if key[0] == "node" else delta.edges).append(item) + for key, item in before.items(): + if key not in after: + (delta.removed_nodes if key[0] == "node" else delta.removed_edges).append(item) + return delta + + +class StreamNormalizer: + def __init__(self, reader: GraphReader) -> None: + self._reader = reader + + def process(self, record: dict[str, Any], kind: Kind = "span") -> StreamDelta: + """Normalize one item against the stored graph and return what changed. + + ``kind`` says what ``record`` is: ``"span"`` (an OTel span, the default) or + ``"event"`` (an OTel log record). Events are routed by ``event.name`` (see events.py); their + handlers are stubs for now, so they change nothing. + """ + if kind == "event": + return self._process_event(record) + elif kind == "span": + return self._process_trace(record) + else: + raise ValueError(f"Unknown kind {kind!r}, expected one of {KINDS}") + + def _process_event(self, event: dict[str, Any]) -> StreamDelta: + event_attrs = get_event_attrs(event) + session_id = get_event_session_id(event, event_attrs) + + # Dry run on an empty registry to learn which nodes this event touches. + scratch = Registry() + if not dispatch_event(event, event_attrs, scratch): + return StreamDelta() + touched = {node.id for node in scratch.nodes.values()} + for _etype, source_id, target_id in scratch.edges: + touched.update((source_id, target_id)) + # Agents are keyed on a span id but matched by name (an a2a.agent_card has no span id), + # so the reader is also handed their names: it loads the Agents called that. + touched.update(node.name for node in scratch.all_of(Agent)) + + # The dry run can't see what depends on stored state (e.g. an edge to an agent found + # through an existing AgentCall), so if the real run touches more, load those too + # and run again -- otherwise their stored copies would be missed and re-sent. + for _ in range(2): + stored_nodes, stored_edges = self._reader.load(session_id, sorted(touched)) + registry, signals = _rehydrate(stored_nodes, stored_edges) + before = _snapshot(registry, signals) + dispatch_event(event, event_attrs, registry) + after = _snapshot(registry, signals) + produced: set[str] = set() + for key, item in after.items(): + if before.get(key) != item: + produced.update( + (str(item["from_id"]), str(item["to_id"])) + if key[0] == "edge" + else (str(item["id"]),) + ) + if produced <= touched: + break + touched |= produced + return _diff(before, after) + + def _process_trace(self, span: dict[str, Any]) -> StreamDelta: + span_attrs = get_span_attrs(span) + session_id = get_span_session_id(span, span_attrs) + + # Dry run on an empty registry to learn which nodes this span touches. + scratch = Registry() + dispatch_span(span, scratch) + touched = {node.id for node in scratch.nodes.values()} + for _etype, source_id, target_id in scratch.edges: + touched.update((source_id, target_id)) + + stored_nodes, stored_edges = self._reader.load(session_id, sorted(touched)) + registry, signals = _rehydrate(stored_nodes, stored_edges) + + # Already processed (re-delivery / re-run): handlers are not order-independent, + # so replaying a span into a complete graph would change it. Make it a no-op. + span_id = str(span.get("SpanId") or "") + if span_id and any(getattr(n, "spanId", None) == span_id for n in registry.nodes.values()): + return StreamDelta() + + before = _snapshot(registry, signals) + + dispatch_span(span, registry) + if str(span.get("SpanName") or "").endswith(".agent"): + sources = _source_span_ids(span_attrs) + call = registry.find(AgentCall, lambda a: a.spanId == span_id) + if sources and call is not None: + signals[call.id] = sources + + _strip_derived(registry) + apply_heuristics(_handoff_spans(registry, signals), registry) + after = _snapshot(registry, signals) + + return _diff(before, after) + + +class InMemoryGraph: + """A ``GraphReader`` plus writer over plain dicts; for tests and dry runs.""" + + # Set on creation and never changed afterwards, same as the Neo4j store (endTime may advance). + IMMUTABLE_PROPS = frozenset({"startTime"}) + + def __init__(self) -> None: + self.nodes: dict[tuple[str, str], dict[str, Any]] = {} + self.edges: dict[tuple[str, str, str], dict[str, Any]] = {} + + def load( + self, session_id: str, node_ids: Sequence[str] + ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + ids = set(node_ids) + by_id = {node["id"]: node for node in self.nodes.values()} + in_session = {i for i, n in by_id.items() if session_id and n.get("sessionId") == session_id} + named = {i for i, n in by_id.items() if n["node_type"] == "Agent" and n.get("name") in ids} + edges = [ + e + for e in self.edges.values() + if e["from_id"] in in_session + or e["to_id"] in in_session + or e["from_id"] in named + or e["to_id"] in named + or (e["from_id"] in ids and e["to_id"] in ids) + ] + wanted = ( + in_session | named | ids | {e["from_id"] for e in edges} | {e["to_id"] for e in edges} + ) + nodes = [dict(by_id[i]) for i in wanted if i in by_id] + return nodes, [dict(e) for e in edges] + + def apply(self, delta: StreamDelta) -> None: + for edge in delta.removed_edges: + self.edges.pop((edge["edge_type"], edge["from_id"], edge["to_id"]), None) + for node in delta.removed_nodes: + self.nodes.pop((node["node_type"], node["id"]), None) + self.edges = { + k: e for k, e in self.edges.items() if node["id"] not in (e["from_id"], e["to_id"]) + } + for node in delta.nodes: + key = (node["node_type"], node["id"]) + existing = self.nodes.get(key) + if existing is None: + self.nodes[key] = dict(node) + else: + existing.update({k: v for k, v in node.items() if k not in self.IMMUTABLE_PROPS}) + for edge in delta.edges: + self.edges[(edge["edge_type"], edge["from_id"], edge["to_id"])] = dict(edge) diff --git a/backend/norm/src/norm/ioa_observe/trajectory.py b/backend/norm/src/norm/ioa_observe/trajectory.py index e860a9a..69b81fa 100644 --- a/backend/norm/src/norm/ioa_observe/trajectory.py +++ b/backend/norm/src/norm/ioa_observe/trajectory.py @@ -23,26 +23,37 @@ from .registry import Registry -def add_state_pair_transition( - registry: Registry, - owner_id: str, - session_id: str, - span_id: str, - input_text: str, - output_text: str, -) -> None: - """Create initial/final State nodes for owner_id and wire hasInitialState/ - hasFinalState/hasState, plus the Transition connecting them - (State->Transition->State) and the correspondsToTransition/ - representsExecution pair back to owner_id -- every ExecutionElement needs - one.""" - initial_state = registry.add( +def add_initial_state( + registry: Registry, owner_id: str, session_id: str, span_id: str, input_text: str +) -> State: + """Create owner_id's initial State (hasInitialState + hasState).""" + state = registry.add( State( id=_hash_id("state", span_id, "initial", input_text), sessionId=session_id, content=input_text or "(no content recorded)", ) ) + registry.add_edge(hasInitialState(source_id=owner_id, target_id=state.id)) + registry.add_edge(hasState(source_id=owner_id, target_id=state.id)) + return state + + +def add_final_state_transition( + registry: Registry, + owner_id: str, + session_id: str, + span_id: str, + initial_state_id: str, + input_text: str, + output_text: str, + mark_final: bool = True, +) -> State: + """Create owner_id's final State (hasFinalState + hasState) and the Transition + initial -> final, with correspondsToTransition/representsExecution to owner_id. + + With ``mark_final=False`` the State is only attached with hasState: it is a + final-looking State that a later one has already superseded as owner_id's final.""" final_state = registry.add( State( id=_hash_id("state", span_id, "final", output_text), @@ -50,9 +61,8 @@ def add_state_pair_transition( content=output_text or "(no content recorded)", ) ) - registry.add_edge(hasInitialState(source_id=owner_id, target_id=initial_state.id)) - registry.add_edge(hasState(source_id=owner_id, target_id=initial_state.id)) - registry.add_edge(hasFinalState(source_id=owner_id, target_id=final_state.id)) + if mark_final: + registry.add_edge(hasFinalState(source_id=owner_id, target_id=final_state.id)) registry.add_edge(hasState(source_id=owner_id, target_id=final_state.id)) transition = registry.add( @@ -61,15 +71,30 @@ def add_state_pair_transition( sessionId=session_id, ) ) - registry.add_edge(inputTo(source_id=initial_state.id, target_id=transition.id)) + registry.add_edge(inputTo(source_id=initial_state_id, target_id=transition.id)) registry.add_edge(leadsTo(source_id=transition.id, target_id=final_state.id)) - registry.add_edge( - correspondsToTransition( - source_id=owner_id, - target_id=transition.id, - ) - ) + registry.add_edge(correspondsToTransition(source_id=owner_id, target_id=transition.id)) registry.add_edge(representsExecution(source_id=transition.id, target_id=owner_id)) + return final_state + + +def add_state_pair_transition( + registry: Registry, + owner_id: str, + session_id: str, + span_id: str, + input_text: str, + output_text: str, +) -> None: + """Create initial/final State nodes for owner_id and wire hasInitialState/ + hasFinalState/hasState, plus the Transition connecting them + (State->Transition->State) and the correspondsToTransition/ + representsExecution pair back to owner_id -- every ExecutionElement needs + one.""" + initial_state = add_initial_state(registry, owner_id, session_id, span_id, input_text) + add_final_state_transition( + registry, owner_id, session_id, span_id, initial_state.id, input_text, output_text + ) def bridge_states_if_mismatched( diff --git a/backend/norm/src/norm/normalizer.py b/backend/norm/src/norm/normalizer.py index 6f4445a..3567f76 100644 --- a/backend/norm/src/norm/normalizer.py +++ b/backend/norm/src/norm/normalizer.py @@ -1,12 +1,29 @@ # Copyright (c) 2026 Cisco Systems, Inc. and its affiliates # SPDX-License-Identifier: Apache-2.0 import json +from collections.abc import Callable from os import PathLike from pathlib import Path from typing import Any, cast from .ioa_observe import build_kg +OnNormalizedCallback = Callable[ + [list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]], None +] + +_on_normalized: OnNormalizedCallback | None = None + + +def configure(on_normalized: OnNormalizedCallback | None = None) -> None: + """Configure the library. + + ``on_normalized(spans, nodes, edges)`` is invoked every time + :func:`normalize` finishes normalizing a trace. Pass ``None`` to clear it. + """ + global _on_normalized + _on_normalized = on_normalized + def dump_jsonld( nodes: list[dict[str, Any]], @@ -41,7 +58,10 @@ def normalize( ``norm.ioa_observe`` -- so there is no caller-supplied run_id/override plumbing here anymore. """ - return build_kg(spans) + nodes, edges = build_kg(spans) + if _on_normalized is not None: + _on_normalized(spans, nodes, edges) + return nodes, edges def load_json(fn: str | PathLike[str]) -> list[dict[str, Any]]: diff --git a/backend/norm/tests/test_normalizer.py b/backend/norm/tests/test_normalizer.py index 8b7e0dd..e029053 100644 --- a/backend/norm/tests/test_normalizer.py +++ b/backend/norm/tests/test_normalizer.py @@ -6,7 +6,18 @@ import json -from norm import Normalizer, normalize +from norm import Normalizer, StreamDelta, configure, normalize + + +def test_normalize_invokes_configured_callback() -> None: + calls = [] + configure(on_normalized=lambda spans, nodes, edges: calls.append((spans, nodes, edges))) + try: + spans = [_agent_span()] + nodes, edges = normalize(spans) + finally: + configure(on_normalized=None) + assert calls == [(spans, nodes, edges)] def _agent_span(session_id: str = "sess-1") -> dict: @@ -69,3 +80,939 @@ def test_normalizer_process_file_json_returns_flat_list(tmp_path) -> None: flat = Normalizer().process_file_json(path) nodes, edges = normalize([_agent_span()]) assert len(flat) == len(nodes) + len(edges) + + +def test_stream_normalizer_converges_to_batch_result() -> None: + from pathlib import Path + + from norm import InMemoryGraph, StreamNormalizer + from norm.normalizer import load_json + + fixture = Path(__file__).resolve().parent / "fixtures" / "noa_trip_planner.otel_export.jsonl" + spans = load_json(fixture) + + graph = InMemoryGraph() + for span in spans: + # A brand-new normalizer per span: nothing is kept in memory between spans. + graph.apply(StreamNormalizer(graph).process(span)) + + batch_nodes, batch_edges = normalize(spans) + assert {i for _, i in graph.nodes} == {n["id"] for n in batch_nodes} + assert set(graph.edges) == {(e["edge_type"], e["from_id"], e["to_id"]) for e in batch_edges} + + +def test_stream_normalizer_ignores_already_processed_span() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + graph.apply(StreamNormalizer(graph).process(_agent_span())) + assert StreamNormalizer(graph).process(_agent_span()) == StreamDelta() + + +def test_stream_normalizer_kind_event_dispatches_by_name(capsys) -> None: + import pytest + + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + delta = StreamNormalizer(graph).process( + {"LogAttributes": {"event.name": "llm.started"}}, kind="event" + ) + assert delta == StreamDelta() + capsys.readouterr() + assert not graph.nodes + + # "span" is the default and still normalizes the span + delta = StreamNormalizer(graph).process(_agent_span()) + assert delta.nodes + assert "llm.started" not in capsys.readouterr().out + + with pytest.raises(ValueError): + StreamNormalizer(graph).process(_agent_span(), kind="metric") # type: ignore[arg-type] + + +def _session_started_event() -> dict: + return { + "Timestamp": "2026-10-15 14:50:46.939986688", + "SpanId": "211fdd62f03f330c", + "ServiceName": "noa-trip-planner-mas", + "Body": "topology.session.started", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": "topology.session.started", + "event.time": "2026-10-15T14:50:46.938750+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "snapshot.version": "1", + }, + } + + +def test_topology_session_started_creates_session_and_mas() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + delta = normalizer.process(_session_started_event(), kind="event") + graph.apply(delta) + + assert sorted(n["node_type"] for n in delta.nodes) == ["MAS", "Session", "State"] + session = graph.nodes[("Session", "213dad2f")] + assert session["startTime"] == 1792075846.93875 + assert session["spanId"] == "211fdd62f03f330c" + assert sorted(e["edge_type"] for e in delta.edges) == [ + "executesSession", "hasInitialState", "hasState" + ] + assert not any(e["edge_type"] == "hasFinalState" for e in delta.edges) + + # re-delivery changes nothing + assert normalizer.process(_session_started_event(), kind="event") == StreamDelta() + + +def test_topology_session_started_without_session_id_is_noop() -> None: + from norm import InMemoryGraph, StreamNormalizer + + event = _session_started_event() + del event["LogAttributes"]["session.id"] + assert StreamNormalizer(InMemoryGraph()).process(event, kind="event") == StreamDelta() + + + +def _node_event(name: str, **extra: str) -> dict: + return { + "Timestamp": "2026-10-15 14:50:46.959707000", + "SpanId": "427bddeaf7b098f0", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": name, + "event.time": "2026-10-15T14:50:46.959707+00:00" if name.endswith("started") else "2026-10-15T14:50:48.500603+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "agent.name": "moderator", + **extra, + }, + } + + +def test_topology_node_events_build_agent_call_states_and_transition() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + for event in ( + _node_event("topology.node.started", **{"agent.input": "hello"}), + _node_event("topology.node.completed", **{"agent.output": "bye"}), + ): + graph.apply(normalizer.process(event, kind="event")) + + types = sorted(t for t, _ in graph.nodes) + assert types == ["Agent", "AgentCall", "MAS", "MASCall", "Session", "State", "State", "Transition"] + contents = sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State") + assert contents == ["bye", "hello"] + edge_types = {e["edge_type"] for e in graph.edges.values()} + assert {"hasInitialState", "hasFinalState", "inputTo", "leadsTo", "executesAgent", "hasAgentCall"} <= edge_types + + call = next(n for (t, _), n in graph.nodes.items() if t == "AgentCall") + assert call["endTime"] > call["startTime"] + assert call["duration"] > 0 + + # re-delivery changes nothing + assert normalizer.process(_node_event("topology.node.completed", **{"agent.output": "bye"}), kind="event") == StreamDelta() + + +def test_topology_node_completed_alone_creates_everything() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + delta = StreamNormalizer(graph).process( + _node_event("topology.node.completed", **{"agent.output": "bye"}), kind="event" + ) + graph.apply(delta) + states = sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State") + assert states == ["(no content recorded)", "bye"] + assert ("Transition", next(i for t, i in graph.nodes if t == "Transition")) in graph.nodes + + +def test_topology_session_completed_creates_missing_nodes_and_final_state() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + started = _session_started_event() + completed = { + "Timestamp": "2026-10-15 14:56:13.324206336", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + **started["LogAttributes"], + "event.name": "topology.session.completed", + "event.time": "2026-10-15T14:56:13.322348+00:00", + }, + } + # completed alone creates everything started does, plus the states/transition + graph.apply(normalizer.process(completed, kind="event")) + types = sorted(t for t, _ in graph.nodes) + assert types == ["MAS", "Session", "State", "State", "Transition"] + edge_types = {e["edge_type"] for e in graph.edges.values()} + assert {"executesSession", "hasInitialState", "hasFinalState", "inputTo", "leadsTo"} <= edge_types + assert normalizer.process(completed, kind="event") == StreamDelta() + + # started afterwards adds nothing; started-then-completed ends the session later + assert normalizer.process(started, kind="event") == StreamDelta() + graph2 = InMemoryGraph() + n2 = StreamNormalizer(graph2) + graph2.apply(n2.process(started, kind="event")) + graph2.apply(n2.process(completed, kind="event")) + session = graph2.nodes[("Session", "213dad2f")] + assert session["endTime"] > session["startTime"] and session["duration"] > 0 + assert sorted(t for t, _ in graph2.nodes) == types + + + +def _session_completed(time: str) -> dict: + started = _session_started_event() + return { + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + **started["LogAttributes"], + "event.name": "topology.session.completed", + "event.time": time, + }, + } + + +def _final_states(graph) -> list[str]: + return sorted(e["to_id"] for e in graph.edges.values() if e["edge_type"] == "hasFinalState") + + +def test_only_the_latest_session_completed_has_final_state() -> None: + from norm import InMemoryGraph, StreamNormalizer + + early = _session_completed("2026-10-15T14:56:13+00:00") + late = _session_completed("2026-10-15T14:56:18+00:00") + session_edges = lambda g, t: [e for e in g.edges.values() if e["edge_type"] == t and e["from_id"] == "213dad2f"] + + # in order: the final State moves to the later event, the earlier State stays attached + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + for event in (_session_started_event(), early): + graph.apply(normalizer.process(event, kind="event")) + first_final = _final_states(graph) + assert len(first_final) == 1 + graph.apply(normalizer.process(late, kind="event")) + assert len(_final_states(graph)) == 1 and _final_states(graph) != first_final + assert len(session_edges(graph, "hasInitialState")) == 1 + assert len(session_edges(graph, "hasState")) == 3 # initial + one per completion + assert graph.nodes[("Session", "213dad2f")]["endTime"] == 1792076178.0 + + # out of order: the earlier event arriving last does not take the final State back + graph2 = InMemoryGraph() + n2 = StreamNormalizer(graph2) + for event in (late, early): + graph2.apply(n2.process(event, kind="event")) + assert _final_states(graph2) == _final_states(graph) + assert len(session_edges(graph2, "hasState")) == 3 + # replays change nothing + assert n2.process(late, kind="event") == StreamDelta() + assert n2.process(early, kind="event") == StreamDelta() + + +def test_topology_node_completed_uses_agent_input_when_present() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + graph.apply( + StreamNormalizer(graph).process( + _node_event("topology.node.completed", **{"agent.input": "hello", "agent.output": "bye"}), + kind="event", + ) + ) + states = sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State") + assert states == ["bye", "hello"] + + +def test_llm_started_creates_chain_down_to_llm_call() -> None: + from norm import InMemoryGraph, StreamNormalizer + + event = { + "Timestamp": "2026-10-15 14:50:48.493184000", + "SpanId": "0fafd3f43fd76292", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": "llm.started", + "event.time": "2026-10-15T14:50:48.493184+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "agent.name": "moderator", + "ioa_observe.agent.span_id": "427bddeaf7b098f0", + "llm.call.id": "0fafd3f43fd76292", + "llm.name": "azure/gpt-4o", + "llm.input": "hi", + }, + } + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + graph.apply(normalizer.process(event, kind="event")) + + assert sorted(t for t, _ in graph.nodes) == [ + "Agent", "AgentCall", "LLM", "LLMCall", "MAS", "MASCall", "Session", "State", + ] + assert graph.nodes[("LLM", "llm-azure-gpt-4o")]["provider"] == "azure" + agent_call = next(n for (t, _), n in graph.nodes.items() if t == "AgentCall") + assert agent_call["spanId"] == "427bddeaf7b098f0" + edge_types = {e["edge_type"] for e in graph.edges.values()} + assert {"usesLLM", "executesLLM", "hasLLMCall", "hasAgentCall", "hasInitialState"} <= edge_types + assert normalizer.process(event, kind="event") == StreamDelta() + + +def _llm_event(name: str, **extra: str) -> dict: + started = name.endswith("started") + return { + "SpanId": "0fafd3f43fd76292", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": name, + "event.time": "2026-10-15T14:50:48.493184+00:00" if started else "2026-10-15T14:50:48.496038+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "agent.name": "moderator", + "ioa_observe.agent.span_id": "427bddeaf7b098f0", + "llm.call.id": "0fafd3f43fd76292", + "llm.name": "azure/gpt-4o", + "llm.input": "hi", + **extra, + }, + } + + +def test_llm_started_then_completed_builds_states_and_transition() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + completed = _llm_event("llm.completed", **{"llm.output": "yo"}) + for event in (_llm_event("llm.started"), completed): + graph.apply(normalizer.process(event, kind="event")) + + assert sorted(t for t, _ in graph.nodes) == [ + "Agent", "AgentCall", "LLM", "LLMCall", "MAS", "MASCall", "Session", "State", "State", "Transition", + ] + assert sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State") == ["hi", "yo"] + call = next(n for (t, _), n in graph.nodes.items() if t == "LLMCall") + assert call["endTime"] > call["startTime"] + edge_types = {e["edge_type"] for e in graph.edges.values()} + assert {"hasFinalState", "inputTo", "leadsTo", "correspondsToTransition"} <= edge_types + assert normalizer.process(completed, kind="event") == StreamDelta() + + +def test_llm_completed_alone_creates_everything() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + graph.apply( + StreamNormalizer(graph).process(_llm_event("llm.completed", **{"llm.output": "yo"}), kind="event") + ) + assert {"LLM", "LLMCall", "AgentCall", "Transition"} <= {t for t, _ in graph.nodes} + + +def _tool_event(name: str, **extra: str) -> dict: + return { + "SpanId": "abb8ef27ca74034a", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": name, + "event.time": "2026-10-15T14:50:49.664363+00:00" if name.endswith("started") else "2026-10-15T14:50:49.666468+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "ioa_observe.agent.span_id": "5371d7b3072ac5c7", + "tool.name": "read_document", + "tool.input": "in", + **extra, + }, + } + + +def _edges(graph) -> set[tuple[str, str, str]]: + return {(e["edge_type"], e["from_id"], e["to_id"]) for e in graph.edges.values()} + + +def test_tool_events_link_to_existing_agent_call() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + node_started = _node_event("topology.node.started", **{"agent.name": "schedule_agent", "agent.input": "q"}) + node_started["SpanId"] = "5371d7b3072ac5c7" + completed = _tool_event("tool.completed", **{"tool.output": "out", "tool.status": "success"}) + for event in (node_started, _tool_event("tool.started"), completed): + graph.apply(normalizer.process(event, kind="event")) + + assert {"Tool", "ToolCall", "AgentCall", "Agent"} <= {t for t, _ in graph.nodes} + edges = _edges(graph) + assert ("usesTool", "schedule_agent", "read_document") in edges + assert any(e[0] == "hasToolCall" for e in edges) and any(e[0] == "executesTool" for e in edges) + call = next(n for (t, _), n in graph.nodes.items() if t == "ToolCall") + assert call["success"] is True and call["endTime"] > call["startTime"] + assert sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State" and n["content"] in ("in", "out")) == ["in", "out"] + assert normalizer.process(completed, kind="event") == StreamDelta() + + +def test_a2a_tool_started_is_a_handoff_not_a_tool() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + a2a = {"tool.type": "a2a", "tool.name": "handoff_agent", "tool.input": "q"} + started = _tool_event("tool.started", **a2a) + graph.apply(normalizer.process(started, kind="event")) + + # no Tool/ToolCall; the target is an unattached Agent named tool.name + assert list(graph.nodes) == [("Agent", "handoff_agent")] + assert not graph.edges + assert normalizer.process(started, kind="event") == StreamDelta() + + # an Agent of that name already there is used, not duplicated + graph.apply(normalizer.process(_tool_event("tool.completed", **a2a, **{"tool.output": "out"}), kind="event")) + assert list(graph.nodes) == [("Agent", "handoff_agent")] + + # the target's own events then use that node (keyed on the name) and connect it + node = _node_event("topology.node.started", **{"agent.name": "handoff_agent", "agent.input": "q"}) + graph.apply(normalizer.process(node, kind="event")) + assert [i for t, i in graph.nodes if t == "Agent"] == ["handoff_agent"] + assert ("belongsToMAS", "handoff_agent", "noa-trip-planner-mas") in _edges(graph) + + +def test_non_a2a_tool_started_adds_tool_and_tool_call_to_the_agent() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + node = _node_event("topology.node.started", **{"agent.name": "schedule_agent", "agent.input": "q"}) + node["SpanId"] = "5371d7b3072ac5c7" + for event in (node, _tool_event("tool.started", **{"tool.type": "function"})): + graph.apply(normalizer.process(event, kind="event")) + assert {"Tool", "ToolCall"} <= {t for t, _ in graph.nodes} + edges = _edges(graph) + assert ("usesTool", "schedule_agent", "read_document") in edges + assert any(e[0] == "hasToolCall" for e in edges) + + +def _card_event(content: str) -> dict: + return { + "SpanId": "", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "event.name": "a2a.agent_card", + "event.time": "2026-10-15T14:50:40.000000+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "capability.content": content, + }, + } + + +def test_a2a_agent_card_creates_unattached_agent_then_events_connect_it() -> None: + import json + + from norm import InMemoryGraph, StreamNormalizer + + span = "schedule_agent" # an Agent is keyed on its name + card = json.dumps({"name": "schedule_agent", "description": "Finds schedules.", "skills": []}) + agents = lambda g: sorted(i for t, i in g.nodes if t == "Agent") + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + graph.apply(normalizer.process(_card_event(card), kind="event")) + assert sorted(graph.nodes) == [ + ("Agent", "schedule_agent"), + ("MAS", "noa-trip-planner-mas"), + ("Session", "213dad2f"), + ] + assert graph.nodes[("Agent", "schedule_agent")]["description"] == "Finds schedules." + assert ("belongsToMAS", "schedule_agent", "noa-trip-planner-mas") in _edges(graph) + assert ("executesSession", "213dad2f", "noa-trip-planner-mas") in _edges(graph) + assert normalizer.process(_card_event(card), kind="event") == StreamDelta() + + # the agent's own event uses the same node (keyed on the name), keeps the description + # and adds its call + node_started = _node_event("topology.node.started", **{"agent.name": "schedule_agent", "agent.input": "q"}) + graph.apply(normalizer.process(node_started, kind="event")) + assert agents(graph) == [span] + assert graph.nodes[("Agent", span)]["description"] == "Finds schedules." + assert graph.nodes[("Agent", span)]["name"] == "schedule_agent" + edges = _edges(graph) + assert ("belongsToMAS", span, "noa-trip-planner-mas") in edges + assert any(e[0] == "executesAgent" and e[2] == span for e in edges) + + # a card arriving after the agent exists updates its description only: no second Agent + late = InMemoryGraph() + late_normalizer = StreamNormalizer(late) + late.apply(late_normalizer.process(node_started, kind="event")) + assert late.nodes[("Agent", span)]["description"] == "" + edges_before = _edges(late) + late.apply(late_normalizer.process(_card_event(card), kind="event")) + assert agents(late) == [span] + assert late.nodes[("Agent", span)]["description"] == "Finds schedules." + newer = json.dumps({"name": "schedule_agent", "description": "Updated.", "version": "2"}) + late.apply(late_normalizer.process(_card_event(newer), kind="event")) + assert late.nodes[("Agent", span)]["description"] == "Updated." + assert _edges(late) == edges_before + # a card without a description leaves the existing one alone + nameless = json.dumps({"name": "schedule_agent"}) + assert late_normalizer.process(_card_event(nameless), kind="event") == StreamDelta() + + for bad in ("not json", "[]", "{}"): + empty = InMemoryGraph() + empty.apply(StreamNormalizer(empty).process(_card_event(bad), kind="event")) + assert not empty.nodes + + +def test_llm_events_key_the_agent_on_its_name() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + graph.apply(normalizer.process(_llm_event("llm.started"), kind="event")) + assert [i for t, i in graph.nodes if t == "Agent"] == ["moderator"] + assert any(e[0] == "usesLLM" and e[1] == "moderator" for e in _edges(graph)) + call = next(n for (t, _), n in graph.nodes.items() if t == "AgentCall") + assert call["spanId"] == "427bddeaf7b098f0" # the span id is the call's + + +def test_tool_event_before_agent_is_linked_when_agent_arrives() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + graph.apply(normalizer.process(_tool_event("tool.started"), kind="event")) + assert not {t for t, _ in graph.nodes} & {"Agent", "AgentCall"} + assert not any(e[0] in ("hasToolCall", "usesTool") for e in _edges(graph)) + + node_started = _node_event("topology.node.started", **{"agent.name": "schedule_agent", "agent.input": "q"}) + node_started["SpanId"] = "5371d7b3072ac5c7" + graph.apply(normalizer.process(node_started, kind="event")) + edges = _edges(graph) + assert ("usesTool", "schedule_agent", "read_document") in edges + assert any(e[0] == "hasToolCall" for e in edges) + + +def test_agent_card_without_session_is_skipped() -> None: + import json + + from norm import InMemoryGraph, StreamNormalizer + + event = _card_event(json.dumps({"name": "schedule_agent", "description": "d"})) + del event["LogAttributes"]["session.id"] + graph = InMemoryGraph() + graph.apply(StreamNormalizer(graph).process(event, kind="event")) + assert not graph.nodes + + +def test_agent_card_agent_is_reused_not_duplicated_in_the_same_mas() -> None: + import json + + from norm import InMemoryGraph, StreamNormalizer + + card = json.dumps({"name": "moderator", "description": "Routes."}) + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + graph.apply(normalizer.process(_card_event(card), kind="event")) + graph.apply(normalizer.process(_node_event("topology.node.started", **{"agent.input": "q"}), kind="event")) + assert [i for t, i in graph.nodes if t == "Agent"] == ["moderator"] + assert graph.nodes[("Agent", "moderator")]["description"] == "Routes." + assert ("belongsToMAS", "moderator", "noa-trip-planner-mas") in _edges(graph) + + +def test_agent_span_uses_the_unattached_agent_of_the_same_name() -> None: + import json + + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + card = json.dumps({"name": "concierge", "description": "Plans trips."}) + card_event = _card_event(card) + card_event["LogAttributes"]["session.id"] = "app_s1" # the span's MAS + graph.apply(normalizer.process(card_event, kind="event")) + assert ("Agent", "concierge") in graph.nodes + + # the span has no agent_id: the agent is known by its name, same as the card's + span = _agent_span() + del span["SpanAttributes"]["agent_id"] + span["SpanAttributes"]["ioa_observe.entity.name"] = "concierge" + graph.apply(normalizer.process(span)) + assert [i for t, i in graph.nodes if t == "Agent"] == ["concierge"] + assert graph.nodes[("Agent", "concierge")]["description"] == "Plans trips." + edges = _edges(graph) + assert ("belongsToMAS", "concierge", "app") in edges + assert any(e[0] == "executesAgent" and e[2] == "concierge" for e in edges) + + +def test_agent_span_does_not_reuse_the_agent_of_another_mas() -> None: + import json + + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + card_event = _card_event(json.dumps({"name": "concierge", "description": "Plans trips."})) + graph.apply(normalizer.process(card_event, kind="event")) # MAS noa-trip-planner-mas + span = _agent_span() + del span["SpanAttributes"]["agent_id"] + span["SpanAttributes"]["ioa_observe.entity.name"] = "concierge" + graph.apply(normalizer.process(span)) # MAS app + assert sorted(i for t, i in graph.nodes if t == "Agent") == ["app:concierge", "concierge"] + assert graph.nodes[("Agent", "app:concierge")]["description"] == "" + + +def _a2a_message_event(name: str, time: str) -> dict: + return { + "SpanId": "eac988190841ce1d", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": name, + "event.time": time, + "session.id": "noa-trip-planner-mas_213dad2f", + "message.id": "m-1", + "message.input": '{"messageId": "m-1"}', + "source.agent": "moderator", + "source.agent.span.id": "468f32a08faa2d00", + "target.agent": "Schedule Agent", + "span.id": "eac988190841ce1d", + }, + } + + +def test_a2a_message_sent_builds_sender_chain_and_keeps_the_message() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + sent = _a2a_message_event("a2a.message.sent", "2026-10-15T14:50:47.000000+00:00") + graph.apply(normalizer.process(sent, kind="event")) + + assert sorted(t for t, _ in graph.nodes) == [ + "Agent", "Agent", "AgentCall", "MAS", "MASCall", "Session", "State", + ] + assert {"moderator", "Schedule Agent"} == {i for t, i in graph.nodes if t == "Agent"} + call = next(i for t, i in graph.nodes if t == "AgentCall") + state = next(i for t, i in graph.nodes if t == "State") + edges = _edges(graph) + assert ("hasState", call, state) in edges + assert not any(e[0] in ("hasInitialState", "hasFinalState") for e in edges) + assert not any("Schedule Agent" in (e[1], e[2]) for e in edges) # recipient left unattached + assert normalizer.process(sent, kind="event") == StreamDelta() + + +def test_a2a_message_received_keeps_the_message_on_the_recipients_call() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + received = _a2a_message_event("a2a.message.received", "2026-10-15T14:50:47.020000+00:00") + + # the recipient's call is not there yet: the same nodes as sent, but no State + graph.apply(normalizer.process(received, kind="event")) + assert "State" not in {t for t, _ in graph.nodes} + assert "Schedule Agent" in {i for t, i in graph.nodes if t == "Agent"} + + # once the recipient's own node.started has created its call, the message is kept on it + node = _node_event("topology.node.started", **{"agent.name": "Schedule Agent", "agent.input": "q"}) + node["SpanId"] = "4163c385d3710aac" + graph.apply(normalizer.process(node, kind="event")) + graph.apply(normalizer.process(received, kind="event")) + receiver_call = next( + n["id"] for (t, _), n in graph.nodes.items() if t == "AgentCall" and n["spanId"] == "4163c385d3710aac" + ) + message_states = [ + e["to_id"] for e in graph.edges.values() + if e["edge_type"] == "hasState" and e["from_id"] == receiver_call + and graph.nodes[("State", e["to_id"])]["content"] == '{"messageId": "m-1"}' + ] + assert len(message_states) == 1 + assert normalizer.process(received, kind="event") == StreamDelta() + + +def _tools_discovered_event(with_span: bool = True, content: str | None = None) -> dict: + import json + + attrs = { + "event.name": "mcp.tools.discovered", + "event.time": "2026-10-15T14:50:48.000000+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "capability.content": content + if content is not None + else json.dumps( + { + "tools": [ + {"name": "list_docs", "description": "List documents."}, + {"name": "read_doc", "description": None}, + {"description": "nameless, skipped"}, + ] + } + ), + } + if with_span: + attrs["ioa_observe.agent.span_id"] = "4163c385d3710aac" + return {"SpanId": "b0877c49a582c991", "ServiceName": "someone-else", "LogAttributes": attrs} + + +def test_mcp_tools_discovered_creates_mas_session_and_tools_and_links_the_agent() -> None: + from norm import InMemoryGraph, StreamNormalizer + + # no ioa_observe.agent.span_id: skipped + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + assert normalizer.process(_tools_discovered_event(with_span=False), kind="event") == StreamDelta() + + # the agent of that span does not exist yet: MAS, Session and Tools only, nothing linked + event = _tools_discovered_event() + graph.apply(normalizer.process(event, kind="event")) + assert sorted(t for t, _ in graph.nodes) == ["MAS", "Session", "Tool", "Tool"] + # the MAS is named by the first part of session.id, not by application_id / ServiceName + assert ("MAS", "noa-trip-planner-mas") in graph.nodes + assert ("Session", "213dad2f") in graph.nodes + assert graph.nodes[("Tool", "list_docs")]["description"] == "List documents." + assert ("Tool", "read_doc") in graph.nodes + edges = _edges(graph) + assert ("executesSession", "213dad2f", "noa-trip-planner-mas") in edges + assert not any(e[0] == "usesTool" for e in edges) + assert normalizer.process(event, kind="event") == StreamDelta() + + # once the agent's call exists (found by its span id), the tools are linked to its Agent + node = _node_event("topology.node.started", **{"agent.name": "schedule_agent", "agent.input": "q"}) + node["SpanId"] = "4163c385d3710aac" + graph.apply(normalizer.process(node, kind="event")) + graph.apply(normalizer.process(event, kind="event")) + edges = _edges(graph) + assert ("usesTool", "schedule_agent", "list_docs") in edges + assert ("usesTool", "schedule_agent", "read_doc") in edges + assert [i for t, i in graph.nodes if t == "Agent"] == ["schedule_agent"] + + # bad content: the session and MAS are still ensured, no tools + bare = InMemoryGraph() + bare.apply(StreamNormalizer(bare).process(_tools_discovered_event(content="not json"), kind="event")) + assert sorted(t for t, _ in bare.nodes) == ["MAS", "Session"] + + +def test_mcp_tools_discovered_reuses_existing_agent_and_tool() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + node = _node_event("topology.node.started", **{"agent.name": "schedule_agent", "agent.input": "q"}) + node["SpanId"] = "4163c385d3710aac" + tool = _tool_event("tool.started", **{"tool.name": "list_docs"}) + tool["LogAttributes"]["ioa_observe.agent.span_id"] = "4163c385d3710aac" + for event in (node, tool): + graph.apply(normalizer.process(event, kind="event")) + assert graph.nodes[("Tool", "list_docs")]["description"] == "" + + graph.apply(normalizer.process(_tools_discovered_event(), kind="event")) + # the tool gets its description, nothing is duplicated + assert graph.nodes[("Agent", "schedule_agent")]["name"] == "schedule_agent" + assert sum(1 for t, _ in graph.nodes if t == "Agent") == 1 + assert graph.nodes[("Tool", "list_docs")]["description"] == "List documents." + + +def _mcp_message_sent_event(message_input: str) -> dict: + return { + "SpanId": "853e8b719de2f1ac", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": "mcp.message.sent", + "event.time": "2026-10-15T14:50:48.000000+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "message.input": message_input, + "source.agent": "Schedule Agent", + "source.agent.span.id": "4163c385d3710aac", + "span.id": "853e8b719de2f1ac", + "target.agent": "debug", + }, + } + + +def test_mcp_message_sent_creates_tool_and_tool_call_for_the_sender() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + # empty input, no tool name, or not JSON: ignored + for ignored in ("", "{}", '{"arguments": {}}', "not json"): + assert normalizer.process(_mcp_message_sent_event(ignored), kind="event") == StreamDelta() + + call_input = '{"arguments": {}, "name": "list_schedule_documents"}' + sent = _mcp_message_sent_event(call_input) + graph.apply(normalizer.process(sent, kind="event")) + assert sorted(t for t, _ in graph.nodes) == [ + "Agent", "AgentCall", "MAS", "MASCall", "Session", "State", "Tool", "ToolCall", + ] + assert graph.nodes[("Agent", "Schedule Agent")]["name"] == "Schedule Agent" + assert ("Tool", "list_schedule_documents") in graph.nodes + edges = _edges(graph) + assert ("usesTool", "Schedule Agent", "list_schedule_documents") in edges + assert {"hasToolCall", "executesTool", "executesAgent", "hasAgentCall", "hasInitialState"} <= {e[0] for e in edges} + assert next(n["content"] for (t, _), n in graph.nodes.items() if t == "State") == call_input + assert next(n["spanId"] for (t, _), n in graph.nodes.items() if t == "ToolCall") == "853e8b719de2f1ac" + assert normalizer.process(sent, kind="event") == StreamDelta() + + # the agent's own node.started finds the AgentCall made here: one call, one agent + node = _node_event("topology.node.started", **{"agent.name": "Schedule Agent", "agent.input": "q"}) + node["SpanId"] = "4163c385d3710aac" + graph.apply(normalizer.process(node, kind="event")) + assert sum(1 for t, _ in graph.nodes if t == "AgentCall") == 1 + assert sum(1 for t, _ in graph.nodes if t == "Agent") == 1 + + +def test_mcp_message_completed_closes_the_tool_call_with_its_output() -> None: + from norm import InMemoryGraph, StreamNormalizer + + call_input = '{"arguments": {}, "name": "list_schedule_documents"}' + output = '{"content": [{"text": "Celestia.txt"}], "isError": false}' + sent = _mcp_message_sent_event(call_input) + completed = _mcp_message_sent_event(call_input) + completed["LogAttributes"].update( + {"event.name": "mcp.message.completed", "event.time": "2026-10-15T14:50:50.000000+00:00", "message.output": output} + ) + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + for event in (sent, completed): + graph.apply(normalizer.process(event, kind="event")) + assert sorted(t for t, _ in graph.nodes) == [ + "Agent", "AgentCall", "MAS", "MASCall", "Session", "State", "State", "Tool", "ToolCall", "Transition", + ] + call = next(n for (t, _), n in graph.nodes.items() if t == "ToolCall") + assert call["success"] is True and call["endTime"] > call["startTime"] + assert sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State") == sorted([call_input, output]) + assert {"hasFinalState", "inputTo", "leadsTo", "correspondsToTransition"} <= {e["edge_type"] for e in graph.edges.values()} + assert normalizer.process(completed, kind="event") == StreamDelta() + + # completed alone builds everything sent does, plus the final State and Transition + alone = InMemoryGraph() + alone.apply(StreamNormalizer(alone).process(completed, kind="event")) + assert sorted(t for t, _ in alone.nodes) == sorted(t for t, _ in graph.nodes) + + # an error result marks the call failed; tools/list (empty input) is ignored + failed = _mcp_message_sent_event(call_input) + failed["LogAttributes"].update({"event.name": "mcp.message.completed", "message.output": '{"isError": true}'}) + bad = InMemoryGraph() + bad.apply(StreamNormalizer(bad).process(failed, kind="event")) + assert next(n for (t, _), n in bad.nodes.items() if t == "ToolCall")["success"] is False + listing = _mcp_message_sent_event("{}") + listing["LogAttributes"]["event.name"] = "mcp.message.completed" + assert StreamNormalizer(InMemoryGraph()).process(listing, kind="event") == StreamDelta() + + +def test_one_tool_call_per_message_id_across_tool_and_mcp_events() -> None: + from norm import InMemoryGraph, StreamNormalizer + + call_input = '{"arguments": {}, "name": "list_schedule_documents"}' + message_id = "b74c5046d0194ed7b005eee95857a563" + sent = _mcp_message_sent_event(call_input) + done = _mcp_message_sent_event(call_input) + sent["LogAttributes"]["message.id"] = done["LogAttributes"]["message.id"] = message_id + done["LogAttributes"].update( + {"event.name": "mcp.message.completed", "event.time": "2026-10-15T14:50:50+00:00", "message.output": '{"isError": false}'} + ) + # the tool.* events of the same call: another span, the same tool.call.id + tool_started = _tool_event( + "tool.started", **{"tool.name": "list_schedule_documents", "tool.call.id": message_id, "tool.input": "{}"} + ) + tool_started["LogAttributes"]["ioa_observe.agent.span_id"] = "4163c385d3710aac" + tool_done = _tool_event( + "tool.completed", + **{"tool.name": "list_schedule_documents", "tool.call.id": message_id, "tool.output": "[files]", "tool.status": "success"}, + ) + tool_done["LogAttributes"]["ioa_observe.agent.span_id"] = "4163c385d3710aac" + + for order in ((tool_started, tool_done, sent, done), (sent, done, tool_started, tool_done), (done, tool_done)): + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + for event in order: + graph.apply(normalizer.process(event, kind="event")) + assert [i for (t, i) in graph.nodes if t == "ToolCall"] == [message_id] + assert [i for (t, i) in graph.nodes if t == "Tool"] == ["list_schedule_documents"] + # one initial and one final State: whichever event came first set them + assert sum(1 for e in graph.edges.values() if e["edge_type"] == "hasInitialState" and e["from_id"] == message_id) == 1 + assert sum(1 for e in graph.edges.values() if e["edge_type"] == "hasFinalState" and e["from_id"] == message_id) == 1 + + +def test_mcp_message_received_is_handled_like_sent() -> None: + from norm import InMemoryGraph, StreamNormalizer + + call_input = '{"arguments": {}, "name": "list_schedule_documents"}' + sent = _mcp_message_sent_event(call_input) + received = _mcp_message_sent_event(call_input) + received["SpanId"] = received["LogAttributes"]["span.id"] = "787755dd67a73066" # the server side's own span + received["LogAttributes"]["event.name"] = "mcp.message.received" + for event in (sent, received): + event["LogAttributes"]["message.id"] = "b74c5046d0194ed7b005eee95857a563" + del event["LogAttributes"]["application_id"] # as on the real received events + + alone = InMemoryGraph() + alone.apply(StreamNormalizer(alone).process(received, kind="event")) + assert sorted(t for t, _ in alone.nodes) == [ + "Agent", "AgentCall", "MAS", "MASCall", "Session", "State", "Tool", "ToolCall", + ] + assert ("MAS", "noa-trip-planner-mas") in alone.nodes # named by session.id's first part + + # sent and received of one message: a single ToolCall + both = InMemoryGraph() + normalizer = StreamNormalizer(both) + for event in (sent, received): + both.apply(normalizer.process(event, kind="event")) + assert [i for t, i in both.nodes if t == "ToolCall"] == ["b74c5046d0194ed7b005eee95857a563"] + assert sorted(t for t, _ in both.nodes) == sorted(t for t, _ in alone.nodes) + assert normalizer.process(received, kind="event") == StreamDelta() + + # tools/list (empty input) is ignored + listing = _mcp_message_sent_event("{}") + listing["LogAttributes"]["event.name"] = "mcp.message.received" + assert StreamNormalizer(InMemoryGraph()).process(listing, kind="event") == StreamDelta() + + +def _edge_updated_event(source_span: str, target: str) -> dict: + return { + "SpanId": "4163c385d3710aac", + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": { + "application_id": "noa-trip-planner-mas", + "event.name": "topology.edge.updated", + "event.time": "2026-10-15T14:50:49.000000+00:00", + "session.id": "noa-trip-planner-mas_213dad2f", + "source.agent": "planner", + "source.agent.span.id": source_span, + "target.agent": target, + }, + } + + +def test_edge_updated_bridges_the_handoff_between_two_agent_calls() -> None: + from norm import InMemoryGraph, StreamNormalizer + + graph = InMemoryGraph() + normalizer = StreamNormalizer(graph) + first = _node_event("topology.node.started", **{"agent.name": "planner", "agent.input": "q"}) + first["SpanId"] = "aaaa" + first_done = _node_event( + "topology.node.completed", **{"agent.name": "planner", "agent.input": "q", "agent.output": "plan"} + ) + first_done["SpanId"] = "aaaa" + second = _node_event("topology.node.started", **{"agent.name": "worker", "agent.input": "do the plan"}) + second["SpanId"] = "bbbb" + second["LogAttributes"]["event.time"] = "2026-10-15T14:50:48.900000+00:00" + for event in (first, first_done, second): + graph.apply(normalizer.process(event, kind="event")) + assert "ProcessingCall" not in {t for t, _ in graph.nodes} + + # the source is found by its span id, the target by its name only + handoff = _edge_updated_event("aaaa", "worker") + graph.apply(normalizer.process(handoff, kind="event")) + assert "ProcessingCall" in {t for t, _ in graph.nodes} + assert "Processing" in {t for t, _ in graph.nodes} + assert any(e["edge_type"] == "hasProcessingCall" for e in graph.edges.values()) + assert normalizer.process(handoff, kind="event") == StreamDelta() + + # a target that is no agent with a call (an MCP server) matches nothing + assert normalizer.process(_edge_updated_event("aaaa", "debug"), kind="event") == StreamDelta() + # nor does an unknown source span + assert normalizer.process(_edge_updated_event("nope", "worker"), kind="event") == StreamDelta() diff --git a/backend/norm/tests/test_otel_io.py b/backend/norm/tests/test_otel_io.py index a49adf6..428c4a8 100644 --- a/backend/norm/tests/test_otel_io.py +++ b/backend/norm/tests/test_otel_io.py @@ -13,7 +13,7 @@ import json -from norm.ioa_observe.fields import attrs +from norm.ioa_observe.fields import get_span_attrs from norm.ioa_observe.otel_io import infer_run_id, load_otel_export @@ -82,4 +82,19 @@ def test_skips_spans_without_session_id(self) -> None: def test_reads_straight_off_span_attributes(self) -> None: span = _clickhouse_row() - assert attrs(span)["agent_id"] == "concierge_agent" + assert get_span_attrs(span)["agent_id"] == "concierge_agent" + + +def test_event_session_id_strips_application_prefix() -> None: + from norm.ioa_observe.fields import get_event_attrs, get_event_session_id + + event = { + "ServiceName": "noa-trip-planner-mas", + "LogAttributes": {"session.id": "noa-trip-planner-mas_60bf9dec-fde8-4281-a7a6-591cf903ebb1"}, + } + assert get_event_session_id(event, get_event_attrs(event)) == "60bf9dec-fde8-4281-a7a6-591cf903ebb1" + # an application_id containing "_" is stripped as a whole prefix + event = {"ServiceName": "my_app", "LogAttributes": {"session.id": "my_app_abc"}} + assert get_event_session_id(event, get_event_attrs(event)) == "abc" + # no session.id at all + assert get_event_session_id({}, {}) == "" diff --git a/backend/norm/tests/test_pipeline_functional.py b/backend/norm/tests/test_pipeline_functional.py index 7312874..0906b97 100644 --- a/backend/norm/tests/test_pipeline_functional.py +++ b/backend/norm/tests/test_pipeline_functional.py @@ -24,7 +24,7 @@ import pytest from norm import dump_jsonld, infer_run_id, load_otel_export, normalize, verify_kg -from norm.ioa_observe.fields import attrs +from norm.ioa_observe.fields import get_span_attrs FIXTURE = Path(__file__).resolve().parent / "fixtures" / "noa_trip_planner.otel_export.jsonl" @@ -81,7 +81,7 @@ def test_every_chat_span_with_model_info_produces_one_llm_call(loaded_spans) -> s for s in loaded_spans if str(s.get("SpanName") or "").endswith(".chat") - and (attrs(s).get("gen_ai.provider.name") or attrs(s).get("gen_ai.request.model")) + and (get_span_attrs(s).get("gen_ai.provider.name") or get_span_attrs(s).get("gen_ai.request.model")) ] nodes, _edges = normalize(loaded_spans) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 2d5de6f..a58e9d6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -31,6 +31,7 @@ members = [ "mce/mce-provider-ragas", "norm", "workers/norm-worker", + "workers/rt-norm-worker", "workers/embedding-worker", "workers/hierarchical-grouping-worker", "workers/mce-worker", @@ -70,6 +71,7 @@ ioc-scope-mce-provider-opik = { workspace = true } ioc-scope-mce-provider-ragas = { workspace = true } ioc-scope-norm = { workspace = true } norm-worker = { workspace = true } +rt-norm-worker = { workspace = true } hierarchical-grouping-worker = { workspace = true } mce-worker = { workspace = true } normal-behaviour-worker = { workspace = true } diff --git a/backend/uv.lock b/backend/uv.lock index f9c0fcb..13a3f64 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 5 requires-python = "==3.13.*" resolution-markers = [ "sys_platform == 'win32'", @@ -39,6 +39,7 @@ members = [ "mce-worker", "norm-worker", "normal-behaviour-worker", + "rt-norm-worker", "semantic-analysis-worker", "stateful-eval-worker", "stateful-evals", @@ -848,7 +849,7 @@ name = "cuda-bindings" version = "13.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/97/7d75105a92ee18b7764f5cf60672d6a966842348a2a049e663fe690902ff/cuda_bindings-13.4.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3f4cc887eac0d5edb5d7396cc78b336bee151a97e2ba7acf59e8b9380132d0", size = 6316532, upload-time = "2026-10-14T12:35:30.291Z" }, @@ -873,43 +874,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufft = [ - { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufile = [ - { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] curand = [ - { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] [[package]] @@ -1378,9 +1379,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/a1/e720a38852366c589e1a46cf570b886507ad2cf591050c203365638baab0/greenlet-3.5.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:f96f0e30b5a95c7631b12bfe214cbc90ec8fe8cfa36920596c10514a65743519", size = 294627, upload-time = "2026-09-14T14:24:40.102Z" }, { url = "https://files.pythonhosted.org/packages/eb/c3/58187858df41354a11e6a55b421e7af9059798abdab3a384cc51b8567c38/greenlet-3.5.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c75116c9de79949de23006e2d9b35ee82874c594fcf5c0311b439acaa14b8441", size = 614356, upload-time = "2026-09-14T15:12:03.399Z" }, { url = "https://files.pythonhosted.org/packages/ce/b9/3a7e67d5f05c9760b1ad411fa52264bd69cc08e22a2ebfb4018b90628ced/greenlet-3.5.6-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cad5782f93f7f738b62c6527b6f32a60694d924029f299a8b524758cfa53d815", size = 626756, upload-time = "2026-09-14T15:20:44.269Z" }, - { url = "https://files.pythonhosted.org/packages/c6/7c/40400455f5b5a65bb83e94fde66d1be9e5ec518638113f8083ace746c309/greenlet-3.5.6-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a93ee7c6e8fd0f8a83525a51bd777be57ee17787e91d805bd8d6faf9dcada18e", size = 632632, upload-time = "2026-09-14T15:25:07.813Z" }, { url = "https://files.pythonhosted.org/packages/85/cb/ab0c123c514ed4e94c0dc9ee2e86362633e6b998cfc05de7fc9ac2eb9690/greenlet-3.5.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f98e8215e172f567ce80eeaed9107fb4d32b6c44f26983d9b8334658136a205a", size = 623779, upload-time = "2026-09-14T14:36:01.104Z" }, - { url = "https://files.pythonhosted.org/packages/f9/67/1f35cff30a6c51c3f23b63d4afcc7313ab4f97490ba3676fa78178984b27/greenlet-3.5.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7f731ebac68ea06d628658295cb2d217b10186329fcf9a3b6a149045059bf92e", size = 434933, upload-time = "2026-09-14T15:28:38.858Z" }, { url = "https://files.pythonhosted.org/packages/a5/26/fda8a5a06e7073333ccb038133c5893b9e0c4fe29d5992a17e83c241bc6e/greenlet-3.5.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df19e2d0b1620039af5102563fbd96e8938c7f5c3f5828528d641d9fc585525e", size = 1584930, upload-time = "2026-09-14T15:10:08.234Z" }, { url = "https://files.pythonhosted.org/packages/2f/37/50f8813163148d6234e08b23dcad6a9e37f01d148c8ec976e4c44ea2d918/greenlet-3.5.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06c0e933290fba8ffe53ead4ae1b8044b0e9754b75cebf381aa2bc3e50d82fac", size = 1647590, upload-time = "2026-09-14T14:35:51.173Z" }, { url = "https://files.pythonhosted.org/packages/86/da/b7669b09586365654083a62bd0724cf06cb74bd5085a15cdd161271f992f/greenlet-3.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:5b602b4201b965a8354d74e232364a66ff243dd142e350d035f46169bb36e13d", size = 324086, upload-time = "2026-09-14T14:23:48.428Z" }, @@ -1561,8 +1560,8 @@ name = "httpcore2" version = "2.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11", marker = "sys_platform != 'emscripten'" }, - { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "h11" }, + { name = "truststore" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/61/afa9d6b7452f9eca73f73d45324eb0bb52f098fd5e6049bf6b8263be9b4e/httpcore2-2.16.0.tar.gz", hash = "sha256:c72cd0413563b17e1cd8354677dff887466f8ec2551679759e93126534428643", size = 69886, upload-time = "2026-10-21T07:09:45.31Z" } wheels = [ @@ -1919,7 +1918,7 @@ dev = [ [[package]] name = "ioc-scope-mce" -version = "2.0.0rc1" +version = "2.0.0" source = { editable = "mce/mce-meta" } dependencies = [ { name = "ioc-scope-mce-client" }, @@ -1948,7 +1947,7 @@ requires-dist = [ [[package]] name = "ioc-scope-mce-client" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-client" } dependencies = [ { name = "click" }, @@ -1986,7 +1985,7 @@ provides-extras = ["dev"] [[package]] name = "ioc-scope-mce-core" -version = "2.0.0rc1" +version = "2.0.0" source = { editable = "mce/mce-core" } dependencies = [ { name = "click" }, @@ -2042,7 +2041,7 @@ provides-extras = ["dev", "neo4j", "deepeval", "llm", "opik", "ragas", "server", [[package]] name = "ioc-scope-mce-helper" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-helper" } dependencies = [ { name = "ioc-scope-mce-core" }, @@ -2053,7 +2052,7 @@ requires-dist = [{ name = "ioc-scope-mce-core", editable = "mce/mce-core" }] [[package]] name = "ioc-scope-mce-legacy" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-legacy" } dependencies = [ { name = "click" }, @@ -2074,7 +2073,7 @@ requires-dist = [ [[package]] name = "ioc-scope-mce-provider-deepeval" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-provider-deepeval" } dependencies = [ { name = "deepeval" }, @@ -2093,7 +2092,7 @@ requires-dist = [ [[package]] name = "ioc-scope-mce-provider-native" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-provider-native" } dependencies = [ { name = "ioc-scope-mce-core" }, @@ -2108,7 +2107,7 @@ requires-dist = [ [[package]] name = "ioc-scope-mce-provider-opik" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-provider-opik" } dependencies = [ { name = "ioc-scope-mce-core" }, @@ -2127,7 +2126,7 @@ requires-dist = [ [[package]] name = "ioc-scope-mce-provider-ragas" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-provider-ragas" } dependencies = [ { name = "ioc-scope-mce-core" }, @@ -2150,7 +2149,7 @@ requires-dist = [ [[package]] name = "ioc-scope-mce-provider-sdk" -version = "0.1.0" +version = "2.0.0" source = { editable = "mce/mce-provider-sdk" } dependencies = [ { name = "ioc-scope-mce-core" }, @@ -3103,7 +3102,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cuda-nvrtc" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -3142,7 +3141,7 @@ name = "nvidia-cudnn-cu13" version = "9.24.0.43" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, @@ -3154,7 +3153,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3184,9 +3183,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "nvidia-cusparse", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3198,7 +3197,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -4126,6 +4125,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/72/f28ca566f6c23c35bbf7445f65eb0364577b25a026305995e24f78b83d94/rpds_py-2026.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:d151e148117294133bf8af7eeace085e7e87432db15ab6adf640330298a47f6f", size = 217681, upload-time = "2026-10-04T16:29:59.449Z" }, ] +[[package]] +name = "rt-norm-worker" +version = "1.0.0" +source = { editable = "workers/rt-norm-worker" } +dependencies = [ + { name = "click" }, + { name = "ioc-scope-api" }, + { name = "ioc-scope-norm" }, + { name = "pyyaml" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.0.0" }, + { name = "ioc-scope-api", editable = "api" }, + { name = "ioc-scope-norm", editable = "norm" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] +provides-extras = ["dev"] + [[package]] name = "ruff" version = "0.15.22" diff --git a/backend/workers/rt-norm-worker/.gitignore b/backend/workers/rt-norm-worker/.gitignore new file mode 100644 index 0000000..31c21a6 --- /dev/null +++ b/backend/workers/rt-norm-worker/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Project specific +/tmp/ +*.json.tmp +output.txt +.deepeval/ +.ruff_cache/ diff --git a/backend/workers/rt-norm-worker/README.md b/backend/workers/rt-norm-worker/README.md new file mode 100644 index 0000000..84d3d0f --- /dev/null +++ b/backend/workers/rt-norm-worker/README.md @@ -0,0 +1,189 @@ +# rt-norm-worker + +Standalone worker (no `worker-base`) that normalizes OTel **spans** and **logs**, one record at a +time, into the Neo4j knowledge graph, and calls back the URL of every rule a log record matches. + +Nothing is kept between records: each is normalized against the graph as stored. What each log +event creates is in [`backend/norm/docs/events.md`](../../norm/docs/events.md). + +All commands run from `backend/`. + +## Setup + +Neo4j must be running. The worker reads its connection from environment variables (there is no +`.env` file in this repo): `NEO4J_URI` (or `NEO4J_HOST` and `NEO4J_PORT`), `NEO4J_USERNAME`, +`NEO4J_PASSWORD` and, optionally, `NEO4J_DATABASE`. + +```bash +cd backend +export NEO4J_URI=bolt://localhost:7687 NEO4J_USERNAME=neo4j NEO4J_PASSWORD=... +``` + +Not needed with `--dry-run`. + +## Run + +The CLI is `rt-norm-worker-cli` (or `python -m rt_norm_worker.cli`). + +### From a file + +```bash +# log records (events) +uv run --package rt-norm-worker rt-norm-worker-cli --input ~/claris/otel_logs.json --kind event + +# spans (the default kind) +uv run --package rt-norm-worker rt-norm-worker-cli --input ~/claris/otel_traces.json --kind span +``` + +### Without the rules (no callbacks) + +```bash +uv run --package rt-norm-worker rt-norm-worker-cli --no-rules --input ~/claris/otel_logs.json --kind event +``` + +### Dry run (in-memory graph, nothing written to Neo4j) + +```bash +uv run --package rt-norm-worker rt-norm-worker-cli --dry-run --no-rules --input ~/claris/otel_logs.json --kind event +``` + +### From the OTel collector + +```bash +uv run --package rt-norm-worker rt-norm-worker-cli --serve # 0.0.0.0:4318 +uv run --package rt-norm-worker rt-norm-worker-cli --serve --host 127.0.0.1 --port 4318 +``` + +### Options + +| Option | Default | Meaning | +|---|---|---| +| `--input FILE` | `~/claris/otel_traces.json` | JSON array of spans or log records to read | +| `--kind span\|event` | `span` | what each record of the input is | +| `--rules FILE` | `rules.yaml` next to `pyproject.toml` | rules loaded into memory at startup | +| `--no-rules` | off | do not load the rules; no callbacks | +| `--serve` | off | receive records from the collector instead of reading `--input` | +| `--host`, `--port` | `0.0.0.0`, `4318` | where `--serve` listens | +| `--dry-run` | off | throwaway in-memory graph instead of Neo4j | + +## Receiving from the collector (`--serve`) + +The worker is an OTLP/HTTP receiver. Only **OTLP/JSON** is accepted (gzip is fine), so the +collector's exporter needs `encoding: json`: + +```yaml +exporters: + otlphttp/rt-norm: + endpoint: http://rt-norm-worker:4318 + encoding: json +service: + pipelines: + traces: + exporters: [clickhouse, otlphttp/rt-norm] + logs: + exporters: [clickhouse, otlphttp/rt-norm] +``` + +| Request | Does | +|---|---| +| `POST /v1/traces` | normalizes the spans | +| `POST /v1/logs` | normalizes the log records, then runs the rules | +| `GET /healthz` | liveness, `200` | + +`200` once every record of the request is in the graph; `400` malformed JSON; `415` not JSON; +`500` normalization failed (the collector retries). Requests are processed one at a time. + +```bash +# try it by hand +curl -s -X POST localhost:4318/v1/logs -H 'Content-Type: application/json' -d @logs-request.json +curl -s localhost:4318/healthz +``` + +## Rules and callbacks + +A rule is a `name`, a `callback_url` and, for the rules that take one, a `value`. What a name +selects is built into the worker. After a log record is normalized, it is checked against every +rule; for each match the worker **POSTs** to the rule's `callback_url`: + +```json +{"rule": "agent_events", "value": "researcher", "event": { "...the log record..." }} +``` + +Calls run in the background (5 s timeout); a failure is logged, not retried. Spans never call back. + +| Rule | Matches when (on `LogAttributes`) | +|---|---| +| `all` | always | +| `tool_events` / `llm_events` | `event.name` starts with `tool.` / `llm.` | +| `tool_completed` / `llm_completed` | `event.name` is `tool.completed` / `llm.completed` | +| `mcp_traffic` | `network.protocol.name` is `mcp` | +| `agent_handoffs` | `topology.edge.kind` is `agent_handoff` and `source.agent` ≠ `target.agent` | +| `agent_events` (`value`) | `value` is `agent.name`, `source.agent` or `target.agent` | +| `service_events` (`value`) | `value` is the record's `ServiceName` | + +### `rules.yaml` + +See [`rules.example.yaml`](rules.example.yaml). A rule that takes a value can be listed once per value. + +```yaml +rules: + - name: all + callback_url: https://audit.example.com/hooks/events + - name: agent_events + value: researcher + callback_url: https://agents.example.com/hooks/researcher +``` + +### Managing the rules while `--serve` runs + +Each rule has a key: its name, or `name=value`. Rules added this way live only in memory; a +restart goes back to the file. + +```bash +curl -s localhost:4318/rules # list +curl -s -X POST localhost:4318/rules -H 'Content-Type: application/json' \ + -d '{"name": "agent_events", "value": "planner", "callback_url": "http://localhost:9000/planner"}' +curl -s -X POST localhost:4318/rules -H 'Content-Type: application/json' \ + -d '{"rules": [{"name": "llm_events", "callback_url": "http://localhost:9000/llm"}]}' # several +curl -s -X DELETE 'localhost:4318/rules/agent_events%3Dplanner' # remove (key, URL-encoded) +``` + +## Neo4j + +```bash +# wipe the graph (deletes everything!) +uv run --package rt-norm-worker python -I - <<'PY' +from scope.dependencies import get_neo4j_connector +c = get_neo4j_connector() +c.execute_command("MATCH (n) DETACH DELETE n") +c.close() +PY + +# count the nodes by label +uv run --package rt-norm-worker python -I - <<'PY' +from scope.dependencies import get_neo4j_connector +c = get_neo4j_connector() +for r in c.execute("MATCH (n) RETURN labels(n)[0] AS l, count(*) AS n ORDER BY n DESC"): + print(r["l"], r["n"]) +c.close() +PY +``` + +## Tests + +```bash +uv run --package rt-norm-worker --with pytest pytest workers/rt-norm-worker/tests -q # the worker +uv run --package ioc-scope-norm --extra dev pytest norm/tests -q # the normalization +``` + +## Code + +| File | What | +|---|---| +| `cli.py` | command line | +| `worker.py` | reads files; `process_record` normalizes one record, then notifies | +| `server.py` | OTLP/HTTP receiver and the `/rules` endpoints | +| `otlp.py` | flattens OTLP/JSON into the rows the files hold | +| `rules.py` | the rules in memory (`RuleSet`) and how each one matches | +| `callbacks.py` | POSTs to the callback URLs | +| `store.py` | Neo4j read/write | diff --git a/backend/workers/rt-norm-worker/pyproject.toml b/backend/workers/rt-norm-worker/pyproject.toml new file mode 100644 index 0000000..608eaf0 --- /dev/null +++ b/backend/workers/rt-norm-worker/pyproject.toml @@ -0,0 +1,35 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "rt-norm-worker" +version = "1.0.0" +authors = [ + { name = "Outshift Open", email = "oss@cisco.com" } +] +license = "Apache-2.0" +description = "Standalone real-time normalization worker (does not use worker-base)" +readme = "README.md" +requires-python = ">=3.13" +dependencies = [ + "click>=8.0.0", + "ioc-scope-norm", + "ioc-scope-api", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = ["pytest>=7.0.0"] + +[project.scripts] +rt-norm-worker-cli = "rt_norm_worker.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/rt_norm_worker"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/backend/workers/rt-norm-worker/rules.example.yaml b/backend/workers/rt-norm-worker/rules.example.yaml new file mode 100644 index 0000000..507b4dc --- /dev/null +++ b/backend/workers/rt-norm-worker/rules.example.yaml @@ -0,0 +1,28 @@ +# Example rules file. Copy it to rules.yaml (or pass it with --rules). +# +# name which rule: all, tool_events, tool_completed, llm_events, llm_completed, +# mcp_traffic, agent_handoffs, service_events, agent_events +# callback_url URL to call back when an event matches +# value only for service_events and agent_events: the service / agent to match +# +# A rule that takes a value can be listed once per value. +rules: + - name: all + callback_url: https://audit.example.com/hooks/events + - name: tool_completed + callback_url: https://tools.example.com/hooks/tool-completed + - name: llm_completed + callback_url: https://llm.example.com/hooks/llm-completed + - name: mcp_traffic + callback_url: https://mcp.example.com/hooks/traffic + - name: agent_handoffs + callback_url: https://topology.example.com/hooks/handoffs + - name: agent_events + value: researcher + callback_url: https://agents.example.com/hooks/researcher + - name: agent_events + value: planner + callback_url: https://agents.example.com/hooks/planner + - name: service_events + value: weather-agent + callback_url: https://services.example.com/hooks/weather-agent diff --git a/backend/workers/rt-norm-worker/rules.yaml b/backend/workers/rt-norm-worker/rules.yaml new file mode 100644 index 0000000..90b434e --- /dev/null +++ b/backend/workers/rt-norm-worker/rules.yaml @@ -0,0 +1,24 @@ +# Rules of the rt-norm-worker, loaded into memory at startup. Each one is a rule name (its +# meaning is built into the worker, see RULE_TYPES in rules.py), the URL to call back and, +# for the rules that take one, the value to compare with. The URLs are placeholders. +rules: + - name: all + callback_url: http://localhost:8080/callbacks/all + - name: tool_events + callback_url: http://localhost:8080/callbacks/tool_events + - name: tool_completed + callback_url: http://localhost:8080/callbacks/tool_completed + - name: llm_events + callback_url: http://localhost:8080/callbacks/llm_events + - name: llm_completed + callback_url: http://localhost:8080/callbacks/llm_completed + - name: mcp_traffic + callback_url: http://localhost:8080/callbacks/mcp_traffic + - name: agent_handoffs + callback_url: http://localhost:8080/callbacks/agent_handoffs + - name: service_events + value: weather-agent + callback_url: http://localhost:8080/callbacks/service_events + - name: agent_events + value: researcher + callback_url: http://localhost:8080/callbacks/agent_events diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/__init__.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/__init__.py new file mode 100644 index 0000000..d494ce9 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/callbacks.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/callbacks.py new file mode 100644 index 0000000..4cfb278 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/callbacks.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Call the rules' callback URLs for the log records that match them.""" + +from __future__ import annotations + +import json +import logging +import urllib.request +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from rt_norm_worker.rules import Rule, RuleSet + +logger = logging.getLogger(__name__) + +TIMEOUT_SECONDS = 5 + + +def post_json(url: str, payload: dict[str, Any]) -> None: + """POST the payload as JSON. Raises on a connection error or a non-2xx answer.""" + request = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS): + pass + + +class Notifier: + """After a log record is normalized, POST it to the callback URL of every rule it matches. + + The body is ``{"rule": name, "value": value, "event": }``. Calls run on + a small thread pool, so a slow or dead URL never holds up normalization; a failure is + logged and not retried. ``post`` is replaceable (tests pass a recorder; ``wait=True`` + makes the calls synchronous). + """ + + def __init__( + self, + rules: RuleSet, + post: Callable[[str, dict[str, Any]], None] = post_json, + wait: bool = False, + workers: int = 4, + ) -> None: + self.rules = rules + self._post = post + self._wait = wait + self._pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="rt-norm-callback") + + def matching(self, record: dict[str, Any]) -> list[Rule]: + return [rule for rule in self.rules if rule.matches(record)] + + def notify(self, record: dict[str, Any]) -> list[Rule]: + """Call back for every rule the record matches. Returns those rules.""" + matched = self.matching(record) + for rule in matched: + payload = {"rule": rule.name, "value": rule.value, "event": record} + if self._wait: + self._call(rule, payload) + else: + self._pool.submit(self._call, rule, payload) + return matched + + def _call(self, rule: Rule, payload: dict[str, Any]) -> None: + try: + self._post(rule.callback_url, payload) + except Exception as error: + logger.warning("callback of rule %s to %s failed: %s", rule.key, rule.callback_url, error) + + def close(self) -> None: + self._pool.shutdown(wait=True) diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/cli.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/cli.py new file mode 100644 index 0000000..15ba6c4 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/cli.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +import click +from norm import InMemoryGraph + +from rt_norm_worker import worker +from rt_norm_worker.rules import DEFAULT_RULES_PATH, load_rules + + +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--input", + "input_path", + type=click.Path(dir_okay=False), + default=str(worker.DEFAULT_TRACES_PATH), + show_default=True, + help="OTel traces JSON file to read", +) +@click.option( + "--kind", + type=click.Choice(["span", "event"]), + default="span", + show_default=True, + help="What each line of the input is: an OTel span (span) or an OTel log record (event)", +) +@click.option( + "--rules", + "rules_path", + type=click.Path(dir_okay=False, exists=True), + default=str(DEFAULT_RULES_PATH), + show_default=True, + help="Rules file (name, callback_url[, value]), loaded into memory at startup", +) +@click.option( + "--no-rules", + is_flag=True, + default=False, + help="Do not load the rules: no callbacks are made", +) +@click.option( + "--serve", + is_flag=True, + default=False, + help="Receive spans and logs from the OTel collector (OTLP/HTTP, JSON) instead of reading --input", +) +@click.option("--host", default="0.0.0.0", show_default=True, help="Address to listen on with --serve") +@click.option("--port", type=int, default=4318, show_default=True, help="Port to listen on with --serve") +@click.option("--dry-run", is_flag=True, default=False, help="Use a throwaway in-memory graph instead of Neo4j") +def main(input_path: str, kind: str, rules_path: str, no_rules: bool, serve: bool, host: str, port: int, dry_run: bool): + """Normalize OTel spans and logs one by one into Neo4j. + + Reads them from a file (--input, --kind), or with --serve receives them from the OTel + collector's ``otlphttp`` exporter (``encoding: json``) on /v1/traces and /v1/logs.""" + rules = None + if not no_rules: + rules = load_rules(rules_path) + click.echo(f"Loaded {len(rules)} rules from {rules_path}: {', '.join(rules.keys())}", err=True) + if dry_run: + graph = InMemoryGraph() + else: + from scope.dependencies import get_neo4j_connector + + from rt_norm_worker.store import Neo4jStore + + graph = Neo4jStore(get_neo4j_connector()) + if serve: + from rt_norm_worker.server import serve as run_server + + run_server(graph, host, port, rules) + else: + worker.run(graph, input_path, kind, rules) + + +if __name__ == "__main__": + main() diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/otlp.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/otlp.py new file mode 100644 index 0000000..ba01916 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/otlp.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Flatten OTLP/JSON export requests into the records ``norm`` normalizes. + +The OTel collector's ``otlphttp`` exporter (``encoding: json``) sends nested +``resourceSpans`` / ``resourceLogs``. ``norm`` reads the flat rows the collector's ClickHouse +exporter writes (and that the input files hold): ``SpanName``, ``SpanAttributes``, +``LogAttributes``, ``ServiceName``, ``Timestamp``, ``Duration``, ... This module does that +conversion, one record per span / log record. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import UTC, datetime +from typing import Any + + +def _value(value: dict[str, Any]) -> str: + """An OTLP ``AnyValue`` as the string the ClickHouse exporter stores.""" + if "stringValue" in value: + return str(value["stringValue"]) + if "boolValue" in value: + return "true" if value["boolValue"] else "false" + if "intValue" in value: + return str(value["intValue"]) + if "doubleValue" in value: + return str(value["doubleValue"]) + if "arrayValue" in value: + items = value["arrayValue"].get("values") or [] + return "[" + ", ".join(_value(item) for item in items) + "]" + if "kvlistValue" in value: + return str(_attributes(value["kvlistValue"].get("values"))) + if "bytesValue" in value: + return str(value["bytesValue"]) + return "" + + +def _attributes(attributes: list[dict[str, Any]] | None) -> dict[str, str]: + return {str(a["key"]): _value(a.get("value") or {}) for a in attributes or [] if "key" in a} + + +def _timestamp(unix_nano: Any) -> str: + """``YYYY-MM-DD HH:MM:SS.nnnnnnnnn`` (UTC), the format of the ClickHouse ``Timestamp`` column.""" + nanos = int(unix_nano or 0) + seconds, frac = divmod(nanos, 1_000_000_000) + return f"{datetime.fromtimestamp(seconds, UTC):%Y-%m-%d %H:%M:%S}.{frac:09d}" + + +def _enum(value: Any, prefix: str) -> str: + """``SPAN_KIND_INTERNAL`` -> ``Internal`` (JSON may also carry the enum's number).""" + if isinstance(value, str) and value.startswith(prefix): + return value[len(prefix) :].capitalize() + return str(value or "") + + +_SPAN_KINDS = {0: "Unspecified", 1: "Internal", 2: "Server", 3: "Client", 4: "Producer", 5: "Consumer"} +_STATUS_CODES = {0: "Unset", 1: "Ok", 2: "Error"} + + +def _resource(resource_item: dict[str, Any]) -> tuple[dict[str, str], str]: + resource = _attributes((resource_item.get("resource") or {}).get("attributes")) + return resource, resource.get("service.name", "") + + +def spans(request: dict[str, Any]) -> Iterator[dict[str, Any]]: + """The spans of an OTLP ``ExportTraceServiceRequest`` as flat records.""" + for resource_item in request.get("resourceSpans") or []: + resource, service = _resource(resource_item) + for scope_item in resource_item.get("scopeSpans") or []: + scope = scope_item.get("scope") or {} + for span in scope_item.get("spans") or []: + start = int(span.get("startTimeUnixNano") or 0) + end = int(span.get("endTimeUnixNano") or 0) + kind = span.get("kind", "") + status = span.get("status") or {} + code = status.get("code", "") + yield { + "Timestamp": _timestamp(start), + "TraceId": span.get("traceId", ""), + "SpanId": span.get("spanId", ""), + "ParentSpanId": span.get("parentSpanId", ""), + "TraceState": span.get("traceState", ""), + "SpanName": span.get("name", ""), + "SpanKind": _SPAN_KINDS.get(kind, "") if isinstance(kind, int) else _enum(kind, "SPAN_KIND_"), + "ServiceName": service, + "ResourceAttributes": resource, + "ScopeName": scope.get("name", ""), + "ScopeVersion": scope.get("version", ""), + "SpanAttributes": _attributes(span.get("attributes")), + "Duration": str(max(end - start, 0)), + "StatusCode": _STATUS_CODES.get(code, "") if isinstance(code, int) else _enum(code, "STATUS_CODE_"), + "StatusMessage": status.get("message", ""), + } + + +def logs(request: dict[str, Any]) -> Iterator[dict[str, Any]]: + """The log records of an OTLP ``ExportLogsServiceRequest`` as flat records.""" + for resource_item in request.get("resourceLogs") or []: + resource, service = _resource(resource_item) + for scope_item in resource_item.get("scopeLogs") or []: + scope = scope_item.get("scope") or {} + for record in scope_item.get("logRecords") or []: + time = record.get("timeUnixNano") or record.get("observedTimeUnixNano") + yield { + "Timestamp": _timestamp(time), + "TraceId": record.get("traceId", ""), + "SpanId": record.get("spanId", ""), + "SeverityText": record.get("severityText", ""), + "SeverityNumber": record.get("severityNumber", 0), + "ServiceName": service, + "Body": _value(record.get("body") or {}), + "ResourceAttributes": resource, + "ScopeName": scope.get("name", ""), + "ScopeVersion": scope.get("version", ""), + "LogAttributes": _attributes(record.get("attributes")), + } diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/rules.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/rules.py new file mode 100644 index 0000000..e1318df --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/rules.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""The worker's rules: which events to call back about, and where. + +What a rule *selects* is fixed by its ``name`` (see ``RULE_TYPES``); the file or the request +only says which rules are on, their ``callback_url`` and, for the rules that take one, the +``value`` they compare against. For example ``agent_events`` with ``value: researcher`` stands +for "events for the agent called researcher", and the same rule can be added once per agent. + +Rules live in a ``RuleSet`` in the memory of the application. They are loaded from +``rules.yaml`` at startup and can be added or removed while it runs (``/rules``, see server.py). +""" + +from __future__ import annotations + +import json +import threading +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +DEFAULT_RULES_PATH = Path(__file__).resolve().parents[2] / "rules.yaml" + +# name -> (what it selects, takes a value). How each one matches is in _MATCHERS below. +RULE_TYPES: dict[str, tuple[str, bool]] = { + "all": ("Every event", False), + "tool_events": ("All tool events", False), + "tool_completed": ("Completed tool calls", False), + "llm_events": ("All LLM events", False), + "llm_completed": ("Completed LLM calls", False), + "mcp_traffic": ("MCP traffic", False), + "agent_handoffs": ("Agent handoffs only (excluding self-loops)", False), + "service_events": ("Events from the service called `value`", True), + "agent_events": ("Events for the agent called `value`", True), +} + + +def log_attributes(record: dict[str, Any]) -> dict[str, Any]: + """The record's ``LogAttributes`` map (it may also arrive as a JSON string).""" + attributes = record.get("LogAttributes") + if isinstance(attributes, str): + try: + attributes = json.loads(attributes) + except ValueError: + return {} + return attributes if isinstance(attributes, dict) else {} + + +def _event_name(attrs: dict[str, Any]) -> str: + return str(attrs.get("event.name") or "") + + +# name -> does the log record match? ``attrs`` is its LogAttributes, ``value`` the rule's value. +_MATCHERS: dict[str, Callable[[dict[str, Any], dict[str, Any], str], bool]] = { + "all": lambda record, attrs, value: True, + "tool_events": lambda record, attrs, value: _event_name(attrs).startswith("tool."), + "tool_completed": lambda record, attrs, value: _event_name(attrs) == "tool.completed", + "llm_events": lambda record, attrs, value: _event_name(attrs).startswith("llm."), + "llm_completed": lambda record, attrs, value: _event_name(attrs) == "llm.completed", + "mcp_traffic": lambda record, attrs, value: attrs.get("network.protocol.name") == "mcp", + "agent_handoffs": lambda record, attrs, value: ( + attrs.get("topology.edge.kind") == "agent_handoff" + and attrs.get("source.agent") != attrs.get("target.agent") + ), + "service_events": lambda record, attrs, value: str(record.get("ServiceName") or "") == value, + "agent_events": lambda record, attrs, value: value + in (attrs.get("agent.name"), attrs.get("source.agent"), attrs.get("target.agent")), +} + + +@dataclass(frozen=True) +class Rule: + name: str + callback_url: str + value: str = "" + + @property + def key(self) -> str: + """Identifies the rule: its name, plus the value for a rule that takes one.""" + return f"{self.name}={self.value}" if self.value else self.name + + def matches(self, record: dict[str, Any]) -> bool: + """Does this log record (a flat row with ``LogAttributes``) match the rule?""" + return _MATCHERS[self.name](record, log_attributes(record), self.value) + + def as_dict(self) -> dict[str, str]: + data = {"name": self.name, "callback_url": self.callback_url} + if self.value: + data["value"] = self.value + return data + + +def rule_from_dict(item: Any) -> Rule: + """Build a rule from ``{name, callback_url[, value]}``, checking it against ``RULE_TYPES``.""" + if not isinstance(item, dict): + raise ValueError(f"A rule is an object with a name and a callback_url: {item!r}") + name = str(item.get("name") or "") + callback_url = str(item.get("callback_url") or "") + value = str(item.get("value") or "") + if name not in RULE_TYPES: + raise ValueError(f"Unknown rule {name!r}, expected one of {', '.join(RULE_TYPES)}") + if not callback_url: + raise ValueError(f"Rule {name!r} needs a callback_url") + takes_value = RULE_TYPES[name][1] + if takes_value and not value: + raise ValueError(f"Rule {name!r} needs a value") + if value and not takes_value: + raise ValueError(f"Rule {name!r} takes no value") + return Rule(name=name, callback_url=callback_url, value=value) + + +class RuleSet: + """The rules by key, in the order they were added. Safe to use from several threads.""" + + def __init__(self, rules: list[Rule] | None = None) -> None: + self._lock = threading.Lock() + self._rules: dict[str, Rule] = {} + for rule in rules or []: + self.add(rule) + + def add(self, rule: Rule) -> bool: + """Add the rule, or replace the one with the same key (its callback URL changes). + True if it is new.""" + with self._lock: + new = rule.key not in self._rules + self._rules[rule.key] = rule + return new + + def remove(self, key: str) -> bool: + with self._lock: + return self._rules.pop(key, None) is not None + + def get(self, key: str) -> Rule | None: + with self._lock: + return self._rules.get(key) + + def __iter__(self) -> Iterator[Rule]: + with self._lock: + return iter(list(self._rules.values())) + + def __len__(self) -> int: + with self._lock: + return len(self._rules) + + def __contains__(self, key: object) -> bool: + with self._lock: + return key in self._rules + + def keys(self) -> list[str]: + with self._lock: + return list(self._rules) + + +def load_rules(path: Path = DEFAULT_RULES_PATH) -> RuleSet: + """Read a rules file: a ``rules`` list of ``{name, callback_url[, value]}``.""" + with open(Path(path).expanduser(), encoding="utf-8") as f: + content: Any = yaml.safe_load(f) + items = content.get("rules") if isinstance(content, dict) else None + if not isinstance(items, list): + raise ValueError(f"Expected a 'rules' list in {path}") + rules = RuleSet() + for item in items: + rule = rule_from_dict(item) + if rule.key in rules: + raise ValueError(f"Duplicate rule {rule.key!r} in {path}") + rules.add(rule) + return rules diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/server.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/server.py new file mode 100644 index 0000000..1042374 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/server.py @@ -0,0 +1,134 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""OTLP/HTTP receiver: the OTel collector pushes spans and logs here, one request at a time. + +Point the collector's ``otlphttp`` exporter at it with ``encoding: json``:: + + exporters: + otlphttp/rt-norm: + endpoint: http://rt-norm-worker:4318 + encoding: json + +``POST /v1/traces`` and ``POST /v1/logs`` are served, and only OTLP/JSON (optionally +gzip-compressed). Every span / log record of a request is normalized, in order, into the graph. + +The rules in memory are managed on ``/rules``: ``GET /rules`` lists them, ``POST /rules`` adds +one (``{"name": ..., "callback_url": ..., "value": ...}``, or ``{"rules": [...]}``) and +``DELETE /rules/`` removes one (``agent_events=researcher``, or just ``all``). +""" + +from __future__ import annotations + +import gzip +import json +import logging +import threading +from urllib.parse import unquote +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +from norm import Kind + +from rt_norm_worker import otlp +from rt_norm_worker.callbacks import Notifier +from rt_norm_worker.rules import RuleSet, rule_from_dict +from rt_norm_worker.worker import Graph, process_record + +logger = logging.getLogger(__name__) + +_ROUTES: dict[str, tuple[Kind, Any]] = { + "/v1/traces": ("span", otlp.spans), + "/v1/logs": ("event", otlp.logs), +} + + +def make_server( + graph: Graph, + host: str = "0.0.0.0", + port: int = 4318, + rules: RuleSet | None = None, + notifier: Notifier | None = None, +) -> ThreadingHTTPServer: + rules = rules if rules is not None else RuleSet() + notifier = notifier or Notifier(rules) + # Records are normalized against the graph as stored, so two requests must not interleave. + lock = threading.Lock() + + class Handler(BaseHTTPRequestHandler): + def _reply(self, status: int, body: dict[str, Any] | None = None) -> None: + payload = json.dumps(body or {}).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: # health check + if self.path == "/rules": + return self._reply(200, {"rules": [rule.as_dict() | {"key": rule.key} for rule in rules]}) + self._reply(200 if self.path in ("/", "/healthz") else 404) + + def do_DELETE(self) -> None: + if not self.path.startswith("/rules/"): + return self._reply(404, {"error": f"unknown path {self.path}"}) + removed = rules.remove(unquote(self.path[len("/rules/") :])) + self._reply(200 if removed else 404, {} if removed else {"error": "no such rule"}) + + def _add_rules(self) -> None: + try: + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + items = body.get("rules") if isinstance(body, dict) and "rules" in body else [body] + new = [rule_from_dict(item) for item in items] + except ValueError as error: + return self._reply(400, {"error": str(error)}) + for rule in new: + rules.add(rule) + self._reply(201, {"rules": [rule.key for rule in new]}) + + def do_POST(self) -> None: + if self.path == "/rules": + return self._add_rules() + route = _ROUTES.get(self.path.split("?")[0]) + if route is None: + return self._reply(404, {"error": f"unknown path {self.path}"}) + if "json" not in self.headers.get("Content-Type", ""): + return self._reply( + 415, {"error": "only OTLP/JSON is supported: set `encoding: json` on the exporter"} + ) + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + try: + if self.headers.get("Content-Encoding") == "gzip": + body = gzip.decompress(body) + request = json.loads(body or b"{}") + except (OSError, ValueError) as error: + return self._reply(400, {"error": f"invalid request: {error}"}) + + kind, flatten = route + try: + with lock: + for record in flatten(request): + process_record(graph, record, kind, notifier) + except Exception: # the collector retries on 5xx + logger.exception("failed to normalize %s request", kind) + return self._reply(500, {"error": "normalization failed"}) + self._reply(200) + + def log_message(self, format: str, *args: Any) -> None: + logger.debug(format, *args) + + server = ThreadingHTTPServer((host, port), Handler) + server.rules = rules # type: ignore[attr-defined] + return server + + +def serve( + graph: Graph, host: str = "0.0.0.0", port: int = 4318, rules: RuleSet | None = None +) -> None: + server = make_server(graph, host, port, rules) + logger.info("rt-norm-worker listening on %s:%s", host, port) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py new file mode 100644 index 0000000..dcde9a6 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +"""Writes one span's KG delta straight to Neo4j, updating nodes in place.""" + +import logging +import json +from collections.abc import Sequence +from typing import Any + +from norm import StreamDelta + +logger = logging.getLogger(__name__) + +# Node types merged on `name` (unique constraint) instead of `id`; mirrors scope.client.dal. +NAME_KEYED_NODE_TYPES = frozenset({"LLM"}) +# Set when a node is first created and never changed afterwards (endTime is not: it advances +# as a container, e.g. an AgentCall or Session, completes). +IMMUTABLE_PROPS = frozenset({"startTime"}) +_NON_PROPS = frozenset({"node_type"}) +_EDGE_NON_PROPS = frozenset({"edge_type", "from_id", "from_type", "to_id", "to_type"}) + + +def _safe(identifier: str) -> str: + from scope.client.dal import _is_safe_cypher_identifier + + if not _is_safe_cypher_identifier(identifier): + raise ValueError(f"Unsafe Cypher identifier: {identifier!r}") + return identifier + + +def _props(values: dict[str, Any], skip: frozenset[str]) -> dict[str, Any]: + from scope.client.dal import _neo4j_props + + return _neo4j_props({k: v for k, v in values.items() if k not in skip}) + + +def _decode(value: Any) -> Any: + """Undo ``scope.client.dal._neo4j_props``'s JSON-prefix encoding of dict/list values.""" + from scope.client.dal import _JSON_PREFIX + + if isinstance(value, str) and value.startswith(_JSON_PREFIX): + return json.loads(value[len(_JSON_PREFIX) :]) + return value + + +def _node(label: str, props: dict[str, Any]) -> dict[str, Any]: + return {**{k: _decode(v) for k, v in props.items()}, "node_type": label} + + +class Neo4jStore: + """Neo4j implementation of norm's ``GraphReader``, plus writer for its deltas.""" + + def __init__(self, connector: Any): + self._db = connector + + def load( + self, session_id: str, node_ids: Sequence[str] + ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Read the span's session subgraph and the nodes it touches from Neo4j.""" + params = {"sid": session_id, "ids": list(node_ids)} + nodes: dict[tuple[str, str], dict[str, Any]] = {} + for row in self._db.execute( + "MATCH (n) WHERE n.sessionId = $sid OR n.id IN $ids " + "OR (n:Agent AND n.name IN $ids) " + "RETURN labels(n)[0] AS label, properties(n) AS props", + params, + ): + node = _node(row["label"], row["props"]) + nodes[(node["node_type"], node["id"])] = node + + edges: list[dict[str, Any]] = [] + for row in self._db.execute( + "MATCH (a)-[r]->(b) " + "WHERE a.sessionId = $sid OR b.sessionId = $sid " + "OR (a:Agent AND a.name IN $ids) OR (b:Agent AND b.name IN $ids) " + "OR (a.id IN $ids AND b.id IN $ids) " + "RETURN type(r) AS edge_type, a.id AS from_id, b.id AS to_id, " + "labels(a)[0] AS from_label, properties(a) AS from_props, " + "labels(b)[0] AS to_label, properties(b) AS to_props", + params, + ): + edges.append({"edge_type": row["edge_type"], "from_id": row["from_id"], "to_id": row["to_id"]}) + for label, props in ((row["from_label"], row["from_props"]), (row["to_label"], row["to_props"])): + node = _node(label, props) + nodes.setdefault((node["node_type"], node["id"]), node) + return list(nodes.values()), edges + + def apply(self, delta: StreamDelta) -> None: + """Apply a delta: removals first, then nodes, then edges (endpoints must exist).""" + for edge in delta.removed_edges: + self._delete_edge(edge) + for node in delta.removed_nodes: + self._delete_node(node) + for node in delta.nodes: + self._upsert_node(node) + for edge in delta.edges: + self._upsert_edge(edge) + + def _upsert_node(self, node: dict[str, Any]) -> None: + node_type = _safe(str(node["node_type"])) + props = _props(node, _NON_PROPS) + on_match = {k: v for k, v in props.items() if k not in IMMUTABLE_PROPS} + key = "name" if node_type in NAME_KEYED_NODE_TYPES else "id" + self._db.execute_command( + f"MERGE (n:{node_type} {{{key}: $key}}) " + "ON CREATE SET n += $props ON MATCH SET n += $on_match", + {"key": props[key], "props": props, "on_match": on_match}, + ) + + def _upsert_edge(self, edge: dict[str, Any]) -> None: + edge_type = _safe(str(edge["edge_type"])) + from_type = _safe(str(edge["from_type"])) + to_type = _safe(str(edge["to_type"])) + # MERGE (not MATCH) the endpoints: one may not exist yet (e.g. usesLLM before + # the agent span), and is then created as a partial node its own span fills in. + self._db.execute_command( + f"MERGE (a:{from_type} {{id: $from_id}}) MERGE (b:{to_type} {{id: $to_id}}) " + f"MERGE (a)-[r:{edge_type}]->(b) SET r += $props", + { + "from_id": edge["from_id"], + "to_id": edge["to_id"], + "props": _props(edge, _EDGE_NON_PROPS), + }, + ) + + def _delete_edge(self, edge: dict[str, Any]) -> None: + edge_type = _safe(str(edge["edge_type"])) + from_type = _safe(str(edge["from_type"])) + to_type = _safe(str(edge["to_type"])) + self._db.execute_command( + f"MATCH (a:{from_type} {{id: $from_id}})-[r:{edge_type}]->(b:{to_type} {{id: $to_id}}) DELETE r", + {"from_id": edge["from_id"], "to_id": edge["to_id"]}, + ) + + def _delete_node(self, node: dict[str, Any]) -> None: + node_type = _safe(str(node["node_type"])) + self._db.execute_command( + f"MATCH (n:{node_type} {{id: $id}}) DETACH DELETE n", + {"id": node["id"]}, + ) diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py new file mode 100644 index 0000000..c221df8 --- /dev/null +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py @@ -0,0 +1,72 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +import json +from collections.abc import Iterator +from pathlib import Path +from typing import Any, Protocol + +from norm import GraphReader, Kind, StreamNormalizer + +from rt_norm_worker.callbacks import Notifier +from rt_norm_worker.rules import RuleSet + +DEFAULT_TRACES_PATH = Path("~/claris/otel_traces.json").expanduser() + + +def read_records(path: Path = DEFAULT_TRACES_PATH) -> Iterator[dict[str, Any]]: + """Yield the records of an OTel export file (a JSON array of span or log objects) one by one.""" + with open(Path(path).expanduser(), encoding="utf-8") as f: + records = json.load(f) + if not isinstance(records, list): + raise ValueError(f"Expected a JSON array in {path}") + yield from records + + +class Graph(GraphReader, Protocol): + """Where the KG lives: read what a span needs, write back what changed.""" + + def apply(self, delta: Any) -> None: ... + + +def process_record( + graph: Graph, record: dict[str, Any], kind: Kind, notifier: Notifier | None = None +) -> Any: + """Normalize one span / log record against the graph as stored and write what changed. + Then, for a log record (``kind="event"``), call back the rules it matches. + + Nothing is kept between records. Returns the ``StreamDelta`` that was applied.""" + print(json.dumps(record)) + delta = StreamNormalizer(graph).process(record, kind) + print( + f" -> +{len(delta.nodes)} nodes, +{len(delta.edges)} edges, " + f"-{len(delta.removed_nodes)} nodes, -{len(delta.removed_edges)} edges" + ) + graph.apply(delta) + if notifier is not None and kind == "event": + for rule in notifier.notify(record): + print(f" => rule {rule.key}: POST {rule.callback_url}") + return delta + + +def run( + graph: Graph, + path: Path = DEFAULT_TRACES_PATH, + kind: Kind = "span", + rules: RuleSet | None = None, +) -> int: + """Normalize every record of an OTel export file, one by one, into ``graph``. + + ``rules`` is what was loaded from rules.yaml: for logs, each record that matches one is + POSTed to its callback URL once it is normalized. + + Returns the number of records processed. + """ + count = 0 + notifier = Notifier(rules) if rules is not None else None + for record in read_records(path): + process_record(graph, record, kind, notifier) + count += 1 + if notifier is not None: + notifier.close() # let the pending callbacks finish + return count diff --git a/backend/workers/rt-norm-worker/tests/test_rules.py b/backend/workers/rt-norm-worker/tests/test_rules.py new file mode 100644 index 0000000..73fa45a --- /dev/null +++ b/backend/workers/rt-norm-worker/tests/test_rules.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +import json +import threading +import urllib.error +import urllib.request + +import pytest + +from rt_norm_worker.rules import DEFAULT_RULES_PATH, Rule, RuleSet, load_rules, rule_from_dict +from rt_norm_worker.server import make_server + + +def test_default_rules_file_is_loaded_in_order() -> None: + rules = load_rules(DEFAULT_RULES_PATH) + assert rules.keys() == [ + "all", "tool_events", "tool_completed", "llm_events", "llm_completed", + "mcp_traffic", "agent_handoffs", "service_events=weather-agent", "agent_events=researcher", + ] + assert rules.get("agent_events=researcher").value == "researcher" + assert rules.get("all").callback_url == "http://localhost:8080/callbacks/all" + + +def test_a_rule_with_a_value_can_be_added_once_per_value() -> None: + rules = RuleSet() + assert rules.add(Rule("agent_events", "http://a", "researcher")) + assert rules.add(Rule("agent_events", "http://b", "planner")) + assert not rules.add(Rule("agent_events", "http://c", "planner")) # replaces the callback + assert rules.get("agent_events=planner").callback_url == "http://c" + assert len(rules) == 2 + assert rules.remove("agent_events=researcher") and not rules.remove("agent_events=researcher") + + +@pytest.mark.parametrize( + "item", + [ + {"name": "nope", "callback_url": "u"}, + {"name": "all"}, + {"name": "agent_events", "callback_url": "u"}, # needs a value + {"name": "all", "callback_url": "u", "value": "x"}, # takes none + "all", + ], +) +def test_invalid_rules_are_rejected(item) -> None: + with pytest.raises(ValueError): + rule_from_dict(item) + + +def test_duplicate_rules_in_a_file_are_rejected(tmp_path) -> None: + path = tmp_path / "rules.yaml" + path.write_text("rules:\n - {name: all, callback_url: u}\n - {name: all, callback_url: v}\n") + with pytest.raises(ValueError): + load_rules(path) + + +def _call(port: int, method: str, path: str, body: dict | None = None): + data = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", data=data, method=method, headers={"Content-Type": "application/json"} + ) + try: + with urllib.request.urlopen(request) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + +def test_rules_are_managed_over_http() -> None: + from norm import InMemoryGraph + + rules = RuleSet([Rule("all", "http://x")]) + server = make_server(InMemoryGraph(), "127.0.0.1", 0, rules) + threading.Thread(target=server.serve_forever, daemon=True).start() + port = server.server_address[1] + try: + assert _call(port, "POST", "/rules", {"name": "agent_events", "value": "researcher", "callback_url": "http://r"})[0] == 201 + assert "agent_events=researcher" in rules + assert _call(port, "POST", "/rules", {"rules": [{"name": "llm_events", "callback_url": "http://l"}]})[0] == 201 + assert _call(port, "POST", "/rules", {"name": "bogus", "callback_url": "u"})[0] == 400 + status, body = _call(port, "GET", "/rules") + assert status == 200 and [r["key"] for r in body["rules"]] == ["all", "agent_events=researcher", "llm_events"] + assert _call(port, "DELETE", "/rules/agent_events%3Dresearcher")[0] == 200 + assert _call(port, "DELETE", "/rules/agent_events%3Dresearcher")[0] == 404 + assert rules.keys() == ["all", "llm_events"] + finally: + server.shutdown() + server.server_close() + + +def _log(name: str, service: str = "svc", **attrs: str) -> dict: + return {"ServiceName": service, "LogAttributes": {"event.name": name, **attrs}} + + +def test_rules_match_the_log_attributes() -> None: + def keys(rules: list[Rule], record: dict) -> list[str]: + return [r.key for r in rules if r.matches(record)] + + every = [Rule(n, "u") for n, (_, takes) in __import__("rt_norm_worker.rules").rules.RULE_TYPES.items() if not takes] + rules = every + [Rule("agent_events", "u", "researcher"), Rule("service_events", "u", "weather")] + + assert keys(rules, _log("tool.completed")) == ["all", "tool_events", "tool_completed"] + assert keys(rules, _log("llm.started")) == ["all", "llm_events"] + assert keys(rules, _log("mcp.message.sent", **{"network.protocol.name": "mcp"})) == ["all", "mcp_traffic"] + handoff = _log("topology.edge.updated", **{"topology.edge.kind": "agent_handoff", "source.agent": "a", "target.agent": "b"}) + assert "agent_handoffs" in keys(rules, handoff) + loop = _log("topology.edge.updated", **{"topology.edge.kind": "agent_handoff", "source.agent": "a", "target.agent": "a"}) + assert "agent_handoffs" not in keys(rules, loop) + assert "agent_events=researcher" in keys(rules, _log("llm.started", **{"agent.name": "researcher"})) + assert "agent_events=researcher" in keys(rules, _log("a2a.message.sent", **{"source.agent": "researcher"})) + assert "agent_events=researcher" not in keys(rules, _log("llm.started", **{"agent.name": "other"})) + assert "service_events=weather" in keys(rules, _log("llm.started", service="weather")) + as_json = {"ServiceName": "svc", "LogAttributes": json.dumps({"event.name": "tool.started"})} + assert "tool_events" in keys(rules, as_json) + + +def test_matching_logs_are_posted_to_the_callback_after_normalization() -> None: + from norm import InMemoryGraph + + from rt_norm_worker.callbacks import Notifier + from rt_norm_worker.worker import process_record + + calls: list[tuple[str, dict]] = [] + rules = RuleSet([Rule("tool_completed", "http://tool"), Rule("llm_events", "http://llm"), Rule("all", "http://boom")]) + + def post(url: str, payload: dict) -> None: + calls.append((url, payload)) + if url == "http://boom": + raise OSError("down") # a failing callback must not break anything + + notifier = Notifier(rules, post=post, wait=True) + record = _log("tool.completed", **{"session.id": "m_s1", "tool.name": "t"}) + graph = InMemoryGraph() + process_record(graph, record, "event", notifier) + assert ("Tool", "t") in graph.nodes # normalized first + assert [u for u, _ in calls] == ["http://tool", "http://boom"] + assert calls[0][1] == {"rule": "tool_completed", "value": "", "event": record} + + calls.clear() + process_record(graph, {"SpanName": "x"}, "span", notifier) # spans never call back + assert calls == [] diff --git a/backend/workers/rt-norm-worker/tests/test_server.py b/backend/workers/rt-norm-worker/tests/test_server.py new file mode 100644 index 0000000..608e5c1 --- /dev/null +++ b/backend/workers/rt-norm-worker/tests/test_server.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +import gzip +import json +import threading +import urllib.error +import urllib.request + +from norm import InMemoryGraph + +from rt_norm_worker import otlp +from rt_norm_worker.server import make_server + +SESSION = "noa-trip-planner-mas_3aefd85c" + + +def _attrs(**kwargs: str) -> list[dict]: + return [{"key": k, "value": {"stringValue": v}} for k, v in kwargs.items()] + + +LOGS = { + "resourceLogs": [ + { + "resource": {"attributes": _attrs(**{"service.name": "noa-trip-planner-mas"})}, + "scopeLogs": [ + { + "scope": {"name": "ioa_observe.runtime_events"}, + "logRecords": [ + { + "timeUnixNano": "1794000000000000000", + "spanId": "427bddeaf7b098f0", + "body": {"stringValue": "topology.session.started"}, + "attributes": _attrs( + **{ + "event.name": "topology.session.started", + "event.time": "2026-11-10T00:38:20+00:00", + "session.id": SESSION, + "application_id": "noa-trip-planner-mas", + } + ), + } + ], + } + ], + } + ] +} + +TRACES = { + "resourceSpans": [ + { + "resource": {"attributes": _attrs(**{"service.name": "svc"})}, + "scopeSpans": [ + { + "scope": {"name": "t", "version": "1"}, + "spans": [ + { + "traceId": "a" * 32, + "spanId": "b" * 16, + "name": "x.agent", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794000000000000000", + "endTimeUnixNano": "1794000001000000000", + "status": {"code": "STATUS_CODE_OK"}, + "attributes": [ + {"key": "n", "value": {"intValue": "3"}}, + {"key": "ok", "value": {"boolValue": True}}, + ], + } + ], + } + ], + } + ] +} + + +def test_otlp_flattening_matches_the_collector_export_rows() -> None: + [span] = list(otlp.spans(TRACES)) + assert span["SpanName"] == "x.agent" and span["SpanKind"] == "Internal" + assert span["ServiceName"] == "svc" and span["StatusCode"] == "Ok" + assert span["Duration"] == "1000000000" + assert span["SpanAttributes"] == {"n": "3", "ok": "true"} + assert span["Timestamp"].endswith(".000000000") and span["Timestamp"][:4] == "2026" + [log] = list(otlp.logs(LOGS)) + assert log["Body"] == "topology.session.started" and log["ServiceName"] == "noa-trip-planner-mas" + assert log["LogAttributes"]["session.id"] == SESSION + + +def _post(port: int, path: str, body: bytes, content_type: str = "application/json", **headers: str): + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", data=body, headers={"Content-Type": content_type, **headers} + ) + try: + with urllib.request.urlopen(request) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + + +def test_server_normalizes_logs_pushed_by_the_collector() -> None: + graph = InMemoryGraph() + server = make_server(graph, "127.0.0.1", 0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + assert _post(port, "/v1/logs", json.dumps(LOGS).encode()) == 200 + assert ("Session", "3aefd85c") in graph.nodes and ("MAS", "noa-trip-planner-mas") in graph.nodes + # gzip, as the collector sends by default + assert _post(port, "/v1/logs", gzip.compress(json.dumps(LOGS).encode()), **{"Content-Encoding": "gzip"}) == 200 + assert _post(port, "/v1/traces", json.dumps(TRACES).encode()) == 200 + assert _post(port, "/v1/logs", b"{", ) == 400 + assert _post(port, "/v1/logs", b"\x00", content_type="application/x-protobuf") == 415 + assert _post(port, "/v1/nope", b"{}") == 404 + finally: + server.shutdown() + server.server_close() diff --git a/backend/workers/rt-norm-worker/tests/test_worker.py b/backend/workers/rt-norm-worker/tests/test_worker.py new file mode 100644 index 0000000..64b8733 --- /dev/null +++ b/backend/workers/rt-norm-worker/tests/test_worker.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +import json + +from norm import InMemoryGraph + +from rt_norm_worker.store import Neo4jStore +from rt_norm_worker.worker import read_records, run + +SPAN = { + "Timestamp": "2026-07-09 12:00:00.000000000", + "SpanId": "span-1", + "ParentSpanId": "", + "SpanName": "concierge.agent", + "Duration": 1_000_000_000, + "SpanAttributes": {"application_id": "app", "session.id": "app_s1", "agent_id": "concierge"}, +} + + +class FakeDB: + def __init__(self): + self.commands = [] + + def execute_command(self, command, params=None): + self.commands.append((command, params)) + + +def test_read_records(tmp_path): + f = tmp_path / "t.json" + f.write_text(json.dumps([{"SpanId": "a"}, {"SpanId": "b"}])) + assert list(read_records(f)) == [{"SpanId": "a"}, {"SpanId": "b"}] + + +def test_run_writes_each_span_to_graph(tmp_path): + f = tmp_path / "t.json" + f.write_text(json.dumps([SPAN])) + graph = InMemoryGraph() + assert run(graph, f) == 1 + assert any(node_type == "AgentCall" for node_type, _ in graph.nodes) + assert graph.edges + + +def test_neo4j_store_writes_delta(tmp_path): + db = FakeDB() + db.execute = lambda query, params=None: [] # nothing stored yet + f = tmp_path / "t.json" + f.write_text(json.dumps([SPAN])) + assert run(Neo4jStore(db), f) == 1 + assert any("AgentCall" in c for c, _ in db.commands if c.startswith("MERGE (n:")) + assert any("MERGE (a)-[r:" in c for c, _ in db.commands) + + +def test_existing_nodes_keep_start_time_but_update_end_time(): + db = FakeDB() + Neo4jStore(db)._upsert_node( + {"node_type": "AgentCall", "id": "x", "startTime": 1.0, "endTime": 2.0, "name": "n"} + ) + _, params = db.commands[0] + assert params["props"]["startTime"] == 1.0 + assert "startTime" not in params["on_match"] + assert params["on_match"]["endTime"] == 2.0 + assert params["on_match"]["name"] == "n" From 196b5791f6e218eadf26c463443813d91d68879a Mon Sep 17 00:00:00 2001 From: Pavan Sudheendra Date: Thu, 19 Nov 2026 13:16:44 +0000 Subject: [PATCH 2/2] Fix span/event normalization and multi-hop agent ownership (#33) # Description Make span and event normalization converge on the same execution graph, independent of arrival order. Previously, child calls arriving before their agent could remain disconnected. MCP server-side tool spans also pointed to intermediate transport spans rather than directly to agents. Participating services could become extra MAS nodes, and arbitrary execution-graph nodes could be misclassified as Agents. This change: - Backfills ownership and capability-use relationships when agents arrive. - Traverses transport ancestry, including sessionless MCP server spans. - Persists compact ancestry in private `_NormalizationTrace` records. - Merges provisional lifecycle events with authoritative completed spans. - Retains pending handoffs, offered/discovered tools and A2A message states. - Uses the originating application for MAS identity. - Declares graph nodes as Agents only when matched to observed agents. - Corrects Agent/Tool endpoint typing when both share an id. - Preserves earliest starts, latest completions and last-ending child boundaries. - Shares normalization logic between batch processing and the realtime worker. ## Type of Change - [x] Bugfix - [x] New Feature - [ ] Breaking Change - [x] Refactor - [ ] Documentation - [ ] Other (please describe) ## Checklist - [x] I have read the [contributing guidelines](/ioc-project/scope/blob/main/CONTRIBUTING.md) - [ ] Existing issues have been referenced (where applicable) - [ ] I have verified this change is not present in other open pull requests - [ ] Functionality is documented - [ ] All code style checks pass - [ ] New code contribution is covered by automated tests - [ ] All new and existing tests pass --------- Signed-off-by: Pavan Sudheendra --- backend/norm/docs/events.md | 109 +- backend/norm/src/norm/__init__.py | 4 +- backend/norm/src/norm/ioa_observe/build.py | 204 +++- backend/norm/src/norm/ioa_observe/events.py | 137 ++- backend/norm/src/norm/ioa_observe/fields.py | 48 +- .../src/norm/ioa_observe/handlers/__init__.py | 11 +- .../norm/src/norm/ioa_observe/handlers/a2a.py | 91 +- .../src/norm/ioa_observe/handlers/graph.py | 47 +- .../norm/src/norm/ioa_observe/handlers/llm.py | 45 +- .../norm/src/norm/ioa_observe/handlers/mcp.py | 23 +- .../src/norm/ioa_observe/handlers/session.py | 4 +- .../src/norm/ioa_observe/handlers/topology.py | 52 +- .../heuristics/container_boundary_states.py | 80 +- .../src/norm/ioa_observe/reconciliation.py | 299 +++++ backend/norm/src/norm/ioa_observe/registry.py | 29 +- backend/norm/src/norm/ioa_observe/stream.py | 401 ++++--- .../norm/src/norm/ioa_observe/trajectory.py | 105 +- backend/norm/tests/test_normalizer.py | 58 +- .../norm/tests/test_pipeline_functional.py | 5 +- backend/norm/tests/test_reconciliation.py | 1060 +++++++++++++++++ backend/workers/rt-norm-worker/README.md | 53 +- .../src/rt_norm_worker/store.py | 43 +- .../src/rt_norm_worker/worker.py | 2 +- .../rt-norm-worker/tests/test_server.py | 100 +- .../rt-norm-worker/tests/test_store.py | 170 +++ .../rt-norm-worker/tests/test_worker.py | 8 +- 26 files changed, 2790 insertions(+), 398 deletions(-) create mode 100644 backend/norm/src/norm/ioa_observe/reconciliation.py create mode 100644 backend/norm/tests/test_reconciliation.py create mode 100644 backend/workers/rt-norm-worker/tests/test_store.py diff --git a/backend/norm/docs/events.md b/backend/norm/docs/events.md index c3cef5a..319aa0a 100644 --- a/backend/norm/docs/events.md +++ b/backend/norm/docs/events.md @@ -2,8 +2,8 @@ How `norm` turns OTel **log records** (events) into knowledge-graph nodes and edges. It covers the streaming path, `StreamNormalizer.process(record, kind="event")`, used by the -[rt-norm-worker](../../workers/rt-norm-worker/README.md). Spans take a different path -(`dispatch_span`, see [normalization.md](normalization.md)). +[rt-norm-worker](../../workers/rt-norm-worker/README.md). Completed spans use +[`dispatch_span`](../src/norm/ioa_observe/build.py); both paths share persisted reconciliation. Code: [`events.py`](../src/norm/ioa_observe/events.py) routes on `LogAttributes["event.name"]`; the handlers are in [`handlers/`](../src/norm/ioa_observe/handlers/). @@ -12,18 +12,24 @@ the handlers are in [`handlers/`](../src/norm/ioa_observe/handlers/). 1. A dry run on an empty registry finds which node ids (and agent names) the event touches. 2. The reader loads them from the graph store, together with the event's session subgraph. -3. The handler runs against the loaded graph. -4. Only the difference is returned as a `StreamDelta` and written back. +3. The handler merges provisional event data or authoritative completed-span data. +4. Ownership, pending messages/tools/handoffs and derived trajectories are reconciled. +5. Only the difference is returned as a `StreamDelta` and written back. -Nothing is kept between events: the stored graph is the state. Consequences: +The normalizer itself keeps no state between events: it uses the stored graph and +adapter-managed trace context. Consequences: - Handlers are **idempotent**. Replaying an event changes nothing. - Handlers are **order-tolerant**. Most of them create whatever is missing above the node they need (MAS, Session, MASCall, Agent, AgentCall), so a `completed` event arriving alone builds what its `started` would have. -- A node that already exists is **reused, not overwritten**. `startTime` never changes; - `endTime` only moves forward. Initial and final States are set once, by whichever event - arrives first. +- Existing identities are reused. Late starts can correct `startTime` and input; completed + spans replace provisional timing, content and metrics without creating a second call. + Container start/end times encompass their children, and final states follow the child + ending last, not the child starting last. +- Compact `rtNormalization` metadata retains pending relationships, canonical trajectory + identities and processed event ids. Existing persisted worker metadata and the + `handoffSourceSpanIds` property remain readable; no raw telemetry replay buffer is kept. - An event with no `session.id` (or no agent / tool name where one is needed) is ignored. ## Identity @@ -31,7 +37,7 @@ Nothing is kept between events: the stored graph is the state. Consequences: | Node | Id | |---|---| | `Session` | `session.id` without its `_` prefix | -| `MAS` | `application_id` (falls back to `ServiceName`) | +| `MAS` | originating application in a propagated `_`; otherwise `application_id`, then `ServiceName` | | `MASCall` | `mascall:` | | `Agent` | the agent's **name** (`agent.name`, `source.agent`, `target.agent`, `tool.name` of an a2a tool, the card's `name`). The same name in another MAS is another Agent, with id `:`. The span id is the AgentCall's, not the Agent's. | | `AgentCall` | hash of the Agent's id and the span id (`topology.node.*` `span.id`, or `ioa_observe.agent.span_id` of the other events) | @@ -42,7 +48,35 @@ Nothing is kept between events: the stored graph is the state. Consequences: | `State`, `Transition` | hashes of the owner's span and the content | Events that carry only an `ioa_observe.agent.span_id` (`tool.*`, `mcp.tools.discovered`) find the -Agent through the AgentCall of that span. +Agent through the AgentCall of that span. Ownership is backfilled when that call arrives. +Application names may contain underscores; participating MCP services do not create a +second MAS merely because their application or service name differs. + +Span ownership also follows parent chains through MCP/A2A/framework transport spans. +An explicit agent-owner span id takes precedence over that ancestor's ordinary parent. +Intermediate spans need not have a session id: private trace context retains their +`TraceId`, `SpanId`, parent, owner and optional session id, without storing raw payloads +or manufacturing ontology executions. Known session boundaries and trace ids are +respected; missing links and cyclic parent chains remain unresolved rather than guessed. + +Streaming adapters must implement `GraphReader.load_span_contexts(trace_id)` and retain +`StreamDelta.span_contexts` separately from ontology deltas. The realtime worker retains +this context in memory on its store instance, not in Neo4j. It is lost on restart and is +not shared across workers; replay the relevant spans/events into the same instance to +rebuild it. A sessionless transport arriving +last rechecks sessions referenced by that trace. Without `TraceId`, ancestry is scoped +to the explicit session; a span with neither identifier cannot be correlated safely. +Batch normalization uses the same ancestry resolver. + +Agent and Tool identities may share a name/id while remaining different ontology types. +Streaming edge endpoints are resolved using their relationship type (and the concrete +execution relationship for generic `executes`), not whichever same-id node was loaded last. +The existing verifier can still report domain mismatches when it collapses such node ids; +that diagnostic limitation is separate from the typed relationships persisted to Neo4j. + +Execution graph metadata is retained on the MAS. A graph node becomes a declared Agent +only when it matches an observed agent's identity, name or runtime alias. Routers and +tool-serving graph nodes are not assumed to be agents. ## States @@ -92,16 +126,12 @@ Same nodes as `node.started`. ### `topology.edge.updated` -Reads `source.agent.span.id`, `target.agent`. Creates nothing. +Reads `source.agent.span.id`, `target.agent`. -- Adds `executesAgent` from the **AgentCall** of the source span to the **Agent** named - `target.agent`. -- Ignored when either does not exist: a target that is not an agent (an MCP server such as - `debug`) matches nothing. - -`executesAgent` otherwise means "this call is an instance of that agent"; here it also records -the handoff. A call's own agent is therefore the one it was created for, not any `executesAgent` -target (`_own_agent_id`). +- Retains a pending handoff hint until both agent calls exist. +- Connects their execution trajectories through the shared handoff heuristic. +- Never adds a second `executesAgent` target to the sender or invents an Agent for a + tool-serving target. Explicit MCP/tool topology edges are ignored as agent handoffs. ### `llm.started` @@ -119,6 +149,15 @@ Same nodes as `llm.started`. - Moves the LLMCall's `endTime` forward and sets `duration`. - Creates the final State from `llm.output` and the Transition, once. +### `llm.tools.presented` + +Reads `capability.content` (JSON tools, including OpenAI-style function descriptors) and +`ioa_observe.agent.span_id`, falling back to the record's span. + +- Creates or enriches Tool descriptions. +- Persists the offered tool ids and owner span, linking them with `usesTool` when the + owning AgentCall arrives. + ### `tool.started` Reads `tool.name`, `tool.input`, `tool.type`, `tool.call.id`, `ioa_observe.agent.span_id`. @@ -147,8 +186,7 @@ Same nodes as `tool.started`. Reads `session.id`, `capability.content` (JSON): `name`, `description`. - Ignored without a `session.id`. -- Creates `MAS` (the first part of `session.id`, `_`) and `Session` (the - rest), linked by `executesSession`. +- Creates the originating `MAS` and `Session`, linked by `executesSession`. - Adds the `Agent` named by the card, keyed on its **name** (the card has no span id of the agent it describes), with `belongsToMAS`. An Agent of that name already in the MAS is reused and only gets its description. The ontology has no @@ -170,21 +208,20 @@ Reads `source.agent`, `source.agent.span.id`, `target.agent`, `message.id`, `mes - Keeps `message.input` as a State of the sender's AgentCall (`hasState` only, so its initial and final States are untouched). -### `a2a.message.received` - -Same nodes as `a2a.message.sent`. +### `a2a.message.received` / `a2a.message.completed` -- Keeps `message.input` as a State of the **recipient's** latest AgentCall, only if that call - already exists. The event does not carry the recipient's span id, and its call is normally - created a few milliseconds later by its own `topology.node.started`, so usually nothing of - the message is kept. +- Retains incoming `message.input` and completion `message.output` as separate message + States, even before the corresponding AgentCall exists. +- Received messages are reconciled with the recipient's call by name and event time; + completion replies attach to the source agent span. +- Does not fabricate a recipient AgentCall or replace an agent's own final State. ### `mcp.message.sent` / `mcp.message.received` Reads `message.input` (JSON), `message.id`, `source.agent`, `source.agent.span.id`. - Ignored when `message.input` is empty or has no `name` (a `tools/list` request). -- Ensures the sender's chain: MAS (from `application_id`, else the first part of `session.id`), +- Ensures the sender's chain: originating MAS, Session, MASCall, Agent, AgentCall. - Creates `Tool` (`message.input.name`) and `ToolCall` keyed on `message.id`, with `executesTool`, `hasToolCall` and `usesTool`. @@ -206,13 +243,13 @@ Same nodes as `mcp.message.sent`. Reads `ioa_observe.agent.span_id`, `session.id`, `capability.content` (JSON with a `tools` list). - Ignored without `ioa_observe.agent.span_id` or a session. -- Creates `MAS` (named by the first part of `session.id`, `_`) and `Session`. +- Creates the originating `MAS` and `Session`. - Creates a `Tool` for every entry in `tools` with its `description`, and fills the description of a Tool that already exists without one. - The event names the agent only by span id, so the agent is the one of the AgentCall of that span, and every tool is linked to it with `usesTool`. If that call does not exist yet (the - event arrives before the agent's `topology.node.started`), the tools are created **without** - the link: nothing is remembered to link them later. + event arrives before the agent's `topology.node.started` or completed span), owner hints + and tool ids are persisted so the link is backfilled. ## Not handled @@ -222,15 +259,13 @@ These are ignored; `dispatch_event` returns `False` for names it does not know. |---|---| | `slim.message.sent`, `slim.message.received` | `TODO` | | `mcp.tools.published` | `TODO` | -| `a2a.message.completed` | no case | -| `llm.tools.presented` | no case | ## Known gaps -- The handoff between agents is only the `executesAgent` edge from `topology.edge.updated`. The - ontology has no Agent-to-Agent relationship. - `tool.*` events without `tool.call.id` and without `ioa_observe.agent.span_id` create a ToolCall that nothing links to an agent. - When an agent runs twice it has one `Agent` and two `AgentCall` nodes. The same agent name in another MAS is a different Agent. -- `mcp.tools.discovered` arriving before the agent's `topology.node.started` does not link its tools. +- Previously misclassified graph data is not automatically migrated or deleted. Compare + end-to-end runs on a fresh graph. Ontology/SHACL findings are separate diagnostics; + normalization does not weaken the verifier to hide them. diff --git a/backend/norm/src/norm/__init__.py b/backend/norm/src/norm/__init__.py index 3192edf..c31b530 100644 --- a/backend/norm/src/norm/__init__.py +++ b/backend/norm/src/norm/__init__.py @@ -2,8 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 from .compare import KGCompareResult, compare_kg, read_kg_json, write_kg_json from .ioa_observe import build_kg -from .ioa_observe.stream import GraphReader, InMemoryGraph, Kind, StreamDelta, StreamNormalizer from .ioa_observe.otel_io import infer_run_id, load_otel_export +from .ioa_observe.registry import SpanContext +from .ioa_observe.stream import GraphReader, InMemoryGraph, Kind, StreamDelta, StreamNormalizer from .normalizer import Normalizer, OnNormalizedCallback, configure, dump_jsonld, normalize from .verifier import ( KGCheckSkipped, @@ -26,6 +27,7 @@ "InMemoryGraph", "Kind", "StreamDelta", + "SpanContext", "StreamNormalizer", "build_kg", "check_edge_domain_range", diff --git a/backend/norm/src/norm/ioa_observe/build.py b/backend/norm/src/norm/ioa_observe/build.py index 3038ff7..bf3f23c 100644 --- a/backend/norm/src/norm/ioa_observe/build.py +++ b/backend/norm/src/norm/ioa_observe/build.py @@ -12,9 +12,34 @@ from __future__ import annotations +from copy import deepcopy from typing import Any -from .fields import get_span_attrs +from scope_ontology.models.edges import ( + executes, + executesAgent, + executesLLM, + executesMAS, + executesProcessing, + executesSession, + executesTool, +) +from scope_ontology.models.nodes import ( + Agent, + AgentCall, + LLMCall, + Session, + Tool, + ToolCall, +) + +from .fields import ( + _to_float, + get_span_attrs, + get_span_session_id, + get_span_start_time, + normalize_record, +) from .handlers import ( handle_agent, handle_chat, @@ -23,13 +48,30 @@ handle_session_start, handle_tool, ) +from .handlers.agent import _agent_id_for from .heuristics import ( assign_container_boundary_states, chain_agent_handoffs_fallback, chain_capability_calls, link_agent_handoffs, ) -from .registry import Registry +from .heuristics.handoff import _source_span_ids +from .reconciliation import ( + handoff_spans, + reconcile_relationships, + remember_span_ancestry, + strip_derived, +) +from .registry import HANDOFF_SOURCES_PROP, Registry +from .trajectory import ( + _clear_boundaries, + _remember_trajectory, + prune_trajectories, + restore_trajectories, +) + +_CALLS = (AgentCall, LLMCall, ToolCall) + _HANDLERS: dict[str, Any] = { "session.start": handle_session_start, @@ -37,7 +79,7 @@ } -def dispatch_span(span: dict[str, Any], registry: Registry) -> None: +def _dispatch_span(span: dict[str, Any], registry: Registry) -> None: """Run the one per-span handler matching ``span``'s ``SpanName`` (if any).""" span_name = str(span.get("SpanName") or "") span_attrs = get_span_attrs(span) @@ -57,14 +99,39 @@ def dispatch_span(span: dict[str, Any], registry: Registry) -> None: def apply_heuristics(spans: list[dict[str, Any]], registry: Registry) -> None: """Cross-span clean-up of the trajectory, once ``spans`` are all handled.""" + reconcile_relationships(registry) + strip_derived(registry) + restore_trajectories(registry, registry.metadata_for) + registry.nodes = dict(sorted(registry.nodes.items())) + registry.edges = dict( + sorted(registry.edges.items(), key=lambda item: (item[0][0].__name__, *item[0][1:])) + ) # Agent handoffs - explicit_handoff_targets = link_agent_handoffs(spans, registry) + explicit_handoff_targets = link_agent_handoffs(handoff_spans(registry), registry) # Fallback: timestamp-order sibling AgentCalls the above left unlinked chain_agent_handoffs_fallback(registry, explicit_handoff_targets) # Capability calls within agent call chain_capability_calls(registry) # Session and MAS call i/o assign_container_boundary_states(registry) + prune_trajectories(registry, registry.metadata_for) + typed_execution = ( + executesAgent, + executesLLM, + executesTool, + executesProcessing, + executesMAS, + executesSession, + ) + registry.remove_edges_where( + lambda edge, source, target: ( + edge is executes + and not any((typed, source, target) in registry.edges for typed in typed_execution) + ) + ) + for edge, source, target in list(registry.edges): + if edge in typed_execution: + registry.add_edge(executes(source_id=source, target_id=target)) def serialize_registry( @@ -86,6 +153,8 @@ def serialize_registry( value = getattr(item, attr, None) if value is not None: payload[attr] = value + extra = registry.extras.get((type(item).__name__, item.id), {}) + payload.update({key: value for key, value in extra.items() if value}) nodes.append(payload) return nodes, edges @@ -99,3 +168,130 @@ def build_kg( dispatch_span(span, registry) apply_heuristics(spans, registry) return serialize_registry(registry) + + +def dispatch_span(record: dict[str, Any], registry: Registry) -> None: + record, attrs = normalize_record(record, "span") + remember_span_ancestry(record, attrs, registry) + meta = registry.metadata_for + name = str(record.get("SpanName") or "") + if name in ("session.start", "session.end"): + session = registry.get(Session, get_span_session_id(record, attrs)) + if session is None: + initial_attrs = attrs + if name == "session.end": + initial_attrs = { + **attrs, + "ioa_start_time": _to_float(attrs.get("session.ended_at")) + or get_span_start_time(record, attrs), + } + handle_session_start(record, initial_attrs, registry) + session = registry.get(Session, get_span_session_id(record, attrs)) + if session is not None: + _dispatch_span(record, registry) + if name == "session.start": + meta(session)["sessionSpanId"] = str(record.get("SpanId") or "") + start = min(session.startTime, get_span_start_time(record, attrs)) + registry.upsert( + session.model_copy( + update={ + "startTime": start, + "duration": (session.endTime - start) * 1000, + } + ) + ) + return + if name.endswith(".graph"): + handle_graph(record, attrs, registry) + return + if name.endswith(".agent"): + display = str( + attrs.get("ioa_observe.entity.name") + or attrs.get("agent_id") + or record.get("AgentId") + or "" + ) + application = str(attrs.get("application_id") or "") + runtime_id = str(record.get("AgentId") or attrs.get("agent_id") or "") + attrs = { + **attrs, + "agent_id": _agent_id_for(registry, display, (application,)), + } + else: + runtime_id = "" + scratch = Registry() + _dispatch_span( + { + **record, + "SpanAttributes": attrs, + **({"AgentId": attrs["agent_id"]} if name.endswith(".agent") else {}), + }, + scratch, + ) + for cls in _CALLS: + for call in scratch.all_of(cls): + existing = registry.find( + cls, + lambda n: ( + n.sessionId == call.sessionId + and (n.spanId == call.spanId or call.spanId in meta(n).get("spanIds", [])) + ), + ) + tool_call_id = str(attrs.get("gen_ai.tool.call.id") or "") if cls is ToolCall else "" + if existing is None and tool_call_id: + existing = registry.get(ToolCall, tool_call_id) + canonical_id = tool_call_id or (existing.id if existing is not None else call.id) + if existing is not None and existing.id != canonical_id: + old_metadata = deepcopy(meta(existing)) + registry.merge_node_into(cls, existing.id, canonical_id) + else: + old_metadata = {} + if call.id != canonical_id: + scratch.merge_node_into(cls, call.id, canonical_id) + call = call.model_copy(update={"id": canonical_id}) + scratch.add(call) + meta(call).update(old_metadata) + if record.get("TraceId"): + meta(call)["traceId"] = str(record["TraceId"]) + if meta(call).get("spanComplete") and registry.get(cls, call.id) is not None: + return + # The completed span replaces provisional event boundaries, not message States. + _clear_boundaries(registry, call.id) + registry.upsert(call) + _remember_trajectory(scratch, call, meta) + for key in ( + "spanIds", + "spanPriority", + "input", + "inputPriority", + "output", + "outputPriority", + "complete", + ): + meta(call).pop(key, None) + meta(call)["spanComplete"] = True + owner = str(attrs.get("ioa_observe.agent.span_id") or call.parentSpanId or "") + if owner: + meta(call)["ownerSpanId"] = owner + owner_agent = str(record.get("AgentId") or attrs.get("agent_id") or "") + if owner_agent and cls in (LLMCall, ToolCall): + meta(call)["ownerAgentId"] = owner_agent + if cls is AgentCall: + sources = _source_span_ids(attrs) + if sources: + registry.extras.setdefault(("AgentCall", call.id), {})[HANDOFF_SOURCES_PROP] = ( + sources + ) + for node in scratch.nodes.values(): + existing_node = registry.add(node) + if ( + isinstance(node, (Agent, Tool)) + and isinstance(existing_node, (Agent, Tool)) + and node.description + and not existing_node.description + ): + registry.upsert(existing_node.model_copy(update={"description": node.description})) + if isinstance(node, Agent) and runtime_id: + meta(node)["aliases"] = sorted(set(meta(node).get("aliases", []) + [runtime_id])) + for edge in scratch.edges.values(): + registry.add_edge(edge) diff --git a/backend/norm/src/norm/ioa_observe/events.py b/backend/norm/src/norm/ioa_observe/events.py index 08c7c1e..e280e56 100644 --- a/backend/norm/src/norm/ioa_observe/events.py +++ b/backend/norm/src/norm/ioa_observe/events.py @@ -4,14 +4,22 @@ from __future__ import annotations +import logging +from copy import deepcopy from typing import Any +from scope_ontology.models.edges import executesTool +from scope_ontology.models.nodes import AgentCall, LLMCall, ToolCall + +from .fields import get_event_start_time, normalize_record from .handlers import ( handle_a2a_agent_card, + handle_a2a_message_completed, handle_a2a_message_received, handle_a2a_message_sent, handle_llm_completed, handle_llm_started, + handle_llm_tools_presented, handle_mcp_message_completed, handle_mcp_message_received, handle_mcp_message_sent, @@ -24,10 +32,30 @@ handle_topology_session_completed, handle_topology_session_started, ) +from .reconciliation import event_context, merge_span_context from .registry import Registry +from .trajectory import ( + _clear_boundaries, + _remember_trajectory, + add_final_state_transition, + add_initial_state, +) + +logger = logging.getLogger(__name__) +_CALLS = (AgentCall, LLMCall, ToolCall) def dispatch_event(event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry) -> bool: + """Apply a lifecycle record without overwriting an authoritative completed span.""" + event, event_attrs = normalize_record({**event, "LogAttributes": event_attrs}, "event") + context = event_context(event, event_attrs) + if context is not None: + key = (context.trace_id, context.span_id) + registry.span_contexts[key] = merge_span_context(context, registry.span_contexts.get(key)) + return apply_event(event, event_attrs, registry) + + +def _dispatch_event(event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry) -> bool: """Run the case for the event's ``event.name``. False if the name is unknown.""" match str(event_attrs.get("event.name") or ""): case "topology.session.started": @@ -39,13 +67,13 @@ def dispatch_event(event: dict[str, Any], event_attrs: dict[str, Any], registry: case "topology.node.completed": handle_topology_node_completed(event, event_attrs, registry) case "topology.edge.updated": - # TODO: handle topology.edge.updated - #handle_topology_edge_updated(event, event_attrs, registry) - pass + handle_topology_edge_updated(event, event_attrs, registry) case "llm.started": handle_llm_started(event, event_attrs, registry) case "llm.completed": handle_llm_completed(event, event_attrs, registry) + case "llm.tools.presented": + handle_llm_tools_presented(event, event_attrs, registry) case "tool.started": handle_tool_started(event, event_attrs, registry) case "tool.completed": @@ -54,6 +82,8 @@ def dispatch_event(event: dict[str, Any], event_attrs: dict[str, Any], registry: handle_a2a_message_sent(event, event_attrs, registry) case "a2a.message.received": handle_a2a_message_received(event, event_attrs, registry) + case "a2a.message.completed": + handle_a2a_message_completed(event, event_attrs, registry) case "a2a.agent_card": handle_a2a_agent_card(event, event_attrs, registry) case "slim.message.sent": @@ -76,3 +106,104 @@ def dispatch_event(event: dict[str, Any], event_attrs: dict[str, Any], registry: case _: return False return True + + +def apply_event(record: dict[str, Any], attrs: dict[str, Any], registry: Registry) -> bool: + + meta = registry.metadata_for + name = str(attrs.get("event.name") or "") + tool_call_id = str(attrs.get("tool.call.id") or attrs.get("message.id") or "") + event_span_id = str(record.get("SpanId") or "") + existing_tool = registry.find(ToolCall, lambda n: n.spanId == event_span_id) + if ( + tool_call_id + and existing_tool is not None + and existing_tool.id != tool_call_id + and registry.get(ToolCall, tool_call_id) is None + ): + old_metadata = deepcopy(meta(existing_tool)) + registry.merge_node_into(ToolCall, existing_tool.id, tool_call_id) + renamed = existing_tool.model_copy(update={"id": tool_call_id}) + registry.add(renamed) + meta(renamed).update(old_metadata) + authoritative = [ + n.model_copy(deep=True) + for cls in _CALLS + for n in registry.all_of(cls) + if meta(n).get("spanComplete") + ] + if not _dispatch_event(record, attrs, registry): + logger.debug("Unsupported runtime event %s", name) + return False + for node in authoritative: + registry.upsert(node) + event_time = get_event_start_time(record, attrs) + span_id = str(attrs.get("llm.call.id") or record.get("SpanId") or "") + prefix = name.rsplit(".", 1)[0] + cls = { + "topology.node": AgentCall, + "llm": LLMCall, + "tool": ToolCall, + "mcp.message": ToolCall, + }.get(prefix) + if cls is not None: + call_id = str(attrs.get("tool.call.id") or attrs.get("message.id") or "") + call = registry.get(cls, call_id) if call_id else None + call = call or registry.find(cls, lambda n: n.spanId == span_id) + if call is not None and not meta(call).get("spanComplete"): + start = min(call.startTime, event_time) + end = max(call.endTime, event_time) + call_meta = meta(call) + call_meta["spanIds"] = sorted(set(call_meta.get("spanIds", []) + [span_id])) + priority = 2 if prefix != "mcp.message" else (0 if name.endswith(".received") else 1) + previous = call_meta.get("spanPriority", -1) + chosen_span = call.spanId + if priority > previous or (priority == previous and span_id < chosen_span): + chosen_span = span_id + call_meta["spanPriority"] = priority + call = call.model_copy( + update={ + "startTime": start, + "endTime": end, + "duration": (end - start) * 1000, + "spanId": chosen_span, + } + ) + if isinstance(call, ToolCall): + tool = registry.edge_target(executesTool, call.id) or "" + call = call.model_copy(update={"name": f"Tool call ({tool}) {chosen_span}"}) + registry.upsert(call) + field_prefix = { + "topology.node": "agent", + "mcp.message": "message", + }.get(prefix, prefix) + priority = 1 if prefix == "mcp.message" else 2 + for field in ("input", "output"): + key = f"{field_prefix}.{field}" + if key in attrs and priority >= call_meta.get(f"{field}Priority", 0): + call_meta[field] = str(attrs[key] or "") + call_meta[f"{field}Priority"] = priority + if name.endswith(".completed"): + call_meta["complete"] = True + _clear_boundaries(registry, call.id) + input_text = call_meta.get("input", "") + initial = add_initial_state( + registry, + call.id, + call.sessionId, + call.spanId, + input_text, + ) + if call_meta.get("complete"): + add_final_state_transition( + registry, + call.id, + call.sessionId, + call.spanId, + initial.id, + input_text, + call_meta.get("output", ""), + ) + _remember_trajectory(registry, call, meta) + + return True diff --git a/backend/norm/src/norm/ioa_observe/fields.py b/backend/norm/src/norm/ioa_observe/fields.py index 12b32c8..676a72f 100644 --- a/backend/norm/src/norm/ioa_observe/fields.py +++ b/backend/norm/src/norm/ioa_observe/fields.py @@ -5,18 +5,21 @@ from __future__ import annotations import hashlib +import json +import logging from datetime import datetime, timezone from typing import Any # Span field derivation functions + def get_span_attrs(span: dict[str, Any]) -> dict[str, Any]: """Return a span's SpanAttributes map, or {} if absent.""" return span.get("SpanAttributes") or {} def get_span_application_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: - return str(span_attrs.get("application_id") or span.get("ServiceName") or "") + return _application_id(span, span_attrs) def get_span_session_id(span: dict[str, Any], span_attrs: dict[str, Any]) -> str: @@ -109,14 +112,17 @@ def _hash_id(prefix: str, *parts: str) -> str: digest = hashlib.sha1("\x1f".join(str(p) for p in parts).encode("utf-8")).hexdigest()[:12] return f"{prefix}:{digest}" + # Event field derivation functions + def get_event_attrs(event: dict[str, Any]) -> dict[str, Any]: """Return an event's (OTel log record's) LogAttributes map, or {} if absent.""" return event.get("LogAttributes") or {} + def get_event_application_id(event: dict[str, Any], event_attrs: dict[str, Any]) -> str: - return str(event_attrs.get("application_id") or event.get("ServiceName") or "") + return _application_id(event, event_attrs) def get_event_session_id(event: dict[str, Any], event_attrs: dict[str, Any]) -> str: @@ -146,3 +152,41 @@ def get_event_start_time(event: dict[str, Any], event_attrs: dict[str, Any]) -> pass timestamp = str(event.get("Timestamp") or "") return _parse_timestamp_ns(timestamp) / 1e9 if timestamp else 0.0 + + +logger = logging.getLogger(__name__) + + +def _application_id(record: dict[str, Any], attrs: dict[str, Any]) -> str: + application = str(attrs.get("application_id") or record.get("ServiceName") or "") + session = str(record.get("SessionId") or attrs.get("session.id") or "") + # Observe propagates the originating application's session into participating services. + if "_" in session and not session.startswith(f"{application}_"): + return session.rpartition("_")[0] + return application + + +def normalize_record(record: dict[str, Any], kind: str) -> tuple[dict[str, Any], dict[str, Any]]: + key = "SpanAttributes" if kind == "span" else "LogAttributes" + attrs = record.get(key) or {} + if isinstance(attrs, str): + attrs = json.loads(attrs) + if not isinstance(attrs, dict): + raise ValueError(f"{key} must be an object") + attrs = dict(attrs) + application = _application_id(record, attrs) + if application: + attrs["application_id"] = application + return {**record, key: attrs}, attrs + + +def parse_json_object(raw: Any, label: str) -> dict[str, Any]: + try: + value = json.loads(raw) if isinstance(raw, str) else raw + except ValueError as exc: + logger.warning("Invalid %s: %s", label, exc) + return {} + if not isinstance(value, dict): + logger.warning("%s must be an object", label) + return {} + return value diff --git a/backend/norm/src/norm/ioa_observe/handlers/__init__.py b/backend/norm/src/norm/ioa_observe/handlers/__init__.py index e7185c9..ce44632 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/__init__.py +++ b/backend/norm/src/norm/ioa_observe/handlers/__init__.py @@ -2,17 +2,22 @@ # SPDX-License-Identifier: Apache-2.0 """One handler module per ``SpanName`` shape -- see build.py for dispatch.""" -from .a2a import handle_a2a_agent_card, handle_a2a_message_received, handle_a2a_message_sent +from .a2a import ( + handle_a2a_agent_card, + handle_a2a_message_completed, + handle_a2a_message_received, + handle_a2a_message_sent, +) from .agent import handle_agent from .chat import handle_chat from .graph import handle_graph +from .llm import handle_llm_completed, handle_llm_started, handle_llm_tools_presented from .mcp import ( handle_mcp_message_completed, handle_mcp_message_received, handle_mcp_message_sent, handle_mcp_tools_discovered, ) -from .llm import handle_llm_completed, handle_llm_started from .session import handle_session_end, handle_session_start from .tool import handle_tool from .tool_events import handle_tool_completed, handle_tool_started @@ -28,11 +33,13 @@ "handle_a2a_agent_card", "handle_a2a_message_received", "handle_a2a_message_sent", + "handle_a2a_message_completed", "handle_agent", "handle_chat", "handle_graph", "handle_llm_completed", "handle_llm_started", + "handle_llm_tools_presented", "handle_mcp_message_completed", "handle_mcp_message_received", "handle_mcp_message_sent", diff --git a/backend/norm/src/norm/ioa_observe/handlers/a2a.py b/backend/norm/src/norm/ioa_observe/handlers/a2a.py index 2a2519c..ca6a65a 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/a2a.py +++ b/backend/norm/src/norm/ioa_observe/handlers/a2a.py @@ -10,9 +10,9 @@ from scope_ontology.models.edges import belongsToMAS, hasState from scope_ontology.models.nodes import Agent, AgentCall, State -from ..fields import _hash_id +from ..fields import _hash_id, get_event_application_id, get_event_session_id, get_event_start_time from ..registry import Registry -from .agent import _agent_id_for, _ensure_agent, _latest_call_of +from .agent import _agent_id_for, _ensure_agent from .topology import _ensure_agent_call, _ensure_session_and_mas @@ -61,8 +61,13 @@ def handle_a2a_message_sent( state = registry.add( State( id=_hash_id( - "state", "a2a.message.sent", session_id, source_span_id, - target_name, message_id, content, + "state", + "a2a.message.sent", + session_id, + source_span_id, + target_name, + message_id, + content, ), sessionId=session_id, content=content, @@ -74,35 +79,11 @@ def handle_a2a_message_sent( def handle_a2a_message_received( event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry ) -> None: - """Ensure the same nodes as ``a2a.message.sent``, and retain the incoming message as a - State of the recipient's AgentCall without changing its initial or final State. - - The event names the recipient only by ``target.agent`` -- its span is not in it, and - its AgentCall is normally created a few milliseconds later by its own - ``topology.node.started`` -- so the State is added only if that call is already there; - otherwise nothing of the message is kept.""" - parties = _ensure_message_parties(event, event_attrs, registry) - if parties is None: - return - _, session_id, source_span_id, target_name, message_id, content = parties - receiver_call = _latest_call_of(registry, target_name, session_id) - if receiver_call is None or not content: - return - state = registry.add( - State( - id=_hash_id( - "state", "a2a.message.received", session_id, source_span_id, - target_name, message_id, content, - ), - sessionId=session_id, - content=content, - ) - ) - registry.add_edge(hasState(source_id=receiver_call.id, target_id=state.id)) + _handle_a2a_message(event, event_attrs, registry) def _agent_card(event_attrs: dict[str, Any]) -> dict[str, Any]: - """The agent card carried in ``capability.content`` (a JSON object), or {} if absent or malformed.""" + """The agent card in ``capability.content``, or {} if absent or malformed.""" content = event_attrs.get("capability.content") if isinstance(content, str): try: @@ -126,10 +107,13 @@ def handle_a2a_agent_card( one of that name already there is reused and only gets its description.""" card = _agent_card(event_attrs) name = str(card.get("name") or "") - mas_name = str(event_attrs.get("session.id") or "").partition("_")[0] + mas_name = get_event_application_id(event, event_attrs) if not name or not mas_name: return - if _ensure_session_and_mas(event, {**event_attrs, "application_id": mas_name}, registry) is None: + if ( + _ensure_session_and_mas(event, {**event_attrs, "application_id": mas_name}, registry) + is None + ): return description = str(card.get("description") or "") agent = _ensure_agent(registry, _agent_id_for(registry, name, (mas_name,)), name, (mas_name,)) @@ -164,3 +148,46 @@ def handle_a2a_tool_started( # target. The ontology has no Agent -> Agent relationship; a handoff is otherwise a # bridge ProcessingCall between the two AgentCalls (heuristics/handoff.py), and the # target's AgentCall does not exist yet when this event arrives. + + +def _handle_a2a_message( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + record, attrs = event, event_attrs + meta = registry.metadata_for + name = str(attrs.get("event.name") or "") + source = str(attrs.get("source.agent.span.id") or "") + call = registry.find(AgentCall, lambda n: n.spanId == source) + content = str( + attrs.get("message.output" if name.endswith("completed") else "message.input") or "" + ) + session_id = get_event_session_id(record, attrs) + if session_id and content: + _ensure_session_and_mas(record, attrs, registry) + state = registry.add( + State( + id=_hash_id( + "state", + name, + session_id, + str(attrs.get("message.id") or ""), + content, + ), + sessionId=session_id, + content=content, + ) + ) + if name.endswith("received"): + meta(state)["receiverName"] = str(attrs.get("target.agent") or "") + meta(state)["eventTime"] = get_event_start_time(record, attrs) + else: + meta(state)["ownerSpanId"] = source + if call is not None and name.endswith("completed"): + registry.add_edge(hasState(source_id=call.id, target_id=state.id)) + return + + +def handle_a2a_message_completed( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + _handle_a2a_message(event, event_attrs, registry) diff --git a/backend/norm/src/norm/ioa_observe/handlers/graph.py b/backend/norm/src/norm/ioa_observe/handlers/graph.py index cbba519..65b8c29 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/graph.py +++ b/backend/norm/src/norm/ioa_observe/handlers/graph.py @@ -1,6 +1,6 @@ # Copyright (c) 2026 Cisco Systems, Inc. and its affiliates # SPDX-License-Identifier: Apache-2.0 -"""Handler for the "*.graph" span name: declared MAS/Agent structural nodes. +"""Retain graph metadata and declare only matching observed agents. This is the one authoritative source of "declared" structural data, so it always overwrites (``registry.upsert``) rather than fills-in-if-absent -- @@ -11,19 +11,15 @@ from __future__ import annotations -import json -import logging from typing import Any from scope_ontology.models.base import KGBase from scope_ontology.models.edges import belongsToMAS from scope_ontology.models.nodes import MAS, Agent, StructuralElement -from ..fields import get_span_application_id +from ..fields import get_span_application_id, parse_json_object from ..registry import Registry -log = logging.getLogger(__name__) - def _declare( registry: Registry, @@ -69,21 +65,26 @@ def handle_graph( graph_raw = span_attrs.get("gen_ai.ioa.graph") if not graph_raw: return - try: - graph_data: dict = json.loads(graph_raw) - except (TypeError, ValueError, json.JSONDecodeError): - log.warning( - "Could not parse gen_ai.ioa.graph for span %s -- skipping structural " - "graph extraction (span name %s)", - span.get("SpanId"), - span.get("SpanName"), - ) - return + graph_data = parse_json_object(graph_raw, "gen_ai.ioa.graph") + graph_nodes = graph_data.get("nodes", {}) + if not isinstance(graph_nodes, dict): + raise ValueError("gen_ai.ioa.graph.nodes must be an object") + mas = registry.get(MAS, application_id) + if mas is not None and graph_data: + registry.metadata_for(mas)["graphNodes"] = graph_nodes + - graph_nodes: dict[str, dict] = graph_data.get("nodes") or {} - for node_id, node in graph_nodes.items(): - if node_id in ("__start__", "__end__"): - continue - agent_name = str(node.get("name") or node_id) - agent: Agent = _declare(registry, Agent, node_id, name=agent_name) - registry.add_edge(belongsToMAS(source_id=agent.id, target_id=application_id)) +def reconcile_graph_agents(registry: Registry) -> None: + """Graph nodes describe execution, not roles; require runtime agent evidence.""" + for mas in registry.all_of(MAS): + graph_nodes = registry.metadata_for(mas).get("graphNodes", {}) + for agent in registry.all_of(Agent): + if mas.id not in registry.edges_from(belongsToMAS, agent.id): + continue + aliases = [agent.id, agent.name, *registry.metadata_for(agent).get("aliases", [])] + if any( + key in aliases or str(value.get("name") or key) in aliases + for key, value in graph_nodes.items() + if isinstance(value, dict) + ): + registry.upsert(agent.model_copy(update={"declared": True})) diff --git a/backend/norm/src/norm/ioa_observe/handlers/llm.py b/backend/norm/src/norm/ioa_observe/handlers/llm.py index a2ce4b8..3f29fbe 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/llm.py +++ b/backend/norm/src/norm/ioa_observe/handlers/llm.py @@ -13,9 +13,9 @@ hasLLMCall, usesLLM, ) -from scope_ontology.models.nodes import LLM, LLMCall +from scope_ontology.models.nodes import LLM, LLMCall, Session, Tool -from ..fields import _hash_id +from ..fields import _hash_id, get_event_session_id, parse_json_object from ..registry import Registry from ..trajectory import add_final_state_transition, add_initial_state from .agent import _own_agent_id @@ -39,7 +39,9 @@ def _ensure_llm_call( return None agent_call, session_id, _, event_time = ensured agent_id = _own_agent_id(registry, agent_call) or str(event_attrs.get("agent.name") or "") - span_id = str(event_attrs.get("llm.call.id") or event.get("SpanId") or event_attrs.get("span.id") or "") + span_id = str( + event_attrs.get("llm.call.id") or event.get("SpanId") or event_attrs.get("span.id") or "" + ) # llm.name is "/" (as in "azure/gpt-4o"); handle_chat gets them separately. provider, _, model = llm_name.partition("/") @@ -113,3 +115,40 @@ def handle_llm_completed( input_text, str(event_attrs.get("llm.output") or ""), ) + + +def handle_llm_tools_presented( + event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry +) -> None: + record, attrs = event, event_attrs + meta = registry.metadata_for + content = parse_json_object(attrs.get("capability.content") or {}, "capability.content") + owner_span = str(attrs.get("ioa_observe.agent.span_id") or record.get("SpanId") or "") + for item in content.get("tools", []): + if not isinstance(item, dict): + continue + function = item.get("function") or item + if not isinstance(function, dict): + raise ValueError("Presented tool function must be an object") + tool_name = str(function.get("name") or "") + if tool_name: + tool = registry.add( + Tool( + id=tool_name, + name=tool_name, + description=str(function.get("description") or ""), + ) + ) + description = str(function.get("description") or "") + if description and not tool.description: + registry.upsert(tool.model_copy(update={"description": description})) + session = registry.get(Session, get_event_session_id(record, attrs)) + if session is not None: + meta(session)["presentedToolIds"] = sorted( + set(meta(session).get("presentedToolIds", []) + [tool_name]) + ) + if owner_span: + meta(tool)["ownerSpanIds"] = sorted( + set(meta(tool).get("ownerSpanIds", []) + [owner_span]) + ) + return diff --git a/backend/norm/src/norm/ioa_observe/handlers/mcp.py b/backend/norm/src/norm/ioa_observe/handlers/mcp.py index 861f3c0..0d48885 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/mcp.py +++ b/backend/norm/src/norm/ioa_observe/handlers/mcp.py @@ -10,6 +10,7 @@ from scope_ontology.models.edges import hasFinalState, hasInitialState, usesTool from scope_ontology.models.nodes import AgentCall, Tool, ToolCall +from ..fields import get_event_application_id from ..registry import Registry from ..trajectory import add_final_state_transition, add_initial_state from .agent import _own_agent_id @@ -36,12 +37,12 @@ def handle_mcp_tools_discovered( """Ensure the MAS and Session, then create every tool in ``capability.content`` and link each to the agent that discovered them (``usesTool``). - Skipped without an ``ioa_observe.agent.span_id`` or a session. The MAS is named by the - first part of ``session.id`` (``_``). The event names the agent only by + Skipped without an ``ioa_observe.agent.span_id`` or a session. The MAS is the originating + application. The event names the agent only by its span id, but an Agent is keyed on its name, so the agent is the one of the AgentCall of - that span. If that call does not exist yet, the tools are created and not linked.""" + that span. If that call does not exist yet, ownership is retained for reconciliation.""" agent_span_id = str(event_attrs.get("ioa_observe.agent.span_id") or "") - mas_name = str(event_attrs.get("session.id") or "").partition("_")[0] + mas_name = get_event_application_id(event, event_attrs) if not agent_span_id or not mas_name: return session = _ensure_session_and_mas(event, {**event_attrs, "application_id": mas_name}, registry) @@ -60,6 +61,12 @@ def handle_mcp_tools_discovered( tool, _ = registry.add_if_not_exists(Tool(id=name, name=name, description=description)) if description and not tool.description: # created earlier by a tool event, without one tool.description = description # type: ignore[attr-defined] + registry.metadata_for(tool)["ownerSpanIds"] = sorted( + set(registry.metadata_for(tool).get("ownerSpanIds", []) + [agent_span_id]) + ) + registry.metadata_for(session)["presentedToolIds"] = sorted( + set(registry.metadata_for(session).get("presentedToolIds", []) + [tool.id]) + ) if agent_id: registry.add_edge(usesTool(source_id=agent_id, target_id=tool.id)) @@ -93,11 +100,13 @@ def _ensure_mcp_tool_call( tool_name = _message_tool_name(message_input) source_name = str(event_attrs.get("source.agent") or "") source_span_id = str( - event_attrs.get("source.agent.span.id") or event_attrs.get("ioa_observe.agent.span_id") or "" + event_attrs.get("source.agent.span.id") + or event_attrs.get("ioa_observe.agent.span_id") + or "" ) if not tool_name or not source_name or not source_span_id: return None - mas_name = str(event_attrs.get("session.id") or "").partition("_")[0] + mas_name = get_event_application_id(event, event_attrs) agent_attrs = { **event_attrs, "application_id": event_attrs.get("application_id") or mas_name, @@ -111,7 +120,7 @@ def _ensure_mcp_tool_call( **agent_attrs, "tool.name": tool_name, "ioa_observe.agent.span_id": source_span_id, - # the message id is the id of the tool call: tool.started/completed carry it as tool.call.id + # MCP and tool lifecycle events share the same natural call id. "tool.call.id": event_attrs.get("message.id"), }, registry, diff --git a/backend/norm/src/norm/ioa_observe/handlers/session.py b/backend/norm/src/norm/ioa_observe/handlers/session.py index 7d17ab1..75433a2 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/session.py +++ b/backend/norm/src/norm/ioa_observe/handlers/session.py @@ -68,8 +68,8 @@ def handle_session_end( span, span_attrs, ) - session.endTime = ended_at # type: ignore[attr-defined] - session.duration = max(0.0, (ended_at - session.startTime) * 1000.0) # type: ignore[attr-defined] + session.endTime = max(session.endTime, ended_at) # type: ignore[attr-defined] + session.duration = max(0.0, (session.endTime - session.startTime) * 1000.0) # type: ignore[attr-defined] end_success = get_span_success(span, span_attrs) if end_success is not None: session.success = end_success # type: ignore[attr-defined] diff --git a/backend/norm/src/norm/ioa_observe/handlers/topology.py b/backend/norm/src/norm/ioa_observe/handlers/topology.py index 79f466f..e454b44 100644 --- a/backend/norm/src/norm/ioa_observe/handlers/topology.py +++ b/backend/norm/src/norm/ioa_observe/handlers/topology.py @@ -17,7 +17,7 @@ hasToolCall, usesTool, ) -from scope_ontology.models.nodes import MAS, Agent, AgentCall, Session, State, ToolCall +from scope_ontology.models.nodes import MAS, AgentCall, Session, State, ToolCall from ..fields import ( _hash_id, @@ -27,30 +27,29 @@ ) from ..registry import Registry from ..trajectory import add_final_state_transition, add_initial_state -from .agent import _agent_id_for, _ensure_agent, _ensure_mas_call, _own_agent_id +from .agent import _agent_id_for, _ensure_agent, _ensure_mas_call def handle_topology_edge_updated( event: dict[str, Any], event_attrs: dict[str, Any], registry: Registry ) -> None: - """The handoff from one agent to another: an ``executesAgent`` edge from the source's - AgentCall to the target Agent. - - The source is the AgentCall of ``source.agent.span.id``; the target is named only - (``target.agent``), so it is the Agent of that name. Nothing is created: the agents' - own events do that, and an edge to something that is not an agent (an MCP server, say) - matches nothing.""" - source_span_id = str(event_attrs.get("source.agent.span.id") or "") - target_name = str(event_attrs.get("target.agent") or "") - session_id = get_event_session_id(event, event_attrs) - if not source_span_id or not target_name or not session_id: + record, attrs = event, event_attrs + meta = registry.metadata_for + if attrs.get("network.protocol.name") == "mcp" or attrs.get("topology.edge.kind") in ( + "tool_call", + "mcp_message", + ): return - source = registry.find( - AgentCall, lambda call: call.spanId == source_span_id and call.sessionId == session_id - ) - target = registry.find(Agent, lambda agent: agent.name == target_name) - if source is not None and target is not None and target.id != _own_agent_id(registry, source): - registry.add_edge(executesAgent(source_id=source.id, target_id=target.id)) + source = str(attrs.get("source.agent.span.id") or "") + target = str(attrs.get("target.agent") or "") + session = _ensure_session_and_mas(record, attrs, registry) + if session is not None and source and target: + pending = meta(session).setdefault("handoffs", []) + hint = {"source": source, "target": target} + if hint not in pending: + pending.append(hint) + pending.sort(key=lambda item: (item["source"], item["target"])) + return def _ensure_session_and_mas( @@ -98,6 +97,14 @@ def handle_topology_session_started( """Ensure the Session and its MAS, and give the Session its initial State.""" session = _ensure_session_and_mas(event, event_attrs, registry) if session is not None: + start = min(session.startTime, get_event_start_time(event, event_attrs)) + session = session.model_copy( + update={ + "startTime": start, + "duration": (session.endTime - start) * 1000, + } + ) + registry.upsert(session) _ensure_session_initial_state(registry, session) @@ -118,7 +125,9 @@ def handle_topology_session_completed( # Session.endTime only ever moves forward, so it is the latest completion seen so far # (it starts at startTime, hence the check for no final State yet). - is_latest = registry.edge_target(hasFinalState, session.id) is None or end_time > session.endTime + is_latest = ( + registry.edge_target(hasFinalState, session.id) is None or end_time > session.endTime + ) if end_time > session.endTime: session.endTime = end_time # type: ignore[attr-defined] session.duration = (end_time - session.startTime) * 1000.0 # type: ignore[attr-defined] @@ -163,8 +172,7 @@ def _ensure_agent_call( # The Agent is keyed on the agent's name, so the Agent that an a2a.agent_card or a handoff # created under that name is the one reused. (The same name in another MAS is another # Agent: see _agent_id_for.) The AgentCall is what carries the span id. - mas_prefix = str(event_attrs.get("session.id") or "").partition("_")[0] - mas_ids = (application_id, mas_prefix) + mas_ids = (application_id,) agent = _ensure_agent(registry, _agent_id_for(registry, agent_id, mas_ids), agent_id, mas_ids) if application_id: registry.add_edge(belongsToMAS(source_id=agent.id, target_id=application_id)) diff --git a/backend/norm/src/norm/ioa_observe/heuristics/container_boundary_states.py b/backend/norm/src/norm/ioa_observe/heuristics/container_boundary_states.py index cecb39e..da7d365 100644 --- a/backend/norm/src/norm/ioa_observe/heuristics/container_boundary_states.py +++ b/backend/norm/src/norm/ioa_observe/heuristics/container_boundary_states.py @@ -20,38 +20,54 @@ from scope_ontology.models.nodes import AgentCall, MASCall, Session from ..registry import Registry -from ..trajectory import adopt_boundary_states +from ..trajectory import _clear_boundaries, adopt_boundary_states def assign_container_boundary_states(registry: Registry) -> None: - for mas_call in registry.all_of(MASCall): - children = [ - registry.get(AgentCall, target_id) - for target_id in registry.edges_from(hasAgentCall, mas_call.id) - ] - ordered = sorted((c for c in children if c is not None), key=lambda c: c.startTime) - if not ordered: - continue - initial_state_id = registry.edge_target(hasInitialState, ordered[0].id) - final_state_id = registry.edge_target(hasFinalState, ordered[-1].id) - if initial_state_id is None or final_state_id is None: - continue - adopt_boundary_states( - registry, mas_call.id, mas_call.sessionId, initial_state_id, final_state_id - ) - - for session in registry.all_of(Session): - children = [ - registry.get(MASCall, target_id) - for target_id in registry.edges_from(hasMASCall, session.id) - ] - ordered = sorted((c for c in children if c is not None), key=lambda c: c.startTime) - if not ordered: - continue - initial_state_id = registry.edge_target(hasInitialState, ordered[0].id) - final_state_id = registry.edge_target(hasFinalState, ordered[-1].id) - if initial_state_id is None or final_state_id is None: - continue - adopt_boundary_states( - registry, session.id, session.sessionId, initial_state_id, final_state_id - ) + for cls, child_cls, edge in ( + (MASCall, AgentCall, hasAgentCall), + (Session, MASCall, hasMASCall), + ): + for container in registry.all_of(cls): + children = [ + registry.get(child_cls, node_id) + for node_id in registry.edges_from(edge, container.id) + ] + children = [child for child in children if child is not None] + if not children: + continue + first = min(children, key=lambda node: (node.startTime, node.id)) + last = max(children, key=lambda node: (node.endTime, node.id)) + initial = registry.edge_target(hasInitialState, first.id) + final = registry.edge_target(hasFinalState, last.id) + if initial and final: + _clear_boundaries(registry, container.id) + adopt_boundary_states(registry, container.id, container.sessionId, initial, final) + + +def update_container_times(registry: Registry) -> None: + meta = registry.metadata_for + for cls, child_cls, edge in ( + (MASCall, AgentCall, hasAgentCall), + (Session, MASCall, hasMASCall), + ): + for container in registry.all_of(cls): + children = [registry.get(child_cls, i) for i in registry.edges_from(edge, container.id)] + children = [c for c in children if c is not None] + if children: + start = min(container.startTime, *(c.startTime for c in children)) + end = max(container.endTime, *(c.endTime for c in children)) + registry.upsert( + container.model_copy( + update={ + "startTime": start, + "endTime": end, + "duration": (end - start) * 1000, + **( + {"spanId": meta(container).get("sessionSpanId", "")} + if cls is Session + else {} + ), + } + ) + ) diff --git a/backend/norm/src/norm/ioa_observe/reconciliation.py b/backend/norm/src/norm/ioa_observe/reconciliation.py new file mode 100644 index 0000000..90a647f --- /dev/null +++ b/backend/norm/src/norm/ioa_observe/reconciliation.py @@ -0,0 +1,299 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 +"""Persisted lifecycle and cross-record relationship reconciliation.""" + +from __future__ import annotations + +import json +import logging +from dataclasses import replace +from typing import Any + +from scope_ontology.models.edges import ( + belongsToMASCall, + correspondsToTransition, + executesAgent, + executesLLM, + executesSession, + executesTool, + hasAgentCall, + hasFinalState, + hasInitialState, + hasLLMCall, + hasMASCall, + hasState, + hasToolCall, + usesLLM, + usesTool, +) +from scope_ontology.models.nodes import ( + Agent, + AgentCall, + LLMCall, + MASCall, + ProcessingCall, + Session, + State, + Tool, + ToolCall, + Transition, +) + +from .fields import _hash_id, get_span_session_id +from .handlers.agent import _agent_id_for, _own_agent_id +from .handlers.graph import reconcile_graph_agents +from .handlers.mcp import _message_tool_name +from .heuristics.container_boundary_states import update_container_times +from .registry import HANDOFF_SOURCES_PROP, Registry, SpanContext + +logger = logging.getLogger(__name__) +_CALLS = (AgentCall, LLMCall, ToolCall) + + +def span_context(record: dict[str, Any], attrs: dict[str, Any]) -> SpanContext | None: + session_id = get_span_session_id(record, attrs) + span_id = str(record.get("SpanId") or "") + trace_id = str(record.get("TraceId") or (f"session:{session_id}" if session_id else "")) + if not trace_id or not span_id: + return None + return SpanContext( + trace_id=trace_id, + span_id=span_id, + parent_span_id=str(record.get("ParentSpanId") or ""), + owner_span_id=str(attrs.get("ioa_observe.agent.span_id") or ""), + session_id=session_id, + ) + + +def event_context(record: dict[str, Any], attrs: dict[str, Any]) -> SpanContext | None: + if attrs.get("event.name") not in ( + "mcp.message.sent", + "mcp.message.received", + "mcp.message.completed", + ): + return None + operation = str(attrs.get("operation.name") or "") + if operation and operation != "tools/call": + return None + if not operation and not _message_tool_name(attrs.get("message.input")): + return None + source_trace = str(attrs.get("source.agent.trace.id") or "") + source_owner = ( + attrs.get("source.agent.span.id") + if not source_trace or source_trace == record.get("TraceId") + else "" + ) + owner = attrs.get("ioa_observe.agent.span_id") or source_owner + if not owner: + return None + return span_context( + record, + { + **attrs, + "ioa_observe.agent.span_id": owner, + }, + ) + + +def merge_span_context(context: SpanContext, previous: SpanContext | None) -> SpanContext: + if previous is None: + return context + return replace( + context, + parent_span_id=context.parent_span_id or previous.parent_span_id, + owner_span_id=context.owner_span_id or previous.owner_span_id, + session_id=context.session_id or previous.session_id, + ) + + +def remember_span_ancestry( + record: dict[str, Any], attrs: dict[str, Any], registry: Registry +) -> None: + context = span_context(record, attrs) + if context is not None: + key = (context.trace_id, context.span_id) + registry.span_contexts[key] = merge_span_context(context, registry.span_contexts.get(key)) + + +def _owning_agent_call( + registry: Registry, child: LLMCall | ToolCall, agents: list[AgentCall] +) -> AgentCall | None: + meta = registry.metadata_for + owner = str(meta(child).get("ownerSpanId") or child.parentSpanId or "") + trace_id = str(meta(child).get("traceId") or "") + context_trace = trace_id or f"session:{child.sessionId}" + candidates = { + agent.spanId: agent + for agent in agents + if agent.sessionId == child.sessionId + and not (trace_id and meta(agent).get("traceId") and meta(agent)["traceId"] != trace_id) + } + visited: set[str] = set() + while owner and owner not in visited: + visited.add(owner) + ancestor = registry.span_contexts.get((context_trace, owner)) + if ancestor is not None and ancestor.session_id not in ("", child.sessionId): + return None + if owner in candidates: + return candidates[owner] + owner = (ancestor.owner_span_id or ancestor.parent_span_id) if ancestor is not None else "" + return None + + +def handoff_sources(registry: Registry) -> dict[str, list[str]]: + return { + call.id: list(registry.extras.get(("AgentCall", call.id), {}).get(HANDOFF_SOURCES_PROP, [])) + for call in registry.all_of(AgentCall) + } + + +def reconcile_relationships(registry: Registry) -> None: + meta = registry.metadata_for + signals = handoff_sources(registry) + agents = registry.all_of(AgentCall) + for cls, contains, execution_edge, uses in ( + (LLMCall, hasLLMCall, executesLLM, usesLLM), + (ToolCall, hasToolCall, executesTool, usesTool), + ): + for child in registry.all_of(cls): + parent = _owning_agent_call(registry, child, agents) + if parent is None: + application = registry.edge_target(executesSession, child.sessionId) or "" + owner_agent = str(meta(child).get("ownerAgentId") or "") + agent_id = ( + _agent_id_for(registry, owner_agent, (application,)) if owner_agent else "" + ) + target = registry.edge_target(execution_edge, child.id) + if target and registry.get(Agent, agent_id) is not None: + registry.add_edge(uses(source_id=agent_id, target_id=target)) + continue + registry.add_edge(contains(source_id=parent.id, target_id=child.id)) + agent = _own_agent_id(registry, parent) + target = registry.edge_target(execution_edge, child.id) + if agent and target: + registry.add_edge(uses(source_id=agent, target_id=target)) + registry.remove_edges_where( + lambda edge, source, _: edge in (usesTool, usesLLM) and registry.get(Agent, source) is None + ) + for tool in registry.all_of(Tool): + for owner in meta(tool).get("ownerSpanIds", []): + for call in agents: + agent = _own_agent_id(registry, call) + if call.spanId == owner and agent: + registry.add_edge(usesTool(source_id=agent, target_id=tool.id)) + reconcile_graph_agents(registry) + for state in registry.all_of(State): + receiver = meta(state).get("receiverName") + owner = meta(state).get("ownerSpanId") + candidates = [] + for call in agents: + agent = registry.get(Agent, _own_agent_id(registry, call) or "") + if call.sessionId == state.sessionId and ( + (owner and call.spanId == owner) + or (receiver and agent is not None and agent.name == receiver) + ): + candidates.append(call) + if candidates: + event_time = meta(state).get("eventTime", 0) + call = min( + candidates, + key=lambda a: ( + a.startTime < event_time, + abs(a.startTime - event_time), + a.id, + ), + ) + registry.remove_edges_where( + lambda edge, _, target: edge is hasState and target == state.id + ) + registry.add_edge(hasState(source_id=call.id, target_id=state.id)) + for session in registry.all_of(Session): + for hint in meta(session).get("handoffs", []): + source = next( + (a for a in agents if a.spanId == hint["source"] and a.sessionId == session.id), + None, + ) + if source is None: + continue + targets = [ + a + for a in agents + if a.sessionId == session.id + and a.id != source.id + and (agent := registry.get(Agent, _own_agent_id(registry, a) or "")) is not None + and agent.name == hint["target"] + ] + if targets: + target = min( + targets, + key=lambda a: ( + a.startTime < source.startTime, + abs(a.startTime - source.startTime), + a.id, + ), + ) + signals[target.id] = sorted(set(signals.get(target.id, []) + [source.spanId])) + for edge, source, target in list(registry.edges): + if edge is hasAgentCall: + registry.add_edge(belongsToMASCall(source_id=target, target_id=source)) + + for call_id, sources in signals.items(): + if sources: + registry.extras.setdefault(("AgentCall", call_id), {})[HANDOFF_SOURCES_PROP] = sources + update_container_times(registry) + + +def strip_derived(registry: Registry) -> None: + """Drop everything ``apply_heuristics`` derives, so it can be rebuilt.""" + # Session events own native states until complete child boundaries can replace them. + containers = { + n.id + for cls, edge in ((Session, hasMASCall), (MASCall, hasAgentCall)) + for n in registry.all_of(cls) + if registry.edges_from(edge, n.id) + and not ( + cls is Session + and registry.edge_target(hasInitialState, n.id) + == _hash_id("state", n.id, "initial", "") + ) + } + bridges = {n.id for n in registry.all_of(ProcessingCall)} + owners = containers | bridges + transitions = { + target + for etype, source, target in registry.edges + if etype is correspondsToTransition and source in owners + } + for bridge_id in bridges: + registry.remove_node(ProcessingCall, bridge_id) + for transition_id in transitions: + registry.remove_node(Transition, transition_id) + gone = bridges | transitions + boundary_edges = (hasInitialState, hasFinalState, hasState) + registry.remove_edges_where( + lambda etype, source, target: ( + source in gone or target in gone or (etype in boundary_edges and source in containers) + ) + ) + + +def handoff_spans(registry: Registry) -> list[dict[str, Any]]: + """Minimal ``*.agent`` spans carrying only the handoff signal, rebuilt from stored state.""" + signals = handoff_sources(registry) + spans: list[dict[str, Any]] = [] + for call in registry.all_of(AgentCall): + sources = signals.get(call.id) + agent_id = registry.edge_target(executesAgent, call.id) + if not sources or agent_id is None: + continue + spans.append( + { + "SpanName": "handoff.agent", + "SpanId": call.spanId, + "SpanAttributes": { + "agent_id": agent_id, + "ioa_observe.handoff.source.span_ids": json.dumps(sources), + }, + } + ) + return spans diff --git a/backend/norm/src/norm/ioa_observe/registry.py b/backend/norm/src/norm/ioa_observe/registry.py index c2e2f52..ef1a0b0 100644 --- a/backend/norm/src/norm/ioa_observe/registry.py +++ b/backend/norm/src/norm/ioa_observe/registry.py @@ -11,17 +11,39 @@ from __future__ import annotations from collections.abc import Callable -from typing import TypeVar, cast +from dataclasses import dataclass +from typing import Any, TypeVar, cast from scope_ontology.models.base import KGBase T = TypeVar("T", bound=KGBase) +MetadataGetter = Callable[[KGBase], dict[str, Any]] +NORMALIZATION_PROP = "rtNormalization" +HANDOFF_SOURCES_PROP = "handoffSourceSpanIds" + + +@dataclass(frozen=True) +class SpanContext: + """Private ancestry evidence, not an ontology node or raw telemetry buffer.""" + + trace_id: str + span_id: str + parent_span_id: str + owner_span_id: str + session_id: str class Registry: def __init__(self) -> None: self.nodes: dict[tuple[str, str], KGBase] = {} self.edges: dict[tuple[type, str, str], KGBase] = {} + self.extras: dict[tuple[str, str], dict[str, Any]] = {} + self.span_contexts: dict[tuple[str, str], SpanContext] = {} + + def metadata_for(self, node: KGBase) -> dict[str, Any]: + """Persist reconciliation evidence without changing ontology models.""" + extra = self.extras.setdefault((type(node).__name__, node.id), {}) + return extra.setdefault(NORMALIZATION_PROP, {}) def get(self, cls: type[T], node_id: str) -> T | None: return cast("T | None", self.nodes.get((cls.__name__, node_id))) @@ -55,6 +77,7 @@ def add_edge(self, edge: KGBase) -> None: def remove_node(self, cls: type[KGBase], node_id: str) -> None: self.nodes.pop((cls.__name__, node_id), None) + self.extras.pop((cls.__name__, node_id), None) def remove_edges_where(self, predicate: Callable[[type, str, str], bool]) -> None: """Drop every edge for which predicate(edge_class, source_id, target_id) holds.""" @@ -104,6 +127,10 @@ def merge_node_into(self, cls: type[KGBase], drop_id: str, keep_id: str) -> None moved = edge.model_copy(update={"source_id": new_source, "target_id": new_target}) self.edges.setdefault((etype, new_source, new_target), moved) self.nodes.pop((cls.__name__, drop_id), None) + extra = self.extras.pop((cls.__name__, drop_id), {}) + kept = self.extras.setdefault((cls.__name__, keep_id), {}) + for key, value in extra.items(): + kept.setdefault(key, value) def all_of(self, cls: type[T]) -> list[T]: """Return every node of this class.""" diff --git a/backend/norm/src/norm/ioa_observe/stream.py b/backend/norm/src/norm/ioa_observe/stream.py index 2f8b612..0a41135 100644 --- a/backend/norm/src/norm/ioa_observe/stream.py +++ b/backend/norm/src/norm/ioa_observe/stream.py @@ -1,10 +1,10 @@ # Copyright (c) 2026 Cisco Systems, Inc. and its affiliates # SPDX-License-Identifier: Apache-2.0 -"""Stateless, span-at-a-time normalization against a graph database. +"""Stateless, record-at-a-time normalization against a graph database. ``build_kg`` needs the whole span array. ``StreamNormalizer.process`` takes a -single span and keeps nothing between calls: the graph already stored in the -database *is* the state. For each span it +single span and keeps nothing between calls: the stored graph and adapter-managed +trace context provide its state. For each span it 1. loads what it needs from the database through a ``GraphReader`` (the span's session subgraph plus the structural nodes the span touches), @@ -14,48 +14,52 @@ 4. returns only the difference against what was loaded, as a ``StreamDelta``, for the caller to write back. +Transport ancestry is loaded separately, including spans without session ids. +It is retained through ``StreamDelta.span_contexts``, never as ontology executions. +Its lifetime and durability depend on the adapter. + The heuristics' derived items (bridge ProcessingCalls, container-boundary Transitions and States) are dropped from the loaded graph and rebuilt every time, so a late-arriving span replaces stale ones instead of piling up. -The one thing a node cannot rebuild is a span's handoff signal -(``ioa_observe.handoff.source.span_ids``): ontology models forbid extra -fields, so it is carried on the AgentCall payload as the non-ontology -``handoffSourceSpanIds`` property. +Reconciliation evidence is persisted as ``rtNormalization`` extras because +ontology models forbid extra fields. Legacy ``handoffSourceSpanIds`` on +AgentCall payloads remain supported. """ from __future__ import annotations import json from collections.abc import Sequence +from copy import deepcopy from dataclasses import dataclass, field from typing import Any, Literal, Protocol from pydantic import ValidationError from scope_ontology.models import edges as edge_models from scope_ontology.models import nodes as node_models -from scope_ontology.models.base import KGBase -from scope_ontology.models.edges import ( - correspondsToTransition, - executesAgent, - hasFinalState, - hasInitialState, - hasState, +from scope_ontology.models.nodes import ( + Agent, + Session, ) -from scope_ontology.models.nodes import Agent, AgentCall, MASCall, ProcessingCall, Session, Transition +from ..verifier import PROPERTY_DOMAIN_RANGE from .build import apply_heuristics, dispatch_span, serialize_registry from .events import dispatch_event -from .fields import get_event_attrs, get_event_session_id, get_span_attrs, get_span_session_id -from .heuristics.handoff import _source_span_ids -from .registry import Registry -from ..verifier import PROPERTY_DOMAIN_RANGE +from .fields import ( + _hash_id, + get_event_session_id, + get_span_session_id, + normalize_record, + parse_json_object, +) +from .handlers.topology import _ensure_session_and_mas +from .reconciliation import event_context, merge_span_context, span_context +from .registry import HANDOFF_SOURCES_PROP, NORMALIZATION_PROP, Registry, SpanContext Kind = Literal["span", "event"] KINDS = ("span", "event") -HANDOFF_SOURCES_PROP = "handoffSourceSpanIds" - @dataclass class StreamDelta: @@ -64,16 +68,22 @@ class StreamDelta: ``nodes``/``edges`` are new or modified items; ``removed_*`` are items the database holds that the recomputed graph no longer contains. Edges carry ``from_type``/``to_type`` so they can be written without their endpoint - nodes being part of the same delta. + nodes being part of the same delta. ``span_contexts`` is private ancestry + evidence that adapters must retain separately from ontology nodes. """ nodes: list[dict[str, Any]] = field(default_factory=list) edges: list[dict[str, Any]] = field(default_factory=list) removed_edges: list[dict[str, Any]] = field(default_factory=list) removed_nodes: list[dict[str, Any]] = field(default_factory=list) + span_contexts: list[SpanContext] = field(default_factory=list) class GraphReader(Protocol): + def load_span_contexts(self, trace_id: str) -> list[SpanContext]: + """Load private ancestry evidence, including sessionless transport spans.""" + ... + def load( self, session_id: str, node_ids: Sequence[str] ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: @@ -97,11 +107,8 @@ def _edge_key(edge: dict[str, Any]) -> tuple[str, str, str, str]: return ("edge", str(edge["edge_type"]), str(edge["from_id"]), str(edge["to_id"])) -def _rehydrate( - nodes: list[dict[str, Any]], edges: list[dict[str, Any]] -) -> tuple[Registry, dict[str, list[str]]]: +def _rehydrate(nodes: list[dict[str, Any]], edges: list[dict[str, Any]]) -> Registry: registry = Registry() - signals: dict[str, list[str]] = {} for raw in nodes: cls = getattr(node_models, str(raw["node_type"]), None) if cls is None: @@ -110,64 +117,25 @@ def _rehydrate( try: node = cls.model_validate(data) except ValidationError: - continue # partial node (only an id) written ahead of its own span; that span will fill it in + # A partial node written ahead of its span will be filled in by that span. + continue registry.upsert(node) - sources = raw.get(HANDOFF_SOURCES_PROP) - if sources: - signals[node.id] = [str(item) for item in sources] + extra = { + key: deepcopy(raw[key]) + for key in (NORMALIZATION_PROP, HANDOFF_SOURCES_PROP) + if raw.get(key) + } + if extra: + registry.extras[(type(node).__name__, node.id)] = extra for raw in edges: cls = getattr(edge_models, str(raw["edge_type"]), None) if cls is None: continue registry.add_edge(cls(source_id=str(raw["from_id"]), target_id=str(raw["to_id"]))) - return registry, signals - - -def _strip_derived(registry: Registry) -> None: - """Drop everything ``apply_heuristics`` derives, so it can be rebuilt.""" - containers = {n.id for n in registry.all_of(Session)} | {n.id for n in registry.all_of(MASCall)} - bridges = {n.id for n in registry.all_of(ProcessingCall)} - owners = containers | bridges - transitions = { - target - for etype, source, target in registry.edges - if etype is correspondsToTransition and source in owners - } - for bridge_id in bridges: - registry.remove_node(ProcessingCall, bridge_id) - for transition_id in transitions: - registry.remove_node(Transition, transition_id) - gone = bridges | transitions - boundary_edges = (hasInitialState, hasFinalState, hasState) - registry.remove_edges_where( - lambda etype, source, target: ( - source in gone or target in gone or (etype in boundary_edges and source in containers) - ) - ) + return registry -def _handoff_spans(registry: Registry, signals: dict[str, list[str]]) -> list[dict[str, Any]]: - """Minimal ``*.agent`` spans carrying only the handoff signal, rebuilt from stored state.""" - spans: list[dict[str, Any]] = [] - for call in registry.all_of(AgentCall): - sources = signals.get(call.id) - agent_id = registry.edge_target(executesAgent, call.id) - if not sources or agent_id is None: - continue - spans.append( - { - "SpanName": "handoff.agent", - "SpanId": call.spanId, - "SpanAttributes": { - "agent_id": agent_id, - "ioa_observe.handoff.source.span_ids": json.dumps(sources), - }, - } - ) - return spans - - -def _endpoint_type(edge_type: str, side: int, node_id: str, node_types: dict[str, str]) -> str: +def _endpoint_type(edge_type: str, side: int, node_id: str, node_types: dict[str, set[str]]) -> str: """Type of an edge endpoint: its node's own type, else the one class the ontology allows there (``side`` 0 = domain, 1 = range). "" if ambiguous. @@ -175,28 +143,48 @@ def _endpoint_type(edge_type: str, side: int, node_id: str, node_types: dict[str emitted by a chat span before the agent span that creates the Agent -- in which case the database holds a partial (id-only) node until then. """ - if node_id in node_types: - return node_types[node_id] candidates = PROPERTY_DOMAIN_RANGE.get(edge_type, ([], []))[side] + actual = node_types.get(node_id, set()) + compatible = { + name + for name in actual + if any(base.__name__ in candidates for base in getattr(node_models, name).__mro__) + } + if len(compatible) == 1: + return next(iter(compatible)) + if len(actual) == 1: + return next(iter(actual)) + if len(actual) > 1: + raise ValueError(f"Ambiguous {edge_type} endpoint {node_id!r}: {sorted(actual)}") return str(candidates[0]) if len(candidates) == 1 else "" -def _snapshot(registry: Registry, signals: dict[str, list[str]]) -> dict[tuple[str, ...], dict[str, Any]]: +def _snapshot(registry: Registry) -> dict[tuple[str, ...], dict[str, Any]]: nodes, edges = serialize_registry(registry) - node_types = {str(n["id"]): str(n["node_type"]) for n in nodes} + node_types: dict[str, set[str]] = {} + for node in nodes: + node_types.setdefault(str(node["id"]), set()).add(str(node["node_type"])) + execution_types = { + (source, target): cls.__name__ + for cls, source, target in registry.edges + if cls.__name__.startswith("executes") and cls.__name__ != "executes" + } items: dict[tuple[str, ...], dict[str, Any]] = {} for node in nodes: - if node["node_type"] == "AgentCall" and signals.get(str(node["id"])): - node[HANDOFF_SOURCES_PROP] = signals[str(node["id"])] items[_node_key(node)] = node for edge in edges: - edge["from_type"] = _endpoint_type(edge["edge_type"], 0, str(edge["from_id"]), node_types) - edge["to_type"] = _endpoint_type(edge["edge_type"], 1, str(edge["to_id"]), node_types) + edge_type = edge["edge_type"] + if edge_type == "executes": + edge_type = execution_types.get((edge["from_id"], edge["to_id"]), edge_type) + edge["from_type"] = _endpoint_type(edge_type, 0, str(edge["from_id"]), node_types) + edge["to_type"] = _endpoint_type(edge_type, 1, str(edge["to_id"]), node_types) items[_edge_key(edge)] = edge return items -def _diff(before: dict[tuple[str, ...], dict[str, Any]], after: dict[tuple[str, ...], dict[str, Any]]) -> StreamDelta: +def _diff( + before: dict[tuple[str, ...], dict[str, Any]], after: dict[tuple[str, ...], dict[str, Any]] +) -> StreamDelta: delta = StreamDelta() for key, item in after.items(): if before.get(key) != item: @@ -212,109 +200,168 @@ def __init__(self, reader: GraphReader) -> None: self._reader = reader def process(self, record: dict[str, Any], kind: Kind = "span") -> StreamDelta: - """Normalize one item against the stored graph and return what changed. - - ``kind`` says what ``record`` is: ``"span"`` (an OTel span, the default) or - ``"event"`` (an OTel log record). Events are routed by ``event.name`` (see events.py); their - handlers are stubs for now, so they change nothing. - """ - if kind == "event": - return self._process_event(record) - elif kind == "span": - return self._process_trace(record) - else: - raise ValueError(f"Unknown kind {kind!r}, expected one of {KINDS}") - - def _process_event(self, event: dict[str, Any]) -> StreamDelta: - event_attrs = get_event_attrs(event) - session_id = get_event_session_id(event, event_attrs) - - # Dry run on an empty registry to learn which nodes this event touches. - scratch = Registry() - if not dispatch_event(event, event_attrs, scratch): - return StreamDelta() - touched = {node.id for node in scratch.nodes.values()} - for _etype, source_id, target_id in scratch.edges: - touched.update((source_id, target_id)) - # Agents are keyed on a span id but matched by name (an a2a.agent_card has no span id), - # so the reader is also handed their names: it loads the Agents called that. - touched.update(node.name for node in scratch.all_of(Agent)) - - # The dry run can't see what depends on stored state (e.g. an edge to an agent found - # through an existing AgentCall), so if the real run touches more, load those too - # and run again -- otherwise their stored copies would be missed and re-sent. - for _ in range(2): - stored_nodes, stored_edges = self._reader.load(session_id, sorted(touched)) - registry, signals = _rehydrate(stored_nodes, stored_edges) - before = _snapshot(registry, signals) - dispatch_event(event, event_attrs, registry) - after = _snapshot(registry, signals) - produced: set[str] = set() - for key, item in after.items(): - if before.get(key) != item: - produced.update( - (str(item["from_id"]), str(item["to_id"])) - if key[0] == "edge" - else (str(item["id"]),) - ) - if produced <= touched: - break - touched |= produced - return _diff(before, after) - - def _process_trace(self, span: dict[str, Any]) -> StreamDelta: - span_attrs = get_span_attrs(span) - session_id = get_span_session_id(span, span_attrs) - - # Dry run on an empty registry to learn which nodes this span touches. + if kind not in ("span", "event"): + raise ValueError(f"Unknown kind {kind!r}") + record, attrs = normalize_record(record, kind) + session_id = ( + get_span_session_id(record, attrs) + if kind == "span" + else get_event_session_id(record, attrs) + ) + context = span_context(record, attrs) if kind == "span" else event_context(record, attrs) + trace_id = str(record.get("TraceId") or (f"session:{session_id}" if session_id else "")) + contexts = ( + { + (item.trace_id, item.span_id): item + for item in self._reader.load_span_contexts(trace_id) + } + if trace_id + else {} + ) + delta = StreamDelta() + if context is not None: + key = (context.trace_id, context.span_id) + context = merge_span_context(context, contexts.get(key)) + if contexts.get(key) != context: + delta.span_contexts.append(context) + contexts[key] = context + sessions = ( + {session_id} + if session_id + else {item.session_id for item in contexts.values() if item.session_id} + ) + # A sessionless transport arriving last may complete ancestry in several sessions. + for current_session in sorted(sessions or {""}): + change = self._process(record, attrs, kind, current_session, contexts) + delta.nodes.extend(change.nodes) + delta.edges.extend(change.edges) + delta.removed_nodes.extend(change.removed_nodes) + delta.removed_edges.extend(change.removed_edges) + return delta + + def _process( + self, + record: dict[str, Any], + attrs: dict[str, Any], + kind: Kind, + session_id: str, + contexts: dict[tuple[str, str], SpanContext], + ) -> StreamDelta: + application = str(attrs.get("application_id") or "") + touched = {session_id, application, str(record.get("SpanId") or "")} + touched.update( + str(attrs.get(k) or "") + for k in ( + "agent_id", + "agent.name", + "source.agent", + "target.agent", + "ioa_observe.entity.name", + ) + ) + graph_data = ( + parse_json_object(attrs["gen_ai.ioa.graph"], "gen_ai.ioa.graph") + if attrs.get("gen_ai.ioa.graph") + else {} + ) + if not isinstance(graph_data.get("nodes", {}), dict): + raise ValueError("gen_ai.ioa.graph.nodes must be an object") + for node_id, node in graph_data.get("nodes", {}).items(): + touched.add(str(node_id)) + if isinstance(node, dict): + touched.add(str(node.get("name") or node_id)) + # Structural ids (tools, models, cards) need loading even before a session exists. scratch = Registry() - dispatch_span(span, scratch) - touched = {node.id for node in scratch.nodes.values()} - for _etype, source_id, target_id in scratch.edges: - touched.update((source_id, target_id)) - - stored_nodes, stored_edges = self._reader.load(session_id, sorted(touched)) - registry, signals = _rehydrate(stored_nodes, stored_edges) - - # Already processed (re-delivery / re-run): handlers are not order-independent, - # so replaying a span into a complete graph would change it. Make it a no-op. - span_id = str(span.get("SpanId") or "") - if span_id and any(getattr(n, "spanId", None) == span_id for n in registry.nodes.values()): - return StreamDelta() - - before = _snapshot(registry, signals) - - dispatch_span(span, registry) - if str(span.get("SpanName") or "").endswith(".agent"): - sources = _source_span_ids(span_attrs) - call = registry.find(AgentCall, lambda a: a.spanId == span_id) - if sources and call is not None: - signals[call.id] = sources - - _strip_derived(registry) - apply_heuristics(_handoff_spans(registry, signals), registry) - after = _snapshot(registry, signals) - - return _diff(before, after) + recognized = False + if kind == "span": + dispatch_span( + { + **record, + "SpanAttributes": {k: v for k, v in attrs.items() if k != "gen_ai.ioa.graph"}, + }, + scratch, + ) + else: + recognized = dispatch_event(record, attrs, scratch) + if not recognized or (not scratch.nodes and event_context(record, attrs) is None): + return StreamDelta() + touched.update(n.id for n in scratch.nodes.values()) + touched.update(n.name for n in scratch.all_of(Agent)) + for _, source, target in scratch.edges: + touched.update((source, target)) + stored_nodes, stored_edges = self._reader.load(session_id, sorted(touched - {""})) + pending_tools = { + tool_id + for node in stored_nodes + if node["node_type"] == "Session" and node["id"] == session_id + for tool_id in (node.get(NORMALIZATION_PROP) or {}).get("presentedToolIds", []) + } + structural_ids = {str(node["id"]) for node in stored_nodes if not node.get("sessionId")} + related_ids = pending_tools | structural_ids + if not related_ids <= touched: + touched.update(related_ids) + stored_nodes, stored_edges = self._reader.load(session_id, sorted(touched - {""})) + stored_nodes = [ + n for n in stored_nodes if not n.get("sessionId") or n["sessionId"] == session_id + ] + ids = {n["id"] for n in stored_nodes} + stored_edges = [e for e in stored_edges if e["from_id"] in ids and e["to_id"] in ids] + registry = _rehydrate(stored_nodes, stored_edges) + registry.span_contexts.update(contexts) + meta = registry.metadata_for + before = deepcopy(_snapshot(registry)) + event_id = str( + attrs.get("event.id") or _hash_id("event", json.dumps(record, sort_keys=True)) + ) + session = registry.get(Session, session_id) + if ( + kind == "event" + and session is not None + and event_id in meta(session).get("eventIds", []) + ): + apply_heuristics([], registry) + return _diff(before, _snapshot(registry)) + if kind == "span": + dispatch_span(record, registry) + else: + if recognized or attrs.get("event.name") in ( + "llm.tools.presented", + "a2a.message.completed", + ): + _ensure_session_and_mas(record, attrs, registry) + dispatch_event(record, attrs, registry) + session = registry.get(Session, session_id) + if session is not None: + meta(session)["eventIds"] = sorted( + set(meta(session).get("eventIds", []) + [event_id]) + ) + apply_heuristics([], registry) + return _diff(before, _snapshot(registry)) class InMemoryGraph: """A ``GraphReader`` plus writer over plain dicts; for tests and dry runs.""" - # Set on creation and never changed afterwards, same as the Neo4j store (endTime may advance). - IMMUTABLE_PROPS = frozenset({"startTime"}) - def __init__(self) -> None: self.nodes: dict[tuple[str, str], dict[str, Any]] = {} self.edges: dict[tuple[str, str, str], dict[str, Any]] = {} + self.span_contexts: dict[tuple[str, str], SpanContext] = {} + + def load_span_contexts(self, trace_id: str) -> list[SpanContext]: + return [item for (trace, _), item in self.span_contexts.items() if trace == trace_id] def load( self, session_id: str, node_ids: Sequence[str] ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: ids = set(node_ids) - by_id = {node["id"]: node for node in self.nodes.values()} - in_session = {i for i, n in by_id.items() if session_id and n.get("sessionId") == session_id} - named = {i for i, n in by_id.items() if n["node_type"] == "Agent" and n.get("name") in ids} + in_session = { + n["id"] for n in self.nodes.values() if session_id and n.get("sessionId") == session_id + } + named = { + n["id"] + for n in self.nodes.values() + if n["node_type"] == "Agent" and n.get("name") in ids + } edges = [ e for e in self.edges.values() @@ -327,10 +374,12 @@ def load( wanted = ( in_session | named | ids | {e["from_id"] for e in edges} | {e["to_id"] for e in edges} ) - nodes = [dict(by_id[i]) for i in wanted if i in by_id] + nodes = [dict(node) for node in self.nodes.values() if node["id"] in wanted] return nodes, [dict(e) for e in edges] def apply(self, delta: StreamDelta) -> None: + for context in delta.span_contexts: + self.span_contexts[(context.trace_id, context.span_id)] = context for edge in delta.removed_edges: self.edges.pop((edge["edge_type"], edge["from_id"], edge["to_id"]), None) for node in delta.removed_nodes: @@ -344,6 +393,6 @@ def apply(self, delta: StreamDelta) -> None: if existing is None: self.nodes[key] = dict(node) else: - existing.update({k: v for k, v in node.items() if k not in self.IMMUTABLE_PROPS}) + existing.update(node) for edge in delta.edges: self.edges[(edge["edge_type"], edge["from_id"], edge["to_id"])] = dict(edge) diff --git a/backend/norm/src/norm/ioa_observe/trajectory.py b/backend/norm/src/norm/ioa_observe/trajectory.py index 69b81fa..10a8f6d 100644 --- a/backend/norm/src/norm/ioa_observe/trajectory.py +++ b/backend/norm/src/norm/ioa_observe/trajectory.py @@ -6,6 +6,7 @@ from typing import Literal +from scope_ontology.models.base import KGBase from scope_ontology.models.edges import ( correspondsToTransition, executesProcessing, @@ -17,10 +18,19 @@ leadsTo, representsExecution, ) -from scope_ontology.models.nodes import Processing, ProcessingCall, State, Transition +from scope_ontology.models.nodes import ( + AgentCall, + LLMCall, + Processing, + ProcessingCall, + Session, + State, + ToolCall, + Transition, +) from .fields import _hash_id -from .registry import Registry +from .registry import MetadataGetter, Registry def add_initial_state( @@ -222,3 +232,94 @@ def adopt_boundary_states( registry.add_edge(leadsTo(source_id=transition.id, target_id=final_state_id)) registry.add_edge(correspondsToTransition(source_id=owner_id, target_id=transition.id)) registry.add_edge(representsExecution(source_id=transition.id, target_id=owner_id)) + + +def _clear_boundaries(registry: Registry, owner_id: str) -> None: + boundaries = { + target + for edge, source, target in registry.edges + if source == owner_id and edge in (hasInitialState, hasFinalState) + } + transitions = set(registry.edges_from(correspondsToTransition, owner_id)) + registry.remove_edges_where( + lambda edge, source, target: ( + source == owner_id + and ( + edge in (hasInitialState, hasFinalState, correspondsToTransition) + or (edge is hasState and target in boundaries) + ) + ) + ) + for transition_id in transitions: + if not registry.edge_source(correspondsToTransition, transition_id): + registry.remove_node(Transition, transition_id) + registry.remove_edges_where(lambda _, source, target: transition_id in (source, target)) + + +def _remember_trajectory(registry: Registry, call: KGBase, meta: MetadataGetter) -> None: + meta(call)["trajectory"] = { + "initial": registry.edge_target(hasInitialState, call.id), + "final": registry.edge_target(hasFinalState, call.id), + "transition": registry.edge_target(correspondsToTransition, call.id), + } + + +def restore_trajectories(registry: Registry, meta: MetadataGetter) -> None: + # Shared heuristics merge equal-content States. Recreate each call's canonical + # boundaries first so a transient partial graph cannot determine their final ids. + for cls in _CALLS: + for call in registry.all_of(cls): + trajectory = meta(call).get("trajectory") + if not trajectory: + continue + initial = registry.get(State, registry.edge_target(hasInitialState, call.id) or "") + final = registry.get(State, registry.edge_target(hasFinalState, call.id) or "") + if initial is None: + continue + _clear_boundaries(registry, call.id) + initial = registry.add(initial.model_copy(update={"id": trajectory["initial"]})) + registry.add_edge(hasInitialState(source_id=call.id, target_id=initial.id)) + registry.add_edge(hasState(source_id=call.id, target_id=initial.id)) + if final is not None and trajectory["final"] and trajectory["transition"]: + final = registry.add(final.model_copy(update={"id": trajectory["final"]})) + registry.add_edge(hasFinalState(source_id=call.id, target_id=final.id)) + registry.add_edge(hasState(source_id=call.id, target_id=final.id)) + transition = registry.add( + Transition( + id=trajectory["transition"], + sessionId=call.sessionId, + ) + ) + for edge, source, target in ( + (inputTo, initial.id, transition.id), + (leadsTo, transition.id, final.id), + (correspondsToTransition, call.id, transition.id), + (representsExecution, transition.id, call.id), + ): + registry.add_edge(edge(source_id=source, target_id=target)) + + +def prune_trajectories(registry: Registry, meta: MetadataGetter) -> None: + # A replaced event trajectory must not leave disconnected States or Transitions behind. + transitions = {target for edge, _, target in registry.edges if edge is correspondsToTransition} + for node in registry.all_of(Transition): + if node.id not in transitions: + registry.remove_node(Transition, node.id) + registry.remove_edges_where(lambda _, source, target: node.id in (source, target)) + referenced = {i for _, source, target in registry.edges for i in (source, target)} + for cls in (State, Processing): + for node in registry.all_of(cls): + pending = isinstance(node, State) and ( + meta(node).get("receiverName") or meta(node).get("ownerSpanId") + ) + if ( + node.id not in referenced + and pending + and registry.get(Session, node.sessionId) is not None + ): + registry.add_edge(hasState(source_id=node.sessionId, target_id=node.id)) + if node.id not in referenced and not pending: + registry.remove_node(cls, node.id) + + +_CALLS = (AgentCall, LLMCall, ToolCall) diff --git a/backend/norm/tests/test_normalizer.py b/backend/norm/tests/test_normalizer.py index e029053..47037d0 100644 --- a/backend/norm/tests/test_normalizer.py +++ b/backend/norm/tests/test_normalizer.py @@ -97,8 +97,13 @@ def test_stream_normalizer_converges_to_batch_result() -> None: graph.apply(StreamNormalizer(graph).process(span)) batch_nodes, batch_edges = normalize(spans) - assert {i for _, i in graph.nodes} == {n["id"] for n in batch_nodes} - assert set(graph.edges) == {(e["edge_type"], e["from_id"], e["to_id"]) for e in batch_edges} + assert graph.nodes == {(n["node_type"], n["id"]): n for n in batch_nodes} + assert { + key: {field: value for field, value in edge.items() if field not in ("from_type", "to_type")} + for key, edge in graph.edges.items() + } == { + (edge["edge_type"], edge["from_id"], edge["to_id"]): edge for edge in batch_edges + } def test_stream_normalizer_ignores_already_processed_span() -> None: @@ -160,7 +165,7 @@ def test_topology_session_started_creates_session_and_mas() -> None: assert session["startTime"] == 1792075846.93875 assert session["spanId"] == "211fdd62f03f330c" assert sorted(e["edge_type"] for e in delta.edges) == [ - "executesSession", "hasInitialState", "hasState" + "executes", "executesSession", "hasInitialState", "hasState" ] assert not any(e["edge_type"] == "hasFinalState" for e in delta.edges) @@ -205,7 +210,10 @@ def test_topology_node_events_build_agent_call_states_and_transition() -> None: graph.apply(normalizer.process(event, kind="event")) types = sorted(t for t, _ in graph.nodes) - assert types == ["Agent", "AgentCall", "MAS", "MASCall", "Session", "State", "State", "Transition"] + assert types == [ + "Agent", "AgentCall", "MAS", "MASCall", "Session", "State", "State", + "Transition", "Transition", "Transition", + ] contents = sorted(n["content"] for (t, _), n in graph.nodes.items() if t == "State") assert contents == ["bye", "hello"] edge_types = {e["edge_type"] for e in graph.edges.values()} @@ -255,7 +263,8 @@ def test_topology_session_completed_creates_missing_nodes_and_final_state() -> N assert {"executesSession", "hasInitialState", "hasFinalState", "inputTo", "leadsTo"} <= edge_types assert normalizer.process(completed, kind="event") == StreamDelta() - # started afterwards adds nothing; started-then-completed ends the session later + # Late starts correct timing without replacing the completion trajectory. + graph.apply(normalizer.process(started, kind="event")) assert normalizer.process(started, kind="event") == StreamDelta() graph2 = InMemoryGraph() n2 = StreamNormalizer(graph2) @@ -263,6 +272,8 @@ def test_topology_session_completed_creates_missing_nodes_and_final_state() -> N graph2.apply(n2.process(completed, kind="event")) session = graph2.nodes[("Session", "213dad2f")] assert session["endTime"] > session["startTime"] and session["duration"] > 0 + assert graph.nodes[("Session", "213dad2f")]["startTime"] == session["startTime"] + assert graph.nodes[("Session", "213dad2f")]["duration"] == session["duration"] assert sorted(t for t, _ in graph2.nodes) == types @@ -465,13 +476,14 @@ def test_a2a_tool_started_is_a_handoff_not_a_tool() -> None: graph.apply(normalizer.process(started, kind="event")) # no Tool/ToolCall; the target is an unattached Agent named tool.name - assert list(graph.nodes) == [("Agent", "handoff_agent")] - assert not graph.edges + assert sorted(t for t, _ in graph.nodes) == ["Agent", "MAS", "Session"] + assert [i for t, i in graph.nodes if t == "Agent"] == ["handoff_agent"] + assert not any(e["edge_type"] == "executesAgent" for e in graph.edges.values()) assert normalizer.process(started, kind="event") == StreamDelta() # an Agent of that name already there is used, not duplicated graph.apply(normalizer.process(_tool_event("tool.completed", **a2a, **{"tool.output": "out"}), kind="event")) - assert list(graph.nodes) == [("Agent", "handoff_agent")] + assert [i for t, i in graph.nodes if t == "Agent"] == ["handoff_agent"] # the target's own events then use that node (keyed on the name) and connect it node = _node_event("topology.node.started", **{"agent.name": "handoff_agent", "agent.input": "q"}) @@ -555,6 +567,8 @@ def test_a2a_agent_card_creates_unattached_agent_then_events_connect_it() -> Non assert _edges(late) == edges_before # a card without a description leaves the existing one alone nameless = json.dumps({"name": "schedule_agent"}) + late.apply(late_normalizer.process(_card_event(nameless), kind="event")) + assert late.nodes[("Agent", span)]["description"] == "Updated." assert late_normalizer.process(_card_event(nameless), kind="event") == StreamDelta() for bad in ("not json", "[]", "{}"): @@ -708,16 +722,17 @@ def test_a2a_message_received_keeps_the_message_on_the_recipients_call() -> None normalizer = StreamNormalizer(graph) received = _a2a_message_event("a2a.message.received", "2026-10-15T14:50:47.020000+00:00") - # the recipient's call is not there yet: the same nodes as sent, but no State + # Retain the pending message without inventing a recipient Agent or AgentCall. graph.apply(normalizer.process(received, kind="event")) - assert "State" not in {t for t, _ in graph.nodes} - assert "Schedule Agent" in {i for t, i in graph.nodes if t == "Agent"} + assert sorted(t for t, _ in graph.nodes) == ["MAS", "Session", "State"] + pending = next(n for (t, _), n in graph.nodes.items() if t == "State") + assert pending["rtNormalization"]["receiverName"] == "Schedule Agent" # once the recipient's own node.started has created its call, the message is kept on it node = _node_event("topology.node.started", **{"agent.name": "Schedule Agent", "agent.input": "q"}) node["SpanId"] = "4163c385d3710aac" graph.apply(normalizer.process(node, kind="event")) - graph.apply(normalizer.process(received, kind="event")) + assert normalizer.process(received, kind="event") == StreamDelta() receiver_call = next( n["id"] for (t, _), n in graph.nodes.items() if t == "AgentCall" and n["spanId"] == "4163c385d3710aac" ) @@ -1002,7 +1017,8 @@ def test_edge_updated_bridges_the_handoff_between_two_agent_calls() -> None: second["LogAttributes"]["event.time"] = "2026-10-15T14:50:48.900000+00:00" for event in (first, first_done, second): graph.apply(normalizer.process(event, kind="event")) - assert "ProcessingCall" not in {t for t, _ in graph.nodes} + # Timestamp fallback can already bridge siblings; explicit hints preserve provenance. + assert "ProcessingCall" in {t for t, _ in graph.nodes} # the source is found by its span id, the target by its name only handoff = _edge_updated_event("aaaa", "worker") @@ -1010,9 +1026,19 @@ def test_edge_updated_bridges_the_handoff_between_two_agent_calls() -> None: assert "ProcessingCall" in {t for t, _ in graph.nodes} assert "Processing" in {t for t, _ in graph.nodes} assert any(e["edge_type"] == "hasProcessingCall" for e in graph.edges.values()) + target_call = next( + n for (t, _), n in graph.nodes.items() if t == "AgentCall" and n["spanId"] == "bbbb" + ) + assert target_call["handoffSourceSpanIds"] == ["aaaa"] assert normalizer.process(handoff, kind="event") == StreamDelta() # a target that is no agent with a call (an MCP server) matches nothing - assert normalizer.process(_edge_updated_event("aaaa", "debug"), kind="event") == StreamDelta() - # nor does an unknown source span - assert normalizer.process(_edge_updated_event("nope", "worker"), kind="event") == StreamDelta() + graph.apply(normalizer.process(_edge_updated_event("aaaa", "debug"), kind="event")) + assert ("Agent", "debug") not in graph.nodes + # Unknown sources are retained for late arrival, not turned into executions. + graph.apply(normalizer.process(_edge_updated_event("nope", "worker"), kind="event")) + hints = graph.nodes[("Session", "213dad2f")]["rtNormalization"]["handoffs"] + assert {"source": "nope", "target": "worker"} in hints + assert not any( + n["spanId"] == "nope" for (t, _), n in graph.nodes.items() if t == "AgentCall" + ) diff --git a/backend/norm/tests/test_pipeline_functional.py b/backend/norm/tests/test_pipeline_functional.py index 0906b97..20c92f0 100644 --- a/backend/norm/tests/test_pipeline_functional.py +++ b/backend/norm/tests/test_pipeline_functional.py @@ -11,9 +11,8 @@ ``opentelemetry-instrumentation-langchain``, which observe-sdk activates by default whenever langchain/langgraph is installed -- OTel GenAI-semconv names like ``execute_task ``/``invoke_agent LangGraph`` that don't match any -dispatched span-name suffix and are silently ignored by design: the current -architecture only ever builds nodes from a span whose name it recognizes, it -never tries to account for every row in the trace. +dispatched span-name suffix. They do not become ontology executions, but their +compact ancestry can still be used to connect recognized child calls to agents. """ from __future__ import annotations diff --git a/backend/norm/tests/test_reconciliation.py b/backend/norm/tests/test_reconciliation.py new file mode 100644 index 0000000..79a2183 --- /dev/null +++ b/backend/norm/tests/test_reconciliation.py @@ -0,0 +1,1060 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +import itertools +import json +import random + +import pytest + +from norm import InMemoryGraph, StreamDelta, StreamNormalizer, normalize + + +def process_record(graph, record, kind): + delta = StreamNormalizer(graph).process(record, kind) + graph.apply(delta) + return delta + + +def span(name, span_id, **attrs): + return { + "SpanName": name, + "SpanId": span_id, + "Timestamp": "2026-11-10 00:00:00.000000000", + "Duration": 2_000_000_000, + "SpanAttributes": {"application_id": "app", "session.id": "app_s1", **attrs}, + } + + +def event(name, span_id="", time="2026-11-10T00:00:01+00:00", service="app", **attrs): + return { + "ServiceName": service, + "SpanId": span_id, + "LogAttributes": { + "event.name": name, + "event.time": time, + "application_id": service, + "session.id": "app_s1", + **attrs, + }, + } + + +def replay(records): + graph = InMemoryGraph() + for record, kind in records: + process_record(graph, record, kind) + return graph + + +def nodes(graph, kind): + return [n for (t, _), n in graph.nodes.items() if t == kind] + + +def stored_copy(graph): + restored = InMemoryGraph() + restored.nodes = {key: json.loads(json.dumps(value)) for key, value in graph.nodes.items()} + restored.edges = {key: json.loads(json.dumps(value)) for key, value in graph.edges.items()} + restored.span_contexts = dict(graph.span_contexts) + return restored + + +@pytest.mark.parametrize("explicit_owner", [False, True]) +def test_mcp_transport_ancestry_backfills_tool_ownership_in_every_order(explicit_owner): + agent = span("planner.agent", "agent", agent_id="planner") + client = span( + "MCP send tools/call search", + "client", + **({"ioa_observe.agent.span_id": "agent"} if explicit_owner else {}), + ) + client["ParentSpanId"] = "framework" if explicit_owner else "agent" + server = span("tools/call search", "server") + server["SpanAttributes"] = {} + server["ParentSpanId"] = "client" + tool = span( + "search.tool", + "tool", + **{"application_id": "tool-service", "ioa_observe.entity.name": "search"}, + ) + tool["ParentSpanId"] = "server" + records = [agent, client, server, tool] + for record in records: + record["TraceId"] = "trace" + expected_nodes, expected_edges = normalize(records) + expected_nodes = {(n["node_type"], n["id"]): n for n in expected_nodes} + expected_edges = {(e["edge_type"], e["from_id"], e["to_id"]): e for e in expected_edges} + for order in itertools.permutations(records): + graph = InMemoryGraph() + for record in order: + process_record(graph, record, "span") + graph = stored_copy(graph) + call = nodes(graph, "ToolCall")[0] + owner = nodes(graph, "AgentCall")[0] + assert ("hasToolCall", owner["id"], call["id"]) in graph.edges + assert ("usesTool", "planner", "search") in graph.edges + assert [mas["id"] for mas in nodes(graph, "MAS")] == ["app"] + assert graph.nodes == expected_nodes + assert { + key: {k: v for k, v in edge.items() if k not in ("from_type", "to_type")} + for key, edge in graph.edges.items() + } == expected_edges + for record in records: + assert process_record(graph, record, "span") == StreamDelta() + + +def test_transport_ancestry_does_not_cross_trace_or_session(): + agent = span("planner.agent", "agent", agent_id="planner") + agent["TraceId"] = "trace" + client = span("tools/call.mcp", "client", **{"ioa_observe.agent.span_id": "agent"}) + client["TraceId"] = "other-trace" + tool = span("search.tool", "tool", **{"ioa_observe.entity.name": "search"}) + tool.update(TraceId="trace", ParentSpanId="client") + graph = replay((record, "span") for record in (agent, client, tool)) + assert not any(edge == "hasToolCall" for edge, _, _ in graph.edges) + client["TraceId"] = "trace" + client["SpanAttributes"]["session.id"] = "app_other" + process_record(graph, client, "span") + assert not any(edge == "hasToolCall" for edge, _, _ in graph.edges) + client["SpanAttributes"]["session.id"] = "app_s1" + process_record(graph, client, "span") + owner = nodes(graph, "AgentCall")[0] + call = nodes(graph, "ToolCall")[0] + assert ("hasToolCall", owner["id"], call["id"]) in graph.edges + + +def test_transport_ancestry_cycle_remains_unowned(): + first = span("tools/call.mcp", "one") + first["ParentSpanId"] = "two" + second = span("tools/call search", "two") + second["ParentSpanId"] = "one" + tool = span("search.tool", "tool", **{"ioa_observe.entity.name": "search"}) + tool["ParentSpanId"] = "one" + graph = replay((record, "span") for record in (tool, first, second)) + assert len(nodes(graph, "ToolCall")) == 1 + assert not any(edge == "hasToolCall" for edge, _, _ in graph.edges) + + +def test_sessionless_transport_is_private_context_not_an_execution(): + record = { + "TraceId": "trace", + "SpanId": "server", + "ParentSpanId": "client", + "SpanName": "tools/call search", + "ServiceName": "tool-service", + "SpanAttributes": {}, + } + graph = InMemoryGraph() + delta = process_record(graph, record, "span") + assert not delta.nodes and not delta.edges + assert not graph.nodes and not graph.edges + assert len(delta.span_contexts) == 1 + graph = stored_copy(graph) + assert graph.load_span_contexts("trace")[0].session_id == "" + assert process_record(graph, record, "span") == StreamDelta() + + +@pytest.mark.parametrize("include_transport_span", [False, True]) +@pytest.mark.parametrize("include_message_input", [False, True]) +def test_mcp_received_event_supplies_tool_parent_ownership_in_every_order( + include_transport_span, include_message_input +): + agent = span("planner.agent", "agent", agent_id="planner") + tool = span("search.tool", "tool", **{"ioa_observe.entity.name": "search"}) + tool["ParentSpanId"] = "server" + received = event( + "mcp.message.received", + "server", + **{ + "source.agent": "planner", + "source.agent.span.id": "agent", + "source.agent.trace.id": "trace", + "operation.name": "tools/call", + "message.id": "request", + "message.input": '{"name": "search", "arguments": {}}', + }, + ) + if not include_message_input: + received["LogAttributes"].pop("message.input") + records = [(agent, "span"), (tool, "span"), (received, "event")] + if include_transport_span: + server = span("tools/call search", "server") + server["SpanAttributes"] = {} + server["ParentSpanId"] = "client" + records.append((server, "span")) + for record, _ in records: + record["TraceId"] = "trace" + for order in itertools.permutations(records): + graph = InMemoryGraph() + for record, kind in order: + process_record(graph, record, kind) + graph = stored_copy(graph) + owner = nodes(graph, "AgentCall")[0] + assert len(nodes(graph, "ToolCall")) == (2 if include_message_input else 1) + for call in nodes(graph, "ToolCall"): + assert ("hasToolCall", owner["id"], call["id"]) in graph.edges + context = graph.span_contexts[("trace", "server")] + assert context.owner_span_id == "agent" + assert context.parent_span_id == ("client" if include_transport_span else "") + for record, kind in records: + assert process_record(graph, record, kind) == StreamDelta() + graph.span_contexts.clear() + for key in list(graph.edges): + if key[0] == "hasToolCall": + del graph.edges[key] + process_record(graph, received, "event") + for call in nodes(graph, "ToolCall"): + assert ("hasToolCall", owner["id"], call["id"]) in graph.edges + + +def test_mcp_received_event_does_not_infer_an_owner_in_another_trace(): + agent = span("planner.agent", "agent", agent_id="planner") + tool = span("search.tool", "tool", **{"ioa_observe.entity.name": "search"}) + tool["ParentSpanId"] = "server" + received = event( + "mcp.message.received", + "server", + **{ + "source.agent": "planner", + "source.agent.span.id": "agent", + "source.agent.trace.id": "other-trace", + "message.id": "request", + "message.input": '{"name": "search", "arguments": {}}', + }, + ) + for record in (agent, tool, received): + record["TraceId"] = "trace" + graph = replay([(agent, "span"), (received, "event"), (tool, "span")]) + call = next(n for n in nodes(graph, "ToolCall") if n["spanId"] == "tool") + assert not any( + edge == "hasToolCall" and target == call["id"] for edge, _, target in graph.edges + ) + + +def test_explicit_transport_owner_takes_precedence_over_outer_agent_parent(): + inner = span("inner.agent", "inner", agent_id="inner") + outer = span("outer.agent", "outer", agent_id="outer") + client = span("tools/call.mcp", "client", **{"ioa_observe.agent.span_id": "inner"}) + client["ParentSpanId"] = "outer" + tool = span("search.tool", "tool", **{"ioa_observe.entity.name": "search"}) + tool["ParentSpanId"] = "client" + for order in ((tool, client, outer, inner), (outer, inner, client, tool)): + graph = replay((record, "span") for record in order) + call = nodes(graph, "ToolCall")[0] + parents = [ + source + for edge, source, target in graph.edges + if edge == "hasToolCall" and target == call["id"] + ] + assert len(parents) == 1 + assert graph.nodes[("AgentCall", parents[0])]["spanId"] == "inner" + + +def test_replaying_legacy_completed_spans_upgrades_missing_ancestry(): + agent = span("planner.agent", "agent", agent_id="planner") + tool = span("search.tool", "tool", **{"ioa_observe.entity.name": "search"}) + tool["ParentSpanId"] = "server" + server = span( + "MCP server tools/call search", "server", **{"ioa_observe.agent.span_id": "agent"} + ) + for record in (agent, tool, server): + record["TraceId"] = "trace" + graph = replay((record, "span") for record in (tool, agent)) + graph.span_contexts.clear() + for node in graph.nodes.values(): + node.get("rtNormalization", {}).pop("traceId", None) + graph = stored_copy(graph) + for record in (tool, agent, server): + process_record(graph, record, "span") + call = nodes(graph, "ToolCall")[0] + owner = nodes(graph, "AgentCall")[0] + assert call["rtNormalization"]["traceId"] == "trace" + assert ("hasToolCall", owner["id"], call["id"]) in graph.edges + assert ("usesTool", "planner", "search") in graph.edges + + +def test_session_completion_spans_keep_latest_end_and_redeliver_without_changes(): + start = span("session.start", "start", ioa_start_time=1000) + latest = span("session.end", "latest", **{"session.ended_at": 1010}) + early = span("session.end", "early", **{"session.ended_at": 1005}) + for endings in ((latest, early), (early, latest)): + graph = replay((record, "span") for record in (start, *endings)) + assert nodes(graph, "Session")[0]["endTime"] == 1010 + for record in (latest, early): + assert process_record(graph, record, "span") == StreamDelta() + + +def test_agent_and_tool_with_same_id_keep_distinct_capability_relationships(): + agent = span("schedule.agent", "agent", agent_id="schedule") + tool = span( + "schedule.tool", + "tool", + **{"ioa_observe.entity.name": "schedule", "ioa_observe.agent.span_id": "agent"}, + ) + graph = replay((record, "span") for record in (tool, agent)) + process_record(graph, span("execute_tool schedule", "transport"), "span") + assert ("Agent", "schedule") in graph.nodes + assert ("Tool", "schedule") in graph.nodes + assert ("usesTool", "schedule", "schedule") in graph.edges + edge = graph.edges[("usesTool", "schedule", "schedule")] + assert (edge["from_type"], edge["to_type"]) == ("Agent", "Tool") + owner = nodes(graph, "AgentCall")[0] + call = nodes(graph, "ToolCall")[0] + assert graph.edges[("executes", owner["id"], "schedule")]["to_type"] == "Agent" + assert graph.edges[("executes", call["id"], "schedule")]["to_type"] == "Tool" + + +@pytest.mark.parametrize("order", [(0, 1, 2, 3), (3, 2, 1, 0), (2, 0, 3, 1)]) +def test_batch_and_stream_share_full_graph_payloads(order): + records = [ + span("planner.agent", "agent", agent_id="planner"), + span( + "model.chat", + "chat", + **{ + "ioa_observe.agent.span_id": "agent", + "gen_ai.provider.name": "test", + "gen_ai.request.model": "model", + }, + ), + span( + "search.tool", + "tool", + **{ + "ioa_observe.agent.span_id": "agent", + "ioa_observe.entity.name": "search", + }, + ), + span( + "workflow.graph", + "graph", + **{ + "gen_ai.ioa.graph": json.dumps( + {"nodes": {"planner": {}, "router": {}, "search": {}}} + ) + }, + ), + ] + ordered = [records[index] for index in order] + batch_nodes, batch_edges = normalize(ordered) + graph = replay((record, "span") for record in ordered) + assert graph.nodes == {(node["node_type"], node["id"]): node for node in batch_nodes} + assert { + key: {k: v for k, v in edge.items() if k not in ("from_type", "to_type")} + for key, edge in graph.edges.items() + } == {(edge["edge_type"], edge["from_id"], edge["to_id"]): edge for edge in batch_edges} + assert [agent["id"] for agent in nodes(graph, "Agent")] == ["planner"] + + +def test_legacy_worker_metadata_survives_reload_and_completed_span_enrichment(): + started = event( + "tool.started", + "tool", + **{ + "tool.name": "search", + "tool.call.id": "natural-id", + "ioa_observe.agent.span_id": "agent", + "tool.input": "provisional", + }, + ) + graph = stored_copy(replay([(started, "event")])) + completed = span( + "search.tool", + "tool", + **{ + "gen_ai.tool.call.id": "natural-id", + "ioa_observe.entity.name": "search", + "ioa_observe.entity.input": "question", + "ioa_observe.entity.output": "answer", + "ioa_observe.agent.span_id": "agent", + }, + ) + process_record(graph, completed, "span") + graph = stored_copy(graph) + process_record(graph, span("planner.agent", "agent", agent_id="planner"), "span") + call = nodes(graph, "ToolCall")[0] + assert call["id"] == "natural-id" + assert call["rtNormalization"]["spanComplete"] is True + assert "inputPriority" not in call["rtNormalization"] + assert len(nodes(graph, "ToolCall")) == 1 + agent = nodes(graph, "AgentCall")[0] + assert ("hasToolCall", agent["id"], call["id"]) in graph.edges + assert process_record(graph, completed, "span") == StreamDelta() + + +def test_legacy_handoff_property_is_preserved_without_new_metadata(): + target = span( + "target.agent", + "target-span", + **{ + "agent_id": "target", + "ioa_start_time": 1001, + "ioa_observe.entity.input": "request", + "ioa_observe.entity.output": "reply", + "ioa_observe.handoff.source.span_ids": json.dumps(["source-span"]), + }, + ) + graph = stored_copy(replay([(target, "span")])) + for node in graph.nodes.values(): + node.pop("rtNormalization", None) + process_record( + graph, + span( + "source.agent", + "source-span", + **{ + "agent_id": "source", + "ioa_start_time": 1000, + "ioa_observe.entity.output": "handoff", + }, + ), + "span", + ) + call = next(n for n in nodes(graph, "AgentCall") if n["spanId"] == "target-span") + assert call["handoffSourceSpanIds"] == ["source-span"] + assert len(nodes(graph, "ProcessingCall")) == 1 + assert len(nodes(graph, "AgentCall")) == 2 + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_cards_and_discovered_tools_preserve_application_names_with_underscores(reverse): + records = [ + event( + "a2a.agent_card", + "card", + service="tool_service", + **{ + "session.id": "origin_mas_s1", + "capability.content": json.dumps({"name": "Planner"}), + }, + ), + event( + "mcp.tools.discovered", + "discovery", + service="tool_service", + **{ + "session.id": "origin_mas_s1", + "ioa_observe.agent.span_id": "agent", + "capability.content": json.dumps({"tools": [{"name": "search"}]}), + }, + ), + event( + "topology.node.started", + "agent", + service="origin_mas", + **{"session.id": "origin_mas_s1", "agent.name": "Planner"}, + ), + ] + graph = replay((record, "event") for record in (reversed(records) if reverse else records)) + assert [mas["id"] for mas in nodes(graph, "MAS")] == ["origin_mas"] + assert [session["id"] for session in nodes(graph, "Session")] == ["s1"] + assert ("usesTool", "Planner", "search") in graph.edges + assert ("belongsToMAS", "Planner", "origin_mas") in graph.edges + + +def test_container_final_state_follows_last_ending_child(): + long = span( + "long.agent", + "long", + **{"agent_id": "long", "ioa_start_time": 1000, "ioa_observe.entity.output": "last"}, + ) + long["Duration"] = 10_000_000_000 + short = span( + "short.agent", + "short", + **{"agent_id": "short", "ioa_start_time": 1001, "ioa_observe.entity.output": "first"}, + ) + for records in ((long, short), (short, long)): + graph = replay((record, "span") for record in records) + for cls in ("Session", "MASCall"): + container = nodes(graph, cls)[0] + assert container["startTime"] == 1000 + assert container["endTime"] == 1010 + final = next( + edge["to_id"] + for edge in graph.edges.values() + if edge["edge_type"] == "hasFinalState" and edge["from_id"] == container["id"] + ) + assert graph.nodes[("State", final)]["content"] == "last" + + +def test_invalid_graph_node_shape_fails_in_batch_and_stream(): + record = span("workflow.graph", "graph", **{"gen_ai.ioa.graph": '{"nodes":[]}'}) + with pytest.raises(ValueError, match="nodes must be an object"): + normalize([record]) + with pytest.raises(ValueError, match="nodes must be an object"): + StreamNormalizer(InMemoryGraph()).process(record) + + +def test_native_session_states_survive_until_child_boundaries_are_usable(): + graph = replay([(event("topology.session.started"), "event")]) + initial = next( + edge["to_id"] + for edge in graph.edges.values() + if edge["edge_type"] == "hasInitialState" and edge["from_id"] == "s1" + ) + process_record( + graph, event("topology.node.started", "agent", **{"agent.name": "planner"}), "event" + ) + assert ("hasInitialState", "s1", initial) in graph.edges + process_record( + graph, + event("topology.session.completed", time="2026-11-10T00:00:04+00:00"), + "event", + ) + assert any( + edge["edge_type"] == "hasFinalState" and edge["from_id"] == "s1" + for edge in graph.edges.values() + ) + process_record( + graph, + event( + "topology.node.completed", + "agent", + time="2026-11-10T00:00:03+00:00", + **{"agent.name": "planner", "agent.output": "done"}, + ), + "event", + ) + final = next( + edge["to_id"] + for edge in graph.edges.values() + if edge["edge_type"] == "hasFinalState" and edge["from_id"] == "s1" + ) + assert graph.nodes[("State", final)]["content"] == "done" + + +def test_pending_handoff_metadata_is_independent_of_arrival_order(): + records = [ + event( + "topology.edge.updated", + "edge", + **{"source.agent.span.id": source, "target.agent": target}, + ) + for source, target in (("one", "two"), ("two", "three")) + ] + first = replay((record, "event") for record in records) + second = replay((record, "event") for record in reversed(records)) + assert first.nodes == second.nodes + assert first.edges == second.edges + assert not nodes(first, "Agent") + + +def test_capability_ownership_is_independent_of_span_arrival_order(): + records = [ + span("planner.agent", "agent", agent_id="planner"), + span( + "model.chat", + "chat", + **{ + "agent_id": "planner", + "ioa_observe.agent.span_id": "agent", + "gen_ai.provider.name": "test", + "gen_ai.request.model": "model", + }, + ), + span( + "search.tool", + "tool", + **{ + "agent_id": "planner", + "ioa_observe.agent.span_id": "agent", + "ioa_observe.entity.name": "search", + }, + ), + ] + for order in itertools.permutations(records): + graph = replay((r, "span") for r in order) + agent = nodes(graph, "AgentCall")[0] + for node_type, edge_type in ( + ("LLMCall", "hasLLMCall"), + ("ToolCall", "hasToolCall"), + ): + assert ( + edge_type, + agent["id"], + nodes(graph, node_type)[0]["id"], + ) in graph.edges + assert process_record(graph, order[-1], "span") == StreamDelta() + + +@pytest.mark.parametrize("event_first", [False, True]) +def test_agent_events_and_spans_share_one_complete_call(event_first): + started = event("topology.node.started", "agent", **{"agent.name": "Planner"}) + completed = span( + "planner.agent", + "agent", + **{ + "agent_id": "planner-id", + "ioa_observe.entity.name": "Planner", + "ioa_observe.entity.input": "question", + "ioa_observe.entity.output": "answer", + }, + ) + records = [(started, "event"), (completed, "span")] + graph = replay(records if event_first else reversed(records)) + assert len(nodes(graph, "Agent")) == len(nodes(graph, "AgentCall")) == 1 + call = nodes(graph, "AgentCall")[0] + assert call["duration"] == 2000 + final = graph.edges[ + ( + "hasFinalState", + call["id"], + next( + e["to_id"] + for e in graph.edges.values() + if e["edge_type"] == "hasFinalState" and e["from_id"] == call["id"] + ), + ) + ] + assert graph.nodes[("State", final["to_id"])]["content"] == "answer" + assert process_record(graph, completed, "span") == StreamDelta() + + +@pytest.mark.parametrize("server_first", [False, True]) +def test_mcp_participant_does_not_create_another_mas(server_first): + started = event("topology.node.started", "agent", **{"agent.name": "Planner"}) + message = event( + "mcp.message.received", + "server", + service="tool-server", + **{ + "source.agent": "Planner", + "source.agent.span.id": "agent", + "target.agent": "debug", + "message.id": "request-1", + "message.input": '{"name":"search","arguments":{}}', + }, + ) + records = [(started, "event"), (message, "event")] + graph = replay(records if not server_first else reversed(records)) + assert [n["id"] for n in nodes(graph, "MAS")] == ["app"] + assert len(nodes(graph, "Session")) == len(nodes(graph, "Agent")) == 1 + assert {e["to_id"] for e in graph.edges.values() if e["edge_type"] == "executesSession"} == { + "app" + } + + +@pytest.mark.parametrize("graph_first", [False, True]) +def test_graph_metadata_declares_only_observed_agents(graph_first): + graph_span = span( + "workflow.graph", + "graph", + **{ + "gen_ai.ioa.graph": json.dumps( + { + "nodes": {n: {"id": n, "name": n} for n in ("planner", "router", "tools")}, + } + ) + }, + ) + agent_span = span("planner.agent", "agent", agent_id="planner") + records = [(graph_span, "span"), (agent_span, "span")] + graph = replay(records if graph_first else reversed(records)) + assert [n["name"] for n in nodes(graph, "Agent")] == ["planner"] + assert nodes(graph, "Agent")[0]["declared"] is True + assert process_record(graph, graph_span, "span") == StreamDelta() + + +@pytest.mark.parametrize("edge_first", [False, True]) +def test_explicit_handoff_is_backfilled_without_assigning_a_second_agent(edge_first): + handoff = event( + "topology.edge.updated", + "transport", + **{ + "source.agent": "Planner", + "source.agent.span.id": "source", + "target.agent": "Worker", + "topology.edge.kind": "a2a_message", + "network.protocol.name": "a2a", + }, + ) + records = [ + ( + event("topology.node.started", "source", **{"agent.name": "Planner"}), + "event", + ), + ( + event( + "topology.node.completed", + "source", + **{ + "agent.name": "Planner", + "agent.output": "plan", + }, + ), + "event", + ), + ( + event( + "topology.node.started", + "target", + **{ + "agent.name": "Worker", + "agent.input": "execute", + }, + ), + "event", + ), + ] + graph = replay( + ([(handoff, "event")] + records) if edge_first else (records + [(handoff, "event")]) + ) + assert nodes(graph, "ProcessingCall") + for call in nodes(graph, "AgentCall"): + assert ( + sum( + e["edge_type"] == "executesAgent" and e["from_id"] == call["id"] + for e in graph.edges.values() + ) + == 1 + ) + assert process_record(graph, handoff, "event") == StreamDelta() + + +def test_presented_tools_and_a2a_completion_are_retained(): + graph = replay( + [ + ( + event("topology.node.started", "agent", **{"agent.name": "Planner"}), + "event", + ), + ( + event( + "llm.tools.presented", + "agent", + **{ + "capability.content": json.dumps( + {"tools": [{"type": "function", "function": {"name": "search"}}]} + ), + }, + ), + "event", + ), + ( + event( + "a2a.message.completed", + "transport", + **{ + "source.agent": "Planner", + "source.agent.span.id": "agent", + "target.agent": "Worker", + "message.id": "m1", + "message.output": "reply", + }, + ), + "event", + ), + ] + ) + assert any(n["name"] == "search" for n in nodes(graph, "Tool")) + assert any(n["content"] == "reply" for n in nodes(graph, "State")) + + +@pytest.mark.parametrize("message_kind", ["received", "completed"]) +def test_unresolved_a2a_messages_stay_connected_until_their_owner_arrives(message_kind): + message = event( + f"a2a.message.{message_kind}", + "transport", + **{ + "source.agent": "Sender", + "source.agent.span.id": "sender", + "target.agent": "Worker Display Name", + "message.id": "message", + "message.input": "request", + "message.output": "response", + }, + ) + unrelated = event( + "topology.node.completed", + "other", + **{"agent.name": "worker_runtime", "agent.input": "input", "agent.output": "output"}, + ) + owner = event( + "topology.node.started", + "receiver" if message_kind == "received" else "sender", + **{"agent.name": "Worker Display Name" if message_kind == "received" else "Sender"}, + ) + for order in ((message,), *itertools.permutations((message, unrelated))): + graph = replay((record, "event") for record in order) + state = next( + n + for n in nodes(graph, "State") + if n["content"] == ("request" if message_kind == "received" else "response") + ) + assert ("hasState", "s1", state["id"]) in graph.edges + assert not any( + edge == "hasState" and target == state["id"] and source != "s1" + for edge, source, target in graph.edges + ) + assert process_record(graph, message, "event") == StreamDelta() + graph = stored_copy(graph) + process_record(graph, owner, "event") + call = next(n for n in nodes(graph, "AgentCall") if n["spanId"] == owner["SpanId"]) + assert ("hasState", call["id"], state["id"]) in graph.edges + assert ("hasState", "s1", state["id"]) not in graph.edges + assert process_record(graph, message, "event") == StreamDelta() + + +@pytest.mark.parametrize("call_kind", ["llm", "tool"]) +def test_completion_before_start_preserves_input_and_corrects_timing(call_kind): + attrs = { + "agent.name": "Planner", + "ioa_observe.agent.span_id": "agent", + "llm.name": "test/model", + "tool.name": "search", + } + records = [ + (event("topology.node.started", "agent", **{"agent.name": "Planner"}), "event"), + ( + event( + f"{call_kind}.completed", + "child", + time="2026-11-10T00:00:02+00:00", + **{**attrs, f"{call_kind}.output": "answer"}, + ), + "event", + ), + ( + event( + f"{call_kind}.started", + "child", + **{**attrs, f"{call_kind}.input": "question"}, + ), + "event", + ), + ] + graph = replay(records) + call = nodes(graph, "LLMCall" if call_kind == "llm" else "ToolCall")[0] + assert call["duration"] == 1000 + states = { + e["edge_type"]: graph.nodes[("State", e["to_id"])]["content"] + for e in graph.edges.values() + if e["from_id"] == call["id"] and e["edge_type"] in ("hasInitialState", "hasFinalState") + } + assert states == {"hasInitialState": "question", "hasFinalState": "answer"} + + +def test_session_envelope_is_corrected_when_parent_span_arrives_last(): + graph = replay( + [ + ( + event("topology.node.started", "agent", **{"agent.name": "Planner"}), + "event", + ), + ( + span( + "planner.agent", + "agent", + **{ + "agent_id": "planner", + "ioa_observe.entity.name": "Planner", + }, + ), + "span", + ), + ] + ) + call = nodes(graph, "AgentCall")[0] + for kind in ("MASCall", "Session"): + container = nodes(graph, kind)[0] + assert container["startTime"] <= call["startTime"] + assert container["endTime"] >= call["endTime"] + + +def test_mixed_records_converge_and_redelivery_is_a_noop(): + records = [ + (event("topology.session.started"), "event"), + (event("topology.node.started", "agent", **{"agent.name": "Planner"}), "event"), + ( + event( + "llm.started", + "chat", + **{ + "agent.name": "Planner", + "ioa_observe.agent.span_id": "agent", + "llm.name": "test/model", + "llm.input": "question", + }, + ), + "event", + ), + ( + event( + "llm.completed", + "chat", + time="2026-11-10T00:00:02+00:00", + **{ + "agent.name": "Planner", + "ioa_observe.agent.span_id": "agent", + "llm.name": "test/model", + "llm.output": "answer", + }, + ), + "event", + ), + ( + event( + "mcp.message.received", + "server", + service="tool-server", + **{ + "source.agent": "Planner", + "source.agent.span.id": "agent", + "target.agent": "debug", + "message.id": "request-1", + "message.input": '{"name":"search","arguments":{}}', + }, + ), + "event", + ), + ( + event( + "tool.started", + "tool", + **{ + "tool.name": "search", + "tool.call.id": "request-1", + "ioa_observe.agent.span_id": "agent", + "tool.input": "query", + }, + ), + "event", + ), + ( + span( + "planner.agent", + "agent", + **{ + "agent_id": "planner", + "ioa_observe.entity.name": "Planner", + "ioa_observe.entity.input": "question", + "ioa_observe.entity.output": "answer", + }, + ), + "span", + ), + ] + expected = replay(records) + for seed in range(10): + order = random.Random(seed).sample(records, len(records)) + graph = replay(order) + assert graph.nodes == expected.nodes + assert graph.edges == expected.edges + for record, kind in records: + assert process_record(graph, record, kind) == StreamDelta() + + +def test_top_level_agent_id_does_not_duplicate_the_agent_named_by_events(): + completed = span("planner.agent", "agent", **{"ioa_observe.entity.name": "Planner"}) + completed["AgentId"] = "planner-runtime-id" + graph = replay( + [ + ( + event("topology.node.started", "agent", **{"agent.name": "Planner"}), + "event", + ), + (completed, "span"), + ] + ) + assert len(nodes(graph, "Agent")) == len(nodes(graph, "AgentCall")) == 1 + + +def test_presented_tools_enrich_existing_tool_definitions(): + graph = replay( + [ + (event("tool.started", "tool", **{"tool.name": "search"}), "event"), + ( + event( + "llm.tools.presented", + "agent", + **{ + "capability.content": json.dumps( + { + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search documents", + }, + } + ] + } + ), + }, + ), + "event", + ), + ] + ) + assert nodes(graph, "Tool")[0]["description"] == "Search documents" + + +def test_presented_tool_ownership_is_backfilled_when_agent_arrives_later(): + graph = replay( + [ + ( + event( + "llm.tools.presented", + "agent", + **{ + "capability.content": '{"tools":[{"function":{"name":"search"}}]}', + }, + ), + "event", + ), + ( + event("topology.node.started", "agent", **{"agent.name": "Planner"}), + "event", + ), + ] + ) + agent = nodes(graph, "Agent")[0] + assert ("usesTool", agent["id"], "search") in graph.edges + + +@pytest.mark.parametrize("event_first", [False, True]) +@pytest.mark.parametrize("call_kind,native_id", [("llm", True), ("tool", True), ("tool", False)]) +def test_completed_capability_spans_enrich_events_without_duplicate_calls( + event_first, + call_kind, + native_id, +): + started = event( + f"{call_kind}.started", + "child", + **{ + "agent.name": "Planner", + "ioa_observe.agent.span_id": "agent", + "llm.name": "test/model", + "tool.name": "search", + "tool.call.id": "request-1", + }, + ) + completed = span( + "model.chat" if call_kind == "llm" else "search.tool", + "child", + **{ + "agent_id": "Planner", + "ioa_observe.agent.span_id": "agent", + "gen_ai.provider.name": "test", + "gen_ai.request.model": "model", + "gen_ai.usage.input_tokens": "42", + "gen_ai.tool.call.id": "request-1", + "ioa_observe.entity.name": "search", + "ioa_observe.entity.output": "answer", + "gen_ai.output.messages": "answer", + }, + ) + if not native_id: + completed["SpanAttributes"].pop("gen_ai.tool.call.id") + records = [(started, "event"), (completed, "span")] + graph = replay(records if event_first else reversed(records)) + calls = nodes(graph, "LLMCall" if call_kind == "llm" else "ToolCall") + assert len(calls) == 1 + assert calls[0]["duration"] == 2000 + if call_kind == "llm": + assert calls[0]["promptTokenCount"] == 42 + else: + assert calls[0]["id"] == "request-1" + assert process_record(graph, completed, "span") == StreamDelta() diff --git a/backend/workers/rt-norm-worker/README.md b/backend/workers/rt-norm-worker/README.md index 84d3d0f..e36164a 100644 --- a/backend/workers/rt-norm-worker/README.md +++ b/backend/workers/rt-norm-worker/README.md @@ -3,11 +3,60 @@ Standalone worker (no `worker-base`) that normalizes OTel **spans** and **logs**, one record at a time, into the Neo4j knowledge graph, and calls back the URL of every rule a log record matches. -Nothing is kept between records: each is normalized against the graph as stored. What each log -event creates is in [`backend/norm/docs/events.md`](../../norm/docs/events.md). +Each record is normalized against the stored graph and compact trace context held in process +memory. The shared handlers are described in +[`backend/norm/docs/events.md`](../../norm/docs/events.md). Identity, lifecycle and ownership +reconciliation live in shared `norm`; the worker provides ingestion, persistence and callbacks. All commands run from `backend/`. +## Reconciliation + +- Child LLM/tool calls retain their owning agent span id. Ownership and capability-use edges + are backfilled when the agent arrives, including when its span finishes after its children. + Parent chains can cross transport/framework spans, including sessionless MCP server spans. + Private in-memory trace context preserves these links across arrival orders within the + lifetime of one worker store, but does not survive restarts. +- Lifecycle events create provisional calls. Completed spans enrich those same calls with + authoritative timing, content and metrics; an event sharing the span id does not suppress + the span. Late started events also correct event-only calls' input and start time. +- A propagated `_` identifies the originating MAS. A different + participating service name (for example an MCP server) does not create another MAS. +- Graph metadata is retained on the MAS, but only graph nodes matched to observed agents are + declared as Agents. Routers and tool-serving graph nodes are not assumed to be agents. +- Topology handoff hints are retained until both agent calls exist. They connect execution + trajectories, not a second `executesAgent` target on the sender. MCP/tool edges do not + manufacture recipient Agents. +- `llm.tools.presented` retains offered tools and links them to the owning agent when known. + A2A incoming messages and completion replies are retained as message States; they do not + replace an agent's own final State. + +Compact reconciliation hints and processed event ids are persisted as `rtNormalization` +properties on the relevant graph nodes. No raw telemetry replay buffer is stored. A new +worker instance can continue reconciliation using these graph hints, but must rebuild any +required transport ancestry by replaying the relevant spans/events. Redelivered +events/completed spans are idempotent once context has been rebuilt. Both Neo4j and the +dry-run store accept earlier start-time corrections. + +These guarantees also apply to direct users of `norm.StreamNormalizer`. Batch span +normalization uses the same identity, ownership, graph-role and trajectory reconciliation. +This is not a migration or cleanup of previously misclassified graph data. Use a fresh graph +for an end-to-end comparison; the worker does not delete unrelated existing MAS/Agent nodes. +Ontology/SHACL checks are diagnostic and are not bypassed or changed by this worker. + +Private ancestry is cached in memory by trace and span id on the `Neo4jStore` instance. +The CLI and HTTP receiver reuse that instance across records/requests; a fresh store starts +with empty context, including a separate file-ingestion command. No `_NormalizationTrace` +nodes are read or written. Existing nodes created by older worker versions are left untouched; +exclude them from views until explicitly cleaned up. + +Restarting the worker loses this context, and multiple workers do not share it. Route records +for a trace to the same worker/store. To repair missing ownership after a restart, replay the +relevant spans and MCP events into that same instance. Already persisted KG relationships are +not removed merely because the cache is empty. The cache currently has no eviction or expiry, +so its memory usage grows with the number of spans processed during the store's lifetime. +No raw telemetry payloads are cached, and older duplicate MAS/Agent identities are not removed. + ## Setup Neo4j must be running. The worker reads its connection from environment variables (there is no diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py index dcde9a6..e4ec8e5 100644 --- a/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/store.py @@ -3,20 +3,17 @@ """Writes one span's KG delta straight to Neo4j, updating nodes in place.""" -import logging import json +import logging from collections.abc import Sequence from typing import Any -from norm import StreamDelta +from norm import StreamDelta, SpanContext logger = logging.getLogger(__name__) # Node types merged on `name` (unique constraint) instead of `id`; mirrors scope.client.dal. NAME_KEYED_NODE_TYPES = frozenset({"LLM"}) -# Set when a node is first created and never changed afterwards (endTime is not: it advances -# as a container, e.g. an AgentCall or Session, completes). -IMMUTABLE_PROPS = frozenset({"startTime"}) _NON_PROPS = frozenset({"node_type"}) _EDGE_NON_PROPS = frozenset({"edge_type", "from_id", "from_type", "to_id", "to_type"}) @@ -49,10 +46,11 @@ def _node(label: str, props: dict[str, Any]) -> dict[str, Any]: class Neo4jStore: - """Neo4j implementation of norm's ``GraphReader``, plus writer for its deltas.""" + """Neo4j graph reader/writer with store-local, in-memory trace context.""" def __init__(self, connector: Any): self._db = connector + self._span_contexts: dict[str, dict[str, SpanContext]] = {} def load( self, session_id: str, node_ids: Sequence[str] @@ -61,7 +59,7 @@ def load( params = {"sid": session_id, "ids": list(node_ids)} nodes: dict[tuple[str, str], dict[str, Any]] = {} for row in self._db.execute( - "MATCH (n) WHERE n.sessionId = $sid OR n.id IN $ids " + "MATCH (n) WHERE ($sid <> '' AND n.sessionId = $sid) OR n.id IN $ids " "OR (n:Agent AND n.name IN $ids) " "RETURN labels(n)[0] AS label, properties(n) AS props", params, @@ -72,16 +70,28 @@ def load( edges: list[dict[str, Any]] = [] for row in self._db.execute( "MATCH (a)-[r]->(b) " - "WHERE a.sessionId = $sid OR b.sessionId = $sid " + "WHERE ($sid <> '' AND (a.sessionId = $sid OR b.sessionId = $sid)) " "OR (a:Agent AND a.name IN $ids) OR (b:Agent AND b.name IN $ids) " "OR (a.id IN $ids AND b.id IN $ids) " - "RETURN type(r) AS edge_type, a.id AS from_id, b.id AS to_id, " + "RETURN type(r) AS edge_type, properties(r) AS props, a.id AS from_id, b.id AS to_id, " "labels(a)[0] AS from_label, properties(a) AS from_props, " "labels(b)[0] AS to_label, properties(b) AS to_props", params, ): - edges.append({"edge_type": row["edge_type"], "from_id": row["from_id"], "to_id": row["to_id"]}) - for label, props in ((row["from_label"], row["from_props"]), (row["to_label"], row["to_props"])): + edges.append( + { + **{k: _decode(v) for k, v in row.get("props", {}).items()}, + "edge_type": row["edge_type"], + "from_id": row["from_id"], + "to_id": row["to_id"], + "from_type": row["from_label"], + "to_type": row["to_label"], + } + ) + for label, props in ( + (row["from_label"], row["from_props"]), + (row["to_label"], row["to_props"]), + ): node = _node(label, props) nodes.setdefault((node["node_type"], node["id"]), node) return list(nodes.values()), edges @@ -96,16 +106,19 @@ def apply(self, delta: StreamDelta) -> None: self._upsert_node(node) for edge in delta.edges: self._upsert_edge(edge) + for context in delta.span_contexts: + self._span_contexts.setdefault(context.trace_id, {})[context.span_id] = context + + def load_span_contexts(self, trace_id: str) -> list[SpanContext]: + return list(self._span_contexts.get(trace_id, {}).values()) def _upsert_node(self, node: dict[str, Any]) -> None: node_type = _safe(str(node["node_type"])) props = _props(node, _NON_PROPS) - on_match = {k: v for k, v in props.items() if k not in IMMUTABLE_PROPS} key = "name" if node_type in NAME_KEYED_NODE_TYPES else "id" self._db.execute_command( - f"MERGE (n:{node_type} {{{key}: $key}}) " - "ON CREATE SET n += $props ON MATCH SET n += $on_match", - {"key": props[key], "props": props, "on_match": on_match}, + f"MERGE (n:{node_type} {{{key}: $key}}) SET n += $props", + {"key": props[key], "props": props}, ) def _upsert_edge(self, edge: dict[str, Any]) -> None: diff --git a/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py b/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py index c221df8..c1af343 100644 --- a/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py +++ b/backend/workers/rt-norm-worker/src/rt_norm_worker/worker.py @@ -24,7 +24,7 @@ def read_records(path: Path = DEFAULT_TRACES_PATH) -> Iterator[dict[str, Any]]: class Graph(GraphReader, Protocol): - """Where the KG lives: read what a span needs, write back what changed.""" + """Read the relevant graph and apply deltas, including corrected earlier timestamps.""" def apply(self, delta: Any) -> None: ... diff --git a/backend/workers/rt-norm-worker/tests/test_server.py b/backend/workers/rt-norm-worker/tests/test_server.py index 608e5c1..6c94265 100644 --- a/backend/workers/rt-norm-worker/tests/test_server.py +++ b/backend/workers/rt-norm-worker/tests/test_server.py @@ -6,6 +6,7 @@ import threading import urllib.error import urllib.request +from copy import deepcopy from norm import InMemoryGraph @@ -22,7 +23,9 @@ def _attrs(**kwargs: str) -> list[dict]: LOGS = { "resourceLogs": [ { - "resource": {"attributes": _attrs(**{"service.name": "noa-trip-planner-mas"})}, + "resource": { + "attributes": _attrs(**{"service.name": "noa-trip-planner-mas"}) + }, "scopeLogs": [ { "scope": {"name": "ioa_observe.runtime_events"}, @@ -84,13 +87,24 @@ def test_otlp_flattening_matches_the_collector_export_rows() -> None: assert span["SpanAttributes"] == {"n": "3", "ok": "true"} assert span["Timestamp"].endswith(".000000000") and span["Timestamp"][:4] == "2026" [log] = list(otlp.logs(LOGS)) - assert log["Body"] == "topology.session.started" and log["ServiceName"] == "noa-trip-planner-mas" + assert ( + log["Body"] == "topology.session.started" + and log["ServiceName"] == "noa-trip-planner-mas" + ) assert log["LogAttributes"]["session.id"] == SESSION -def _post(port: int, path: str, body: bytes, content_type: str = "application/json", **headers: str): +def _post( + port: int, + path: str, + body: bytes, + content_type: str = "application/json", + **headers: str, +): request = urllib.request.Request( - f"http://127.0.0.1:{port}{path}", data=body, headers={"Content-Type": content_type, **headers} + f"http://127.0.0.1:{port}{path}", + data=body, + headers={"Content-Type": content_type, **headers}, ) try: with urllib.request.urlopen(request) as response: @@ -107,13 +121,83 @@ def test_server_normalizes_logs_pushed_by_the_collector() -> None: port = server.server_address[1] try: assert _post(port, "/v1/logs", json.dumps(LOGS).encode()) == 200 - assert ("Session", "3aefd85c") in graph.nodes and ("MAS", "noa-trip-planner-mas") in graph.nodes + assert ("Session", "3aefd85c") in graph.nodes and ( + "MAS", + "noa-trip-planner-mas", + ) in graph.nodes # gzip, as the collector sends by default - assert _post(port, "/v1/logs", gzip.compress(json.dumps(LOGS).encode()), **{"Content-Encoding": "gzip"}) == 200 + assert ( + _post( + port, + "/v1/logs", + gzip.compress(json.dumps(LOGS).encode()), + **{"Content-Encoding": "gzip"}, + ) + == 200 + ) assert _post(port, "/v1/traces", json.dumps(TRACES).encode()) == 200 - assert _post(port, "/v1/logs", b"{", ) == 400 - assert _post(port, "/v1/logs", b"\x00", content_type="application/x-protobuf") == 415 + assert ( + _post( + port, + "/v1/logs", + b"{", + ) + == 400 + ) + assert ( + _post(port, "/v1/logs", b"\x00", content_type="application/x-protobuf") + == 415 + ) assert _post(port, "/v1/nope", b"{}") == 404 finally: server.shutdown() server.server_close() + + +def test_server_enriches_event_placeholders_with_completed_spans() -> None: + graph = InMemoryGraph() + logs = deepcopy(LOGS) + record = logs["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + record["body"]["stringValue"] = "topology.node.started" + record["attributes"] = _attrs( + **{ + "event.name": "topology.node.started", + "event.time": "2026-11-06T16:00:00+00:00", + "session.id": "svc_s1", + "application_id": "svc", + "agent.name": "Planner", + } + ) + traces = deepcopy(TRACES) + span = traces["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + span["spanId"] = record["spanId"] + span["name"] = "planner.agent" + span["attributes"] = _attrs( + **{ + "session.id": "svc_s1", + "application_id": "svc", + "agent_id": "planner-runtime", + "ioa_observe.entity.name": "Planner", + "ioa_observe.entity.output": "answer", + } + ) + server = make_server(graph, "127.0.0.1", 0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + port = server.server_address[1] + assert _post(port, "/v1/logs", json.dumps(logs).encode()) == 200 + assert _post(port, "/v1/traces", json.dumps(traces).encode()) == 200 + calls = [n for (t, _), n in graph.nodes.items() if t == "AgentCall"] + assert len(calls) == 1 + assert calls[0]["duration"] == 1000 + final = next( + e["to_id"] + for e in graph.edges.values() + if e["edge_type"] == "hasFinalState" and e["from_id"] == calls[0]["id"] + ) + assert graph.nodes[("State", final)]["content"] == "answer" + finally: + server.shutdown() + server.server_close() + thread.join() diff --git a/backend/workers/rt-norm-worker/tests/test_store.py b/backend/workers/rt-norm-worker/tests/test_store.py new file mode 100644 index 0000000..700967e --- /dev/null +++ b/backend/workers/rt-norm-worker/tests/test_store.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: Apache-2.0 + +from norm import InMemoryGraph, StreamDelta, SpanContext, StreamNormalizer + +from rt_norm_worker.store import Neo4jStore + + +class PopulatedDB: + def __init__(self): + self.commands = [] + self.reads = [] + + def execute(self, query, params): + self.reads.append((query, params)) + if "RETURN labels(n)" in query: + return [ + { + "label": "AgentCall", + "props": { + "id": "call", + "sessionId": "s1", + "rtNormalization": '__json__:{"spanComplete":true}', + }, + } + ] + return [ + { + "edge_type": "executesAgent", + "props": {"evidence": '__json__:{"source":"span"}'}, + "from_id": "call", + "from_label": "AgentCall", + "from_props": {"id": "call", "sessionId": "s1"}, + "to_id": "planner", + "to_label": "Agent", + "to_props": {"id": "planner", "name": "Planner"}, + } + ] + + def execute_command(self, command, params): + self.commands.append((command, params)) + + +def test_load_decodes_reconciliation_metadata_and_loads_edge_endpoints(): + db = PopulatedDB() + nodes, edges = Neo4jStore(db).load("s1", ["planner"]) + by_id = {n["id"]: n for n in nodes} + assert by_id["call"]["rtNormalization"] == {"spanComplete": True} + assert by_id["planner"] == { + "node_type": "Agent", + "id": "planner", + "name": "Planner", + } + assert edges == [ + { + "edge_type": "executesAgent", + "from_id": "call", + "to_id": "planner", + "from_type": "AgentCall", + "to_type": "Agent", + "evidence": {"source": "span"}, + } + ] + assert db.reads[0][1] == {"sid": "s1", "ids": ["planner"]} + assert "properties(r) AS props" in db.reads[1][0] + + +def test_apply_removes_superseded_edges_and_nodes_before_writing_replacements(): + db = PopulatedDB() + edge = { + "edge_type": "hasInitialState", + "from_id": "call", + "to_id": "old", + "from_type": "AgentCall", + "to_type": "State", + } + Neo4jStore(db).apply( + StreamDelta( + removed_edges=[edge], + removed_nodes=[{"node_type": "State", "id": "old"}], + nodes=[{"node_type": "State", "id": "new", "content": "question"}], + edges=[{**edge, "to_id": "new"}], + ) + ) + queries = [q for q, _ in db.commands] + assert queries[0].endswith("DELETE r") + assert queries[1].endswith("DETACH DELETE n") + assert queries[2].startswith("MERGE (n:State") + assert queries[3].startswith("MERGE (a:AgentCall") + assert db.commands[0][1] == {"from_id": "call", "to_id": "old"} + assert db.commands[1][1] == {"id": "old"} + assert db.commands[3][1]["to_id"] == "new" + + +def test_memory_store_applies_earlier_start_time_corrections(): + graph = InMemoryGraph() + node = {"node_type": "Session", "id": "s1", "startTime": 20.0, "endTime": 30.0} + graph.apply(StreamDelta(nodes=[node])) + graph.apply(StreamDelta(nodes=[{**node, "startTime": 10.0, "duration": 20000.0}])) + assert graph.nodes[("Session", "s1")]["startTime"] == 10.0 + + +def test_private_trace_context_stays_in_memory_and_is_scoped_to_store_lifetime(): + db = PopulatedDB() + store = Neo4jStore(db) + sessionless = SpanContext("trace", "server", "client", "", "") + client = SpanContext("trace", "client", "", "agent", "session") + other_trace = SpanContext("other", "server", "", "other-agent", "other-session") + store.apply(StreamDelta(span_contexts=[sessionless])) + store.apply(StreamDelta(span_contexts=[client, other_trace])) + assert set(store.load_span_contexts("trace")) == {sessionless, client} + assert store.load_span_contexts("other") == [other_trace] + assert store.load_span_contexts("different") == [] + store.apply(StreamDelta(span_contexts=[sessionless])) + assert set(store.load_span_contexts("trace")) == {sessionless, client} + enriched = SpanContext("trace", "server", "client", "agent", "session") + store.apply(StreamDelta(span_contexts=[enriched])) + assert set(store.load_span_contexts("trace")) == {enriched, client} + assert store.load_span_contexts("other") == [other_trace] + assert Neo4jStore(db).load_span_contexts("trace") == [] + assert db.commands == [] + assert db.reads == [] + + +def test_late_sessionless_transport_backfills_ownership_using_store_memory(monkeypatch): + db = PopulatedDB() + store = Neo4jStore(db) + graph = InMemoryGraph() + monkeypatch.setattr(store, "load", graph.load) + common = { + "TraceId": "trace", + "Timestamp": "2026-11-10 00:00:00.000000000", + "Duration": 1_000_000_000, + "SpanAttributes": {"application_id": "app", "session.id": "app_s1"}, + } + records = [ + { + **common, + "SpanName": "planner.agent", + "SpanId": "agent", + "SpanAttributes": {**common["SpanAttributes"], "agent_id": "planner"}, + }, + { + **common, + "SpanName": "search.tool", + "SpanId": "tool", + "ParentSpanId": "server", + "SpanAttributes": {**common["SpanAttributes"], "ioa_observe.entity.name": "search"}, + }, + { + **common, + "SpanName": "tools/call search", + "SpanId": "server", + "ParentSpanId": "agent", + "SpanAttributes": {}, + }, + ] + for record in records: + delta = StreamNormalizer(store).process(record, "span") + store.apply(delta) + graph.apply(delta) + graph.span_contexts.clear() + owner = next(n for (kind, _), n in graph.nodes.items() if kind == "AgentCall") + call = next(n for (kind, _), n in graph.nodes.items() if kind == "ToolCall") + assert ("hasToolCall", owner["id"], call["id"]) in graph.edges + assert store.load_span_contexts("trace") + assert db.reads == [] + assert not any("_NormalizationTrace" in query for query, _ in db.commands) + for record in records: + assert StreamNormalizer(store).process(record, "span") == StreamDelta() diff --git a/backend/workers/rt-norm-worker/tests/test_worker.py b/backend/workers/rt-norm-worker/tests/test_worker.py index 64b8733..4f8aae2 100644 --- a/backend/workers/rt-norm-worker/tests/test_worker.py +++ b/backend/workers/rt-norm-worker/tests/test_worker.py @@ -51,13 +51,13 @@ def test_neo4j_store_writes_delta(tmp_path): assert any("MERGE (a)-[r:" in c for c, _ in db.commands) -def test_existing_nodes_keep_start_time_but_update_end_time(): +def test_existing_nodes_accept_corrected_start_and_end_time(): db = FakeDB() Neo4jStore(db)._upsert_node( {"node_type": "AgentCall", "id": "x", "startTime": 1.0, "endTime": 2.0, "name": "n"} ) _, params = db.commands[0] assert params["props"]["startTime"] == 1.0 - assert "startTime" not in params["on_match"] - assert params["on_match"]["endTime"] == 2.0 - assert params["on_match"]["name"] == "n" + assert params["props"]["endTime"] == 2.0 + assert params["props"]["name"] == "n" + assert "SET n += $props" in db.commands[0][0]