From 0f47ca7382dfb5e338383ae9098d2d6413123eac Mon Sep 17 00:00:00 2001 From: Sri Aradhyula Date: Mon, 16 Nov 2026 18:00:37 -0600 Subject: [PATCH] feat(protocols): detect Deep Agents skill activation Signed-off-by: Sri Aradhyula --- .../detectors/skills.py | 25 +++++++++++++++++++ .../protocols/accord.py | 2 +- .../protocols/concord.py | 2 +- .../tests/test_detectors.py | 23 ++++++++++++++++- .../workers/cognition-protocols-worker.md | 12 ++++++++- 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/detectors/skills.py b/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/detectors/skills.py index 2b9d684..4d33f59 100644 --- a/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/detectors/skills.py +++ b/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/detectors/skills.py @@ -58,6 +58,10 @@ def __init__(self, name: str, *prefixes: str): self._listed = re.compile( rf"^\s*[-*]\s*\*\*({skill})\*\*", re.MULTILINE ) # in a system prompt's skill list + self._catalog_paths = re.compile( + rf"^\s*[-*]\s*\*\*({skill})\*\*:[^\n]*\n\s*-> Read `(/skills/[^`\n]+/SKILL\.md)` for full instructions", + re.MULTILINE, + ) self._activated = re.compile( rf'"name"\s*:\s*"({skill})"' ) # activate_skill arguments @@ -69,10 +73,16 @@ def detect(self, context: dict[str, Any]) -> ProtocolFinding: spans = SessionSpans.of(context) found = ProtocolFinding(self.name) + paths: dict[tuple[str, str], str] = {} for span in spans.llms: listed = self._listed.findall( _system_text(span.get("prompt") or span.get("inputContent")) ) + paths.update({ + (spans.agent_of(span), path): name for name, path in self._catalog_paths.findall( + _system_text(span.get("prompt") or span.get("inputContent")) + ) if ".." not in path.split("/") + }) if listed: bucket = found.enabled_by_agent.setdefault(spans.agent_of(span), []) bucket.extend(s for s in listed if s not in bucket) @@ -84,6 +94,21 @@ def detect(self, context: dict[str, Any]) -> ProtocolFinding: or span.get("inputContent") or span.get("inputParams") ) + if tool == "read_file": + # Deep Agents activates skills by loading their advertised SKILL.md. + # Require an exact catalog path and a successful tool result; this + # proves loading, not script execution or protocol completion. + try: + payload = json.loads(arguments) + except (TypeError, ValueError): + continue + path = payload.get("file_path") if isinstance(payload, dict) else None + name = paths.get((spans.agent_of(span), path)) if isinstance(path, str) else None + if name and span.get("success") is True and not span.get("contains_error"): + bucket = found.activated_by_agent.setdefault(spans.agent_of(span), []) + if name not in bucket: + bucket.append(name) + continue pattern = ( self._activated if tool == "activate_skill" diff --git a/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/accord.py b/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/accord.py index 8f6f5ac..4aab9f3 100644 --- a/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/accord.py +++ b/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/accord.py @@ -13,7 +13,7 @@ def detector() -> SkillProtocolDetector: - return SkillProtocolDetector(NAME, SKILL_PREFIX) + return SkillProtocolDetector(NAME, SKILL_PREFIX, "accord-") def evaluator() -> McEvaluator: diff --git a/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/concord.py b/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/concord.py index b901a41..12186a8 100644 --- a/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/concord.py +++ b/backend/workers/cognition-protocols-worker/src/cognition_protocols_worker/protocols/concord.py @@ -18,7 +18,7 @@ def detector() -> SkillProtocolDetector: - return SkillProtocolDetector(NAME, SKILL_PREFIX) + return SkillProtocolDetector(NAME, SKILL_PREFIX, "concord-") def evaluator() -> McEvaluator: diff --git a/backend/workers/cognition-protocols-worker/tests/test_detectors.py b/backend/workers/cognition-protocols-worker/tests/test_detectors.py index 42c2079..11c7c28 100644 --- a/backend/workers/cognition-protocols-worker/tests/test_detectors.py +++ b/backend/workers/cognition-protocols-worker/tests/test_detectors.py @@ -119,7 +119,7 @@ def test_a_broken_or_clashing_entry_point_is_skipped_not_fatal(monkeypatch): registry.load_entry_points() assert registry.names() == ["accord", "concord", "tooly"] assert registry.get("accord").prefixes == ( - "l9-accord-", + "l9-accord-", "accord-", ) # the clash did not replace the built-in @@ -136,3 +136,24 @@ def test_the_skill_worker_default_is_every_registered_protocol(monkeypatch): def factory(): return SkillProtocolDetector("factory-made", "l9-fact-") + + +@pytest.mark.parametrize('protocol', ['accord', 'concord']) +@pytest.mark.parametrize('success,owner,advertised,activated', [ + (True, 'a1', True, True), (False, 'a1', True, False), + (True, 'a2', True, False), (True, 'a1', False, False), +]) +def test_deepagents_activation_requires_successful_own_catalog_read(protocol, success, owner, advertised, activated): + name = f'{protocol}-initiator' + path = '/skills/hub-example/example/SKILL.md' + ctx = { + 'agent_spans': AGENTS, + 'llm_spans': [{'parentSpanId': 'a1', 'prompt': + f'- **{name}**:\n -> Read `{path}` for full instructions' if advertised else ''}], + 'tool_spans': [{'parentSpanId': owner, 'toolName': 'read_file', + 'toolArguments': {'file_path': path}, 'success': success}], + } + result = detect(ctx, builtin_detectors())[protocol] + assert result.enabled is advertised + assert result.activated is activated + assert result.skills_activated == ([name] if activated else []) diff --git a/docs/analysis/workers/cognition-protocols-worker.md b/docs/analysis/workers/cognition-protocols-worker.md index a394594..3a974e9 100644 --- a/docs/analysis/workers/cognition-protocols-worker.md +++ b/docs/analysis/workers/cognition-protocols-worker.md @@ -38,7 +38,8 @@ All graph access goes through the platform's providers (`SCOPEKGProvider` to rea ## Built-in protocols -ACCORD and CONCORD are built in. +ACCORD and CONCORD are built in. Released `accord-*` / `concord-*` skill names +and legacy `l9-accord-*` / `l9-concord-*` names are supported. | Protocol | Detector (skills) | Evaluator (MCE metrics) | |---|---|---| @@ -54,6 +55,15 @@ Both are skill-based, and handled by the same `SkillProtocolDetector`: storage is still seen as long as the skill name survived. A skill that was called but is not listed in any readable prompt still counts as enabled. +Deep Agents skill loading is also recognized: a successful `read_file` of an exact +`/skills/.../SKILL.md` path advertised in that agent's readable system skill catalog +counts as activation. Failed reads, unadvertised paths, and another agent's catalog +do not count. The catalog must include the `-> Read `` for full instructions` +line directly after the skill entry. This establishes that instructions were loaded, +not that protocol scripts completed or an agreement was reached. Metadata-only +exporters may send just these catalog entries and file paths; full descriptions, +conversation content and tool results are not needed for detection. + The CONCORD metrics read the initiator script's own calls. A session where only a specialist activated the receiver is therefore recorded as activated, and every metric is skipped. A skipped metric is never stored, so it never reads as `0`.