Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions backend/api/scope/client/applications.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
ApplicationNamesResponse,
ApplicationsResponse,
CollectByApplicationResponse,
CognitiveAssessment,
CognitiveFailure,
CognitiveObservabilityMetric,
CognitiveSessionItem,
Expand Down Expand Up @@ -2229,6 +2230,12 @@ def _get_application_sessions_with_cognitive_observability(
row.get("sessionMetrics")
),
cognitiveFailures=_build_cognitive_failures(row.get("failures")),
cognitiveAssessment=CognitiveAssessment(
status=row.get("assessmentStatus") or "unknown",
reason=row.get("assessmentReason") or "",
requiredEvidence=row.get("requiredEvidence") or 0,
reviewedEvidence=row.get("reviewedEvidence") or 0,
),
)
for row in rows or []
]
Expand Down
35 changes: 35 additions & 0 deletions backend/api/scope/client/failures.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,41 @@
logger = logging.getLogger(__name__)


def write_cognitive_assessment(
db: Connector,
session_id: str,
status: str,
reason: str = "",
coverage: Mapping[str, Any] | None = None,
) -> None:
"""Record review completion independently of whether any failure was found."""
if (
status not in {"in_progress", "complete", "incomplete", "error"}
or not session_id
or not session_id.strip()
):
raise ValueError("Invalid cognitive assessment status or session")
coverage = coverage or {}
try:
db.execute(
"MATCH (s:Session {sessionId:$session}) "
"SET s.cognitiveAssessmentStatus=$status, s.cognitiveAssessmentReason=$reason, "
"s.cognitiveAssessmentRequiredEvidence=$required, s.cognitiveAssessmentReviewedEvidence=$reviewed "
"RETURN s.sessionId AS id",
{
"session": session_id,
"status": status,
"reason": reason,
"required": coverage.get("required", 0),
"reviewed": coverage.get("reviewed", 0),
},
)
except Exception as exc:
raise DatabaseError(
f"Failed to write cognitive assessment for '{session_id}': {exc}"
) from exc


def write_cognitive_failure_reports(
db: Connector,
session_id: str,
Expand Down
10 changes: 10 additions & 0 deletions backend/api/scope/models/otel_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,11 +340,21 @@ class CognitiveFailure(BaseModel):
reasoning: str = ""


class CognitiveAssessment(BaseModel):
"""Whether sufficient captured evidence was delivered for the assessment."""

status: Literal["unknown", "in_progress", "complete", "incomplete", "error"] = "unknown"
reason: str = ""
requiredEvidence: int = 0
reviewedEvidence: int = 0


class CognitiveSessionItem(CollectSessionItem):
"""A session with its cognitive observability metrics and failures."""

cognitiveObservabilityMetrics: list[CognitiveObservabilityMetric] = []
cognitiveFailures: list[CognitiveFailure] = []
cognitiveAssessment: CognitiveAssessment = Field(default_factory=CognitiveAssessment)


class SessionsWithCognitiveObservabilityResponse(BaseModel):
Expand Down
6 changes: 5 additions & 1 deletion backend/api/scope/query_builders/ui.py
Original file line number Diff line number Diff line change
Expand Up @@ -3365,7 +3365,11 @@ def neo4j_sessions_with_cognitive_observability_query(
tokens,
sessionMetrics,
failures,
cognitionProtocolReports
cognitionProtocolReports,
s.cognitiveAssessmentStatus AS assessmentStatus,
s.cognitiveAssessmentReason AS assessmentReason,
s.cognitiveAssessmentRequiredEvidence AS requiredEvidence,
s.cognitiveAssessmentReviewedEvidence AS reviewedEvidence
ORDER BY s.startTime DESC
"""
return stmt, {
Expand Down
46 changes: 46 additions & 0 deletions backend/api/tests/test_cognitive_assessment.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
# SPDX-License-Identifier: Apache-2.0
from unittest.mock import Mock

import pytest

from scope.client.failures import write_cognitive_assessment
from scope.core.exceptions import DatabaseError


def test_assessment_uses_session_id_and_parameterized_coverage():
db = Mock()
write_cognitive_assessment(
db,
"session-1",
"incomplete",
"Evidence pending",
{"required": 10, "reviewed": 8},
)
query, params = db.execute.call_args.args
assert "sessionId:$session" in query
assert "cognitiveAssessmentStatus=$status" in query
assert params == {
"session": "session-1",
"status": "incomplete",
"reason": "Evidence pending",
"required": 10,
"reviewed": 8,
}


@pytest.mark.parametrize(
"session,status", [("", "complete"), ("s", "clean"), (None, "complete")]
)
def test_invalid_state_is_rejected_before_database_access(session, status):
db = Mock()
with pytest.raises(ValueError):
write_cognitive_assessment(db, session, status)
db.execute.assert_not_called()


def test_database_failures_are_not_silently_treated_as_complete():
db = Mock()
db.execute.side_effect = RuntimeError("unavailable")
with pytest.raises(DatabaseError):
write_cognitive_assessment(db, "s", "complete")
40 changes: 34 additions & 6 deletions backend/api/tests/test_cognitive_observability.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,23 @@ def _row():
"tokens": 5,
"sessionMetrics": [
{"name": "Policy Safety", "result": 0.9, "metric_id": "x"},
{"name": "task_completion", "result": '{"value": 0.5}'},
{"name": "task_completion", "result": 0.5},
{"name": "Cost", "result": 0.1},
{"name": "Goal Alignment", "result": None, "value": None},
],
"failures": [
{"name": "IR-1", "code": "IR-1", "confidence": "medium", "reasoning": "lost it"},
{"name": "IR-1", "code": "IR-1", "confidence": "high", "reasoning": "lost it again"},
{
"name": "IR-1",
"code": "IR-1",
"confidence": "medium",
"reasoning": "lost it",
},
{
"name": "IR-1",
"code": "IR-1",
"confidence": "high",
"reasoning": "lost it again",
},
{"name": "CC-3", "code": "CC-3", "confidence": "low", "reasoning": "weak"},
{"code": "DC-4", "confidence": "HIGH", "reasoning": "no name"},
{"name": "No confidence", "code": "X"},
Expand All @@ -60,9 +70,7 @@ def test_sessions_with_cognitive_observability_mapping():
("Policy Safety", 0.9, "PERCENTAGE"),
("Task Completion", 0.5, "PERCENTAGE"),
]
assert [
(f.name, f.confidence, f.reasoning) for f in session.cognitiveFailures
] == [
assert [(f.name, f.confidence, f.reasoning) for f in session.cognitiveFailures] == [
("IR-1", "high", "lost it again"),
("DC-4", "high", "no name"),
("CC-3", "low", "weak"),
Expand All @@ -74,3 +82,23 @@ def test_sessions_with_cognitive_observability_empty():
_StubDB([]), Dialect.NEO4J, application_id="app"
)
assert resp.sessions == []


def test_assessment_state_round_trips_and_legacy_data_stays_unknown():
row = _row()
row.update(
assessmentStatus="incomplete",
assessmentReason="Missing evidence",
requiredEvidence=12,
reviewedEvidence=10,
)
response = _get_application_sessions_with_cognitive_observability(
_StubDB([row, _row()]), Dialect.NEO4J, application_id="app"
)
assert response.sessions[0].cognitiveAssessment.model_dump() == {
"status": "incomplete",
"reason": "Missing evidence",
"requiredEvidence": 12,
"reviewedEvidence": 10,
}
assert response.sessions[1].cognitiveAssessment.status == "unknown"
Loading
Loading