From c617cb97eac4820b78f1cd90cb4629eb00e68d46 Mon Sep 17 00:00:00 2001 From: zljie Date: Tue, 14 Apr 2026 14:13:24 +0800 Subject: [PATCH 1/3] feat(core-spec): add semantic_model.behavior with actions/effects --- core-spec/behavior-layer.md | 74 +++++++++++++ core-spec/behavior-layer.schema.json | 97 +++++++++++++++++ core-spec/osi-schema.json | 88 ++++++++++++++- core-spec/spec.md | 80 +++++++++++++- core-spec/spec.yaml | 71 +++++++++++- examples/p2p_behavior_effects_minimal.yaml | 119 +++++++++++++++++++++ examples/tpcds_semantic_model.yaml | 2 +- 7 files changed, 526 insertions(+), 5 deletions(-) create mode 100644 core-spec/behavior-layer.md create mode 100644 core-spec/behavior-layer.schema.json create mode 100644 examples/p2p_behavior_effects_minimal.yaml diff --git a/core-spec/behavior-layer.md b/core-spec/behavior-layer.md new file mode 100644 index 000000000..5e3063650 --- /dev/null +++ b/core-spec/behavior-layer.md @@ -0,0 +1,74 @@ +# OSI Extension: Behavior Layer (Actions, Rules, Effects) + +**Applies to:** OSI Core v0.1.2 +**Goal:** Provide a vendor-agnostic behavior layer for deterministic action planning and attribution (what changed/why), without breaking OSI Core compatibility. + +## 1) Placement & Compatibility + +This extension supports two equivalent placements: + +1. **Preferred (first-class):** `semantic_model[].behavior` +2. **Legacy (embedded):** behavior JSON embedded in `dataset.custom_extensions[].data` (as a JSON string) + +Backwards compatibility guidance: +- New models SHOULD use `semantic_model.behavior`. +- Tools MAY continue to support legacy embedded behavior for older models. +- If both are present, tools SHOULD define precedence (recommended: first-class `semantic_model.behavior` wins). + +## 2) Top-level object: Behavior + +Minimal structure: + +| Field | Type | Required | Notes | +|---|---|---:|---| +| `namespace` | string | Yes | Grouping namespace (e.g. `SAP_P2P`) | +| `behavior_layer_version` | string | Yes | Schema evolution version | +| `actions` | array | No* | Preferred list of actions (`actions` or `action_types` must exist) | +| `action_types` | array | No* | Legacy alias of `actions` | +| `rules` | array | Yes | Constraints/guards for planning and governance | +| `metadata` | object | No | Optional governance metadata | + +## 3) Actions + +Actions describe what a tool/agent can do (API calls, workflows, scripts). OSI does not assume SQL. + +Recommended action fields: +- `id` (stable identifier, e.g. `suppliers/block`) +- `title`, `description` +- `kind`: `command` or `query` +- `operation`: free-form operation name (e.g. `block`, `unblock`, `analyze`) +- `entity_name`: dataset name or conceptual entity +- `io_schema`: optional input/output JSON schema +- `effects`: optional machine-readable impact annotations + +## 4) Effects (impact annotations) + +Effects encode how an action reads/writes/derives datasets/fields. + +Minimal effect fields: +- `entity`: dataset / field / metric / relationship +- `mode`: read / write / derive +- `selectors`: `{ dataset, field_names[] }` (or other selectors) +- optional: `impact_type`, `transition`, `set_value`, `confidence`, `notes` + +Typical uses: +- Deterministic planning: validate state transitions and required prerequisites +- Attribution: explain “what changed/why” for a field or dataset + +## 5) Legacy embedded example (dataset.custom_extensions) + +```yaml +datasets: + - name: suppliers + source: sap.p2p.suppliers + custom_extensions: + - vendor_name: COMMON + data: | + { + "namespace": "SAP_P2P", + "behavior_layer_version": "0.1", + "action_types": [], + "rules": [] + } +``` + diff --git a/core-spec/behavior-layer.schema.json b/core-spec/behavior-layer.schema.json new file mode 100644 index 000000000..999eda1a2 --- /dev/null +++ b/core-spec/behavior-layer.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/behavior-layer.schema.json", + "title": "OSI Behavior Layer (First-class)", + "description": "Schema for semantic_model.behavior (actions/rules/effects). Supports legacy alias action_types.", + "type": "object", + "additionalProperties": true, + "required": ["namespace", "behavior_layer_version", "rules"], + "anyOf": [ + { "required": ["actions"] }, + { "required": ["action_types"] } + ], + "properties": { + "namespace": { "type": "string", "minLength": 1 }, + "behavior_layer_version": { "type": "string", "minLength": 1 }, + "metadata": { "type": "object", "additionalProperties": true }, + "actions": { + "type": "array", + "description": "Preferred: list of actions supported by this semantic model. (Alias of legacy action_types)", + "items": { "$ref": "#/$defs/actionType" } + }, + "action_types": { + "type": "array", + "description": "Legacy alias of actions. New models SHOULD use actions.", + "items": { "$ref": "#/$defs/actionType" } + }, + "rules": { + "type": "array", + "items": { "$ref": "#/$defs/rule" } + } + }, + "$defs": { + "actionType": { + "type": "object", + "additionalProperties": true, + "required": ["id", "title"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "kind": { "type": "string", "enum": ["command", "query"] }, + "operation": { "type": "string", "description": "Free-form operation name." }, + "aggregate": { "type": "string" }, + "entity_name": { "type": "string" }, + "idempotency": { "type": "string", "enum": ["idempotent", "non_idempotent", "unknown"] }, + "applies_to": { "type": "object", "additionalProperties": true }, + "io_schema": { "type": "object", "additionalProperties": true }, + "effects": { + "type": "array", + "description": "Optional: machine-readable impact annotations (what this action reads/writes/derives).", + "items": { "$ref": "#/$defs/effect" } + }, + "examples": { "type": "array", "items": { "type": "string" } }, + "tool_hint": { "type": "object", "additionalProperties": true }, + "tags": { "type": "array", "items": { "type": "string" } }, + "synonyms": { "type": "array", "items": { "type": "string" } }, + "deprecated": { "type": "boolean" }, + "version": { "type": "string" } + } + }, + "effect": { + "type": "object", + "additionalProperties": true, + "required": ["entity", "mode"], + "properties": { + "entity": { "type": "string", "enum": ["dataset", "field", "metric", "relationship"] }, + "mode": { "type": "string", "enum": ["read", "write", "derive"] }, + "impact_type": { "type": "string" }, + "selectors": { "type": "object", "additionalProperties": true }, + "transition": { "type": "object", "additionalProperties": true }, + "set_value": { "type": "string" }, + "confidence": { "type": "string", "enum": ["guaranteed", "likely", "unknown"] }, + "notes": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + } + }, + "rule": { + "type": "object", + "additionalProperties": true, + "required": ["id", "title", "severity", "when", "constraint", "message"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "severity": { "type": "string", "enum": ["error", "warn", "info"] }, + "when": { "type": "object", "additionalProperties": true }, + "if": { "type": "object", "additionalProperties": true }, + "constraint": { "type": "object", "additionalProperties": true }, + "message": { "type": "string", "minLength": 1 }, + "remediation": { "type": "string" }, + "references": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "tags": { "type": "array", "items": { "type": "string" } } + } + } + } +} + diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index 30210d18e..c0b38bd6d 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -7,7 +7,7 @@ "properties": { "version": { "type": "string", - "const": "0.1.1", + "const": "0.1.2", "description": "OSI specification version" }, "dialects": { @@ -92,6 +92,89 @@ "required": ["vendor_name", "data"], "additionalProperties": false }, + "Behavior": { + "type": "object", + "description": "Optional behavior layer for deterministic action planning and attribution (actions/rules/effects).", + "additionalProperties": true, + "required": ["namespace", "behavior_layer_version", "rules"], + "anyOf": [ + { "required": ["actions"] }, + { "required": ["action_types"] } + ], + "properties": { + "namespace": { "type": "string", "minLength": 1 }, + "behavior_layer_version": { "type": "string", "minLength": 1 }, + "metadata": { "type": "object", "additionalProperties": true }, + "actions": { + "type": "array", + "items": { "$ref": "#/$defs/BehaviorAction" } + }, + "action_types": { + "type": "array", + "description": "Legacy alias of actions.", + "items": { "$ref": "#/$defs/BehaviorAction" } + }, + "rules": { + "type": "array", + "items": { "$ref": "#/$defs/BehaviorRule" } + } + } + }, + "BehaviorAction": { + "type": "object", + "additionalProperties": true, + "required": ["id", "title"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "kind": { "type": "string", "enum": ["command", "query"] }, + "operation": { "type": "string" }, + "entity_name": { "type": "string" }, + "idempotency": { "type": "string", "enum": ["idempotent", "non_idempotent", "unknown"] }, + "applies_to": { "type": "object", "additionalProperties": true }, + "io_schema": { "type": "object", "additionalProperties": true }, + "effects": { + "type": "array", + "items": { "$ref": "#/$defs/BehaviorEffect" } + }, + "tags": { "type": "array", "items": { "type": "string" } }, + "synonyms": { "type": "array", "items": { "type": "string" } } + } + }, + "BehaviorEffect": { + "type": "object", + "additionalProperties": true, + "required": ["entity", "mode"], + "properties": { + "entity": { "type": "string", "enum": ["dataset", "field", "metric", "relationship"] }, + "mode": { "type": "string", "enum": ["read", "write", "derive"] }, + "impact_type": { "type": "string" }, + "selectors": { "type": "object", "additionalProperties": true }, + "transition": { "type": "object", "additionalProperties": true }, + "set_value": { "type": "string" }, + "confidence": { "type": "string", "enum": ["guaranteed", "likely", "unknown"] }, + "notes": { "type": "string" } + } + }, + "BehaviorRule": { + "type": "object", + "description": "Declarative rule/constraint for planning and governance.", + "additionalProperties": true, + "required": ["id", "title", "severity", "when", "constraint", "message"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "severity": { "type": "string", "enum": ["error", "warn", "info"] }, + "when": { "type": "object", "additionalProperties": true }, + "if": { "type": "object", "additionalProperties": true }, + "constraint": { "type": "object", "additionalProperties": true }, + "message": { "type": "string", "minLength": 1 }, + "remediation": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + } + }, "DialectExpression": { "type": "object", "description": "Expression in a specific dialect", @@ -330,6 +413,9 @@ }, "description": "Quantifiable measures spanning datasets" }, + "behavior": { + "$ref": "#/$defs/Behavior" + }, "custom_extensions": { "type": "array", "items": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 33da9bda3..72efb85b0 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -1,6 +1,6 @@ # OSI - Core Metadata Specification -**Version:** 0.1.1 +**Version:** 0.1.2 ## Goals @@ -16,7 +16,8 @@ 4. [Relationships](#relationships) 5. [Fields](#fields) 6. [Metrics](#metrics) -7. [Examples](#examples) +7. [Behavior (Actions / Rules / Effects)](#behavior-actions--rules--effects) +8. [Examples](#examples) --- @@ -62,6 +63,7 @@ The top-level container that represents a complete semantic model, including dat | `datasets` | array | Yes | Collection of logical datasets (fact and dimension tables) | | `relationships` | array | No | Defines how logical datasets are connected | | `metrics` | array | No | Quantifiable measures defined as aggregate expessions on fields from logical datsets | +| `behavior` | object | No | Optional behavior layer for deterministic action planning and attribution | | `custom_extensions` | array | No | Vendor-specific attributes for extensibility | ### Example @@ -75,6 +77,11 @@ semantic_model: datasets: [] relationships: [] metrics: [] + behavior: + namespace: "SALES" + behavior_layer_version: "0.1" + actions: [] + rules: [] custom_extensions: - vendor_name: DBT data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' @@ -82,6 +89,75 @@ semantic_model: --- +## Behavior (Actions / Rules / Effects) + +Behavior is an optional, vendor-agnostic layer for **deterministic action planning** and **attribution** (what changed/why). + +This repository supports two equivalent placements: +1) **Preferred (first-class):** `semantic_model[].behavior` +2) **Backwards-compatible (legacy):** embed behavior-layer JSON under `custom_extensions` (e.g. `vendor_name: COMMON`) + +### Schema (semantic_model.behavior) + +| Field | Type | Required | Description | +|------|------|----------|-------------| +| `namespace` | string | Yes | Namespace/group for actions and rules (e.g. `SAP_P2P`) | +| `behavior_layer_version` | string | Yes | Version for behavior schema evolution | +| `actions` | array | No* | Preferred list of actions (`actions` or `action_types` required) | +| `action_types` | array | No* | Legacy alias of `actions` (for backwards compatibility) | +| `rules` | array | Yes | Declarative constraints/guards for planning and governance | +| `metadata` | object | No | Governance metadata (owner/tags/last_updated, etc.) | + +### Action Schema (high-level) + +Actions represent executable intents (API calls, workflows, tools). This spec intentionally does **not** assume SQL. + +| Field | Type | Required | Description | +|------|------|----------|-------------| +| `id` | string | Yes | Action id, e.g. `suppliers/block` | +| `title` | string | Yes | Human-friendly title | +| `kind` | string | No | `command` or `query` | +| `operation` | string | No | Free-form operation name (e.g. `block`, `unblock`, `analyze`) | +| `effects` | array | No | Optional machine-readable impact annotations | + +### Effects (impact annotations) + +Effects encode how an action changes datasets/fields for: +- plan validation (state transitions / prohibited follow-ups) +- attribution (what changed/why for a field or dataset) + +Minimal effect fields: +`entity` (dataset/field/metric/relationship), `mode` (read/write/derive), `selectors` (dataset + field_names), +optional `impact_type`, `transition`, `set_value`. + +### Example + +```yaml +semantic_model: + - name: sap_p2p + datasets: [] + behavior: + namespace: "SAP_P2P" + behavior_layer_version: "0.1" + actions: + - id: suppliers/block + title: Block supplier + kind: command + operation: block + entity_name: suppliers + effects: + - entity: field + mode: write + impact_type: state_transition + selectors: + dataset: suppliers + field_names: [status] + set_value: Blocked + rules: [] +``` + +--- + ## Datasets Logical datasets represent business entities or concepts (fact and dimension tables). They contain fields and define the structure of the data. diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index d27a9bac2..bd5618715 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -1,5 +1,5 @@ # OSI - Core Metadata Spec (YAML Schema) -version: 0.1.1 +version: 0.1.2 # # Goals: # - Standardization: Establish uniform language and structure for semantic model definitions @@ -59,6 +59,75 @@ semantic_model: - vendor_name: string # Must be one of the values from 'vendors' enum above data: string + # Optional: + # Behavior layer (actions/rules/effects) promoted to a first-class, vendor-agnostic structure. + # Intended for deterministic action planning and attribution (what changed/why). + # + # Backwards compatibility: + # - Tools MAY still embed behavior-layer JSON in custom_extensions (e.g. vendor_name: COMMON). + # - If both semantic_model.behavior and embedded behavior are provided, tools SHOULD define precedence + # (recommended: semantic_model.behavior takes precedence). + behavior: {} + +--- +# Behavior Schema (Optional) +# A vendor-agnostic behavior layer for deterministic action planning and attribution. +# This is intentionally separate from SQL/query generation: actions can map to APIs, workflows, or tools. +behavior: + # Required: Namespace for grouping actions/rules (e.g., "SAP_P2P", "CRM") + namespace: string + + # Required: Version of the behavior schema used by this model + behavior_layer_version: string + + # Optional: Metadata for governance/audit + metadata: {} + + # Preferred: List of actions supported by this semantic model + actions: [] + + # Legacy alias (compat): actions MAY also be provided as action_types + # New models SHOULD use actions. + action_types: [] + + # Optional: Declarative rules/constraints for planning and validation + rules: [] + +--- +# Action Schema (within behavior.actions) +actions: + - id: string # e.g. "suppliers/block" + title: string # Human-friendly title + description: string + kind: "command"|"query" + operation: string # free-form, e.g. "block"|"unblock"|"create"|"analyze" + entity_name: string # dataset name or conceptual entity + idempotency: "idempotent"|"non_idempotent"|"unknown" + applies_to: {} # optional selectors (dataset/field/metric/relationship) + io_schema: {} # optional input/output JSON schema for tool execution + effects: [] # optional: machine-readable impact annotations + tags: [] + synonyms: [] + +--- +# Effect Schema (within action.effects) +# Used for: +# - Planning: understand state transitions and prohibited/required follow-up actions +# - Attribution: explain "what changed/why" for fields/datasets +effects: + - entity: "dataset"|"field"|"metric"|"relationship" + mode: "read"|"write"|"derive" + impact_type: "state_transition"|"master_data_mutation"|"transactional_write"|"derived_metric_change"|"other" + selectors: + dataset: string + field_names: [] + transition: + from: string + to: string + set_value: string + confidence: "guaranteed"|"likely"|"unknown" + notes: string + --- # Logical Dataset Schema # Represents business entities or concepts (fact and dimension tables) diff --git a/examples/p2p_behavior_effects_minimal.yaml b/examples/p2p_behavior_effects_minimal.yaml new file mode 100644 index 000000000..1da4c6a5f --- /dev/null +++ b/examples/p2p_behavior_effects_minimal.yaml @@ -0,0 +1,119 @@ +# yaml-language-server: $schema=../core-spec/osi-schema.json +version: 0.1.2 + +semantic_model: + - name: sap_p2p_minimal_behavior + description: "Minimal example: first-class behavior/actions/effects for SAP P2P" + + datasets: + - name: suppliers + source: sap.p2p.suppliers + primary_key: [supplier_id] + description: "Supplier master data" + fields: + - name: supplier_id + expression: + dialects: + - dialect: ANSI_SQL + expression: supplier_id + description: "Supplier ID" + + - name: supplier_name + expression: + dialects: + - dialect: ANSI_SQL + expression: supplier_name + description: "Supplier name" + + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + description: "Supplier lifecycle / blocking status (e.g., Active, Blocked)" + + behavior: + namespace: "SAP_P2P" + behavior_layer_version: "0.1" + metadata: + owner: "example" + tags: ["p2p", "behavior", "effects"] + + actions: + - id: suppliers/block + title: Block supplier + description: Block a supplier and prevent downstream procurement actions. + kind: command + operation: block + entity_name: suppliers + idempotency: idempotent + io_schema: + input_schema: + type: object + required: [supplier_id] + properties: + supplier_id: { type: string } + reason: { type: string } + output_schema: + type: object + properties: + supplier_id: { type: string } + status: { type: string } + effects: + - entity: field + mode: write + impact_type: state_transition + selectors: + dataset: suppliers + field_names: [status] + set_value: Blocked + confidence: guaranteed + notes: "After blocking, procurement actions should be denied by policy." + + - id: suppliers/unblock + title: Unblock supplier + description: Unblock a supplier. + kind: command + operation: unblock + entity_name: suppliers + idempotency: idempotent + io_schema: + input_schema: + type: object + required: [supplier_id] + properties: + supplier_id: { type: string } + comment: { type: string } + output_schema: + type: object + properties: + supplier_id: { type: string } + status: { type: string } + effects: + - entity: field + mode: write + impact_type: state_transition + selectors: + dataset: suppliers + field_names: [status] + transition: + from: Blocked + to: Active + confidence: likely + notes: "Exact target status may vary by ERP configuration." + + rules: + - id: sap_p2p/rule_blocked_vendor_transaction_prevention + title: Blocked supplier prevents procurement transactions + severity: error + when: + entity: dataset + selectors: + dataset: suppliers + field_names: [status] + constraint: + type: state_machine + rule: "If suppliers.status == 'Blocked' then deny procurement transactions." + message: "Supplier is blocked. Procurement transactions are not allowed." + remediation: "Run suppliers/unblock or contact an administrator to unblock the supplier." + diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 98e956b1b..18441e552 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -4,7 +4,7 @@ # This example demonstrates the OSI Core Metadata Spec using the TPC-DS benchmark schema # TPC-DS is a decision support benchmark with a realistic retail business model -version: "0.1.1" +version: "0.1.2" semantic_model: - name: tpcds_retail_model From 3688717ef023720b3e7fa53925925c7fbb4f29b8 Mon Sep 17 00:00:00 2001 From: "Johnson.zhao" Date: Fri, 9 Oct 2026 16:00:51 +0800 Subject: [PATCH 2/3] style(examples): remove trailing blank line --- examples/p2p_behavior_effects_minimal.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/p2p_behavior_effects_minimal.yaml b/examples/p2p_behavior_effects_minimal.yaml index 68d63ebe8..2d6d95955 100644 --- a/examples/p2p_behavior_effects_minimal.yaml +++ b/examples/p2p_behavior_effects_minimal.yaml @@ -132,4 +132,3 @@ behavior: rule: "If suppliers.status == 'Blocked' then deny procurement transactions." message: "Supplier is blocked. Procurement transactions are not allowed." remediation: "Run suppliers/unblock or contact an administrator to unblock the supplier." - From d07d8dc38a0dd2eac43f5f801fc5d9d1d307b582 Mon Sep 17 00:00:00 2001 From: "Johnson.zhao" Date: Tue, 13 Oct 2026 14:50:19 +0800 Subject: [PATCH 3/3] fix(core-spec): make behavior validation portable and consistent --- .github/workflows/validation-ci.yml | 5 + .../tests/test_ossie_behavior_validation.py | 164 ++++++++++ .../SchemaValidatorBehaviorTest.java | 179 +++++++++++ core-spec/behavior-layer.md | 10 +- core-spec/ossie-schema.json | 298 +++++++++++++++++- core-spec/spec.md | 6 +- core-spec/spec.yaml | 3 +- python/src/ossie/models.py | 114 ++++++- python/tests/test_models.py | 168 ++++++++++ python/tests/test_serialization.py | 9 +- scripts/generate-behavior-schema.py | 66 ++++ validation/tests/test_validate.py | 73 ++++- 12 files changed, 1080 insertions(+), 15 deletions(-) create mode 100644 converters/orionbelt/tests/test_ossie_behavior_validation.py create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/validator/SchemaValidatorBehaviorTest.java create mode 100644 scripts/generate-behavior-schema.py diff --git a/.github/workflows/validation-ci.yml b/.github/workflows/validation-ci.yml index dd6f14c3e..f063cc1f5 100644 --- a/.github/workflows/validation-ci.yml +++ b/.github/workflows/validation-ci.yml @@ -32,6 +32,7 @@ on: - "examples/p2p_behavior_effects_minimal.yaml" - ".github/workflows/validation-ci.yml" - "scripts/generate-spec-types.py" + - "scripts/generate-behavior-schema.py" pull_request: branches: ["main"] paths: @@ -44,6 +45,7 @@ on: - "examples/p2p_behavior_effects_minimal.yaml" - ".github/workflows/validation-ci.yml" - "scripts/generate-spec-types.py" + - "scripts/generate-behavior-schema.py" permissions: contents: read @@ -82,6 +84,9 @@ jobs: - name: Check spec.yaml enums are in sync with ossie-schema.json run: uv run scripts/generate-spec-types.py --check + - name: Check embedded behavior schema is in sync with its canonical source + run: python3 scripts/generate-behavior-schema.py --check + - name: Validate canonical example run: uv run validation/validate.py examples/tpcds_semantic_model.yaml diff --git a/converters/orionbelt/tests/test_ossie_behavior_validation.py b/converters/orionbelt/tests/test_ossie_behavior_validation.py new file mode 100644 index 000000000..77c35d618 --- /dev/null +++ b/converters/orionbelt/tests/test_ossie_behavior_validation.py @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The standalone converter needs only its packaged root schema, offline.""" + +import json +import socket +import urllib.request +from copy import deepcopy + +import jsonschema +import pytest + +from ossie_orionbelt.validation import _OSSIE_SCHEMA_PATH, validate_ossie + + +@pytest.fixture +def isolated_schema(tmp_path, monkeypatch): + def reject_request(*args, **kwargs): + raise AssertionError("Schema validation must not fetch a remote resource") + + monkeypatch.setattr(socket.socket, "connect", reject_request) + monkeypatch.setattr(urllib.request, "urlopen", reject_request) + monkeypatch.setattr(jsonschema.validators, "urlopen", reject_request, raising=False) + path = tmp_path / "ossie-schema.json" + path.write_bytes(_OSSIE_SCHEMA_PATH.read_bytes()) + assert list(tmp_path.iterdir()) == [path] + return path + + +@pytest.fixture +def document(): + return { + "version": "0.2.0.dev0", + "name": "purchasing", + "datasets": [{"name": "orders", "source": "warehouse.public.orders"}], + "behavior": { + "namespace": "purchasing", + "behavior_layer_version": "1.0", + "actions": [{ + "id": "approve_order", + "title": "Approve order", + "kind": "command", + "effects": [{ + "entity": "dataset", + "mode": "write", + "impact_type": "state_transition", + "vendor_details": {"system": "ERP"}, + }], + "vendor_details": {"system": "ERP"}, + }], + "rules": [{ + "id": "approval_required", + "title": "Approval required", + "severity": "error", + "when": {"operation": "approve_order"}, + "constraint": {"approved": True}, + "message": "An approval is required", + "vendor_details": {"system": "ERP"}, + }], + "vendor_details": {"system": "ERP"}, + }, + } + + +@pytest.mark.parametrize("alias", ["actions", "action_types", "both"]) +@pytest.mark.parametrize("empty_lists", [False, True]) +def test_behavior_aliases_and_extensions_validate_offline( + isolated_schema, document, alias, empty_lists +): + behavior = document["behavior"] + if empty_lists: + behavior.update(actions=[], rules=[]) + if alias == "action_types": + behavior[alias] = behavior.pop("actions") + elif alias == "both": + behavior["action_types"] = deepcopy(behavior["actions"]) + + # Match consumers that load one schema without a custom resolver/registry. + jsonschema.Draft202012Validator(json.loads(isolated_schema.read_text())).validate(document) + result = validate_ossie(document, schema_path=isolated_schema) + + assert result.schema_validation_performed + assert result.valid, result.schema_errors + result.semantic_errors + + +@pytest.mark.parametrize( + "path,value", + [ + (("namespace",), ""), + (("namespace",), 1), + (("behavior_layer_version",), ""), + (("behavior_layer_version",), 1), + (("actions",), {}), + (("rules",), "invalid"), + (("actions", 0, "id"), ""), + (("actions", 0, "kind"), "invalid"), + (("actions", 0, "effects", 0, "entity"), "invalid"), + (("actions", 0, "effects", 0, "mode"), "invalid"), + (("actions", 0, "effects", 0, "impact_type"), "invalid"), + (("rules", 0, "severity"), "invalid"), + (("rules", 0, "when"), "invalid"), + (("rules", 0, "constraint"), "invalid"), + (("rules", 0, "message"), ""), + ], +) +def test_invalid_behavior_reports_schema_errors_offline(isolated_schema, document, path, value): + target = document["behavior"] + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + + result = validate_ossie(document, schema_path=isolated_schema) + + assert result.schema_validation_performed + assert not result.valid + assert result.schema_errors + + +@pytest.mark.parametrize( + "path", + [ + ("namespace",), ("behavior_layer_version",), ("actions",), ("rules",), + ("actions", 0, "id"), ("actions", 0, "title"), + ("actions", 0, "effects", 0, "entity"), + ("actions", 0, "effects", 0, "mode"), + ("rules", 0, "id"), ("rules", 0, "title"), ("rules", 0, "severity"), + ("rules", 0, "when"), ("rules", 0, "constraint"), ("rules", 0, "message"), + ], +) +def test_missing_behavior_fields_are_rejected_offline(isolated_schema, document, path): + target = document["behavior"] + for part in path[:-1]: + target = target[part] + del target[path[-1]] + + result = validate_ossie(document, schema_path=isolated_schema) + + assert result.schema_validation_performed + assert not result.valid + assert result.schema_errors + + +def test_behavior_remains_optional_offline(isolated_schema, document): + del document["behavior"] + + result = validate_ossie(document, schema_path=isolated_schema) + + assert result.schema_validation_performed + assert result.valid, result.schema_errors + result.semantic_errors diff --git a/converters/salesforce/src/test/java/org/apache/ossie/validator/SchemaValidatorBehaviorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/validator/SchemaValidatorBehaviorTest.java new file mode 100644 index 000000000..9289b9e0d --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/validator/SchemaValidatorBehaviorTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.validator; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import java.io.InputStream; +import java.util.Map; +import java.util.stream.Stream; +import org.apache.ossie.exception.ValidationException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.*; + +class SchemaValidatorBehaviorTest { + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static JsonSchema offlineSchema; + + @BeforeAll + static void loadOnlyPackagedRootSchema() throws Exception { + try (InputStream input = SchemaValidatorBehaviorTest.class.getResourceAsStream( + SchemaValidator.OSSIE_SCHEMA_PATH)) { + assertNotNull(input); + // Keep the production factory's default draft; the explicit $schema + // selects 2020-12. Any external resource load fails immediately. + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7, + builder -> builder.schemaLoaders(loaders -> loaders.add(iri -> { + throw new AssertionError("Unexpected schema resource load: " + iri); + }))); + offlineSchema = factory.getSchema(MAPPER.readTree(input)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"actions", "action_types", "both"}) + void acceptsAliasesAndVendorExtensionsOffline(String alias) throws Exception { + ObjectNode document = document(); + ObjectNode behavior = (ObjectNode) document.get("behavior"); + if (alias.equals("action_types")) { + behavior.set(alias, behavior.remove("actions")); + } else if (alias.equals("both")) { + behavior.set("action_types", behavior.get("actions").deepCopy()); + } + assertValid(document); + } + + @ParameterizedTest + @ValueSource(strings = {"actions", "action_types", "both"}) + void acceptsEmptyListsOffline(String alias) throws Exception { + ObjectNode document = document(); + ObjectNode behavior = (ObjectNode) document.get("behavior"); + behavior.remove("actions"); + behavior.putArray(alias.equals("both") ? "actions" : alias); + if (alias.equals("both")) { + behavior.putArray("action_types"); + } + behavior.putArray("rules"); + assertValid(document); + } + + @Test + void behaviorRemainsOptionalOffline() throws Exception { + ObjectNode document = document(); + document.remove("behavior"); + assertValid(document); + } + + @ParameterizedTest + @MethodSource("invalidValues") + void rejectsInvalidBehaviorOffline(String pointer, Object value) throws Exception { + ObjectNode document = document(); + ObjectNode target = (ObjectNode) document.at(pointer.substring(0, pointer.lastIndexOf('/'))); + target.set(pointer.substring(pointer.lastIndexOf('/') + 1), MAPPER.valueToTree(value)); + assertInvalid(document); + } + + static Stream invalidValues() { + return Stream.of( + Arguments.of("/behavior/namespace", ""), + Arguments.of("/behavior/namespace", 1), + Arguments.of("/behavior/behavior_layer_version", ""), + Arguments.of("/behavior/behavior_layer_version", 1), + Arguments.of("/behavior/actions", Map.of()), + Arguments.of("/behavior/rules", "invalid"), + Arguments.of("/behavior/actions/0/id", ""), + Arguments.of("/behavior/actions/0/kind", "invalid"), + Arguments.of("/behavior/actions/0/effects/0/entity", "invalid"), + Arguments.of("/behavior/actions/0/effects/0/mode", "invalid"), + Arguments.of("/behavior/actions/0/effects/0/impact_type", "invalid"), + Arguments.of("/behavior/rules/0/severity", "invalid"), + Arguments.of("/behavior/rules/0/when", "invalid"), + Arguments.of("/behavior/rules/0/constraint", "invalid"), + Arguments.of("/behavior/rules/0/message", "")); + } + + @ParameterizedTest + @ValueSource(strings = { + "/behavior/namespace", "/behavior/behavior_layer_version", "/behavior/actions", + "/behavior/rules", "/behavior/actions/0/id", "/behavior/actions/0/title", + "/behavior/actions/0/effects/0/entity", "/behavior/actions/0/effects/0/mode", + "/behavior/rules/0/id", "/behavior/rules/0/title", "/behavior/rules/0/severity", + "/behavior/rules/0/when", "/behavior/rules/0/constraint", "/behavior/rules/0/message" + }) + void rejectsMissingRequiredFieldsOffline(String pointer) throws Exception { + ObjectNode document = document(); + ObjectNode target = (ObjectNode) document.at(pointer.substring(0, pointer.lastIndexOf('/'))); + target.remove(pointer.substring(pointer.lastIndexOf('/') + 1)); + assertInvalid(document); + } + + private void assertValid(ObjectNode document) { + assertTrue(offlineSchema.validate(document).isEmpty()); + new SchemaValidator(MAPPER, SchemaValidator.OSSIE_SCHEMA_PATH).validate(asMap(document)); + } + + private void assertInvalid(ObjectNode document) { + assertFalse(offlineSchema.validate(document).isEmpty()); + ValidationException error = assertThrows(ValidationException.class, + () -> new SchemaValidator(MAPPER, SchemaValidator.OSSIE_SCHEMA_PATH).validate(asMap(document))); + assertTrue(error.getMessage().contains("Schema validation failed")); + } + + private Map asMap(JsonNode document) { + return MAPPER.convertValue(document, new TypeReference<>() {}); + } + + private ObjectNode document() throws Exception { + return (ObjectNode) MAPPER.readTree(""" + { + "version": "0.2.0.dev0", + "name": "purchasing", + "datasets": [{"name": "orders", "source": "warehouse.public.orders"}], + "behavior": { + "namespace": "purchasing", + "behavior_layer_version": "1.0", + "actions": [{ + "id": "approve_order", "title": "Approve order", "kind": "command", + "effects": [{"entity": "dataset", "mode": "write", + "impact_type": "state_transition", "vendor_details": {"system": "ERP"}}], + "vendor_details": {"system": "ERP"} + }], + "rules": [{ + "id": "approval_required", "title": "Approval required", "severity": "error", + "when": {"operation": "approve_order"}, "constraint": {"approved": true}, + "message": "An approval is required", "vendor_details": {"system": "ERP"} + }], + "vendor_details": {"system": "ERP"} + } + } + """); + } +} diff --git a/core-spec/behavior-layer.md b/core-spec/behavior-layer.md index e715b8083..8bf68339b 100644 --- a/core-spec/behavior-layer.md +++ b/core-spec/behavior-layer.md @@ -28,7 +28,15 @@ It does not execute actions or enforce business rules. [behavior-layer.schema.json](behavior-layer.schema.json) is the sole canonical schema for Behavior, Action, Effect, and Rule. [ossie-schema.json](ossie-schema.json) -references it with `$ref`; it does not maintain inline copies of those definitions. +embeds that resource so consumers can validate offline using just the core schema. +The root property references the reusable `SemanticModel` definition. The embedded +resource is generated, never edited separately; CI checks it against the canonical +source. After changing the source, run: + +```sh +python3 scripts/generate-behavior-schema.py --apply +``` + The YAML and Markdown specifications describe this same contract. New model documents SHOULD use the optional root-level `behavior` property. diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index b8a6168d7..a0d633109 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -364,7 +364,303 @@ "description": "Quantifiable measures spanning datasets" }, "behavior": { - "$ref": "behavior-layer.schema.json" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/core-spec/behavior-layer.schema.json", + "title": "Apache Ossie Behavior Layer", + "description": "Canonical schema for a semantic model's optional behavior (actions/rules/effects). Supports legacy alias action_types.", + "type": "object", + "additionalProperties": true, + "required": [ + "namespace", + "behavior_layer_version", + "rules" + ], + "anyOf": [ + { + "required": [ + "actions" + ] + }, + { + "required": [ + "action_types" + ] + } + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1 + }, + "behavior_layer_version": { + "type": "string", + "minLength": 1 + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "actions": { + "type": "array", + "description": "Preferred: list of actions supported by this semantic model. (Alias of legacy action_types)", + "items": { + "$ref": "#/$defs/actionType" + } + }, + "action_types": { + "type": "array", + "description": "Legacy alias of actions. New models SHOULD use actions.", + "items": { + "$ref": "#/$defs/actionType" + } + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/rule" + } + } + }, + "$defs": { + "actionType": { + "type": "object", + "additionalProperties": true, + "required": [ + "id", + "title" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "command", + "query" + ] + }, + "operation": { + "type": "string", + "description": "Free-form operation name." + }, + "aggregate": { + "type": "string" + }, + "entity_name": { + "type": "string" + }, + "idempotency": { + "type": "string", + "enum": [ + "idempotent", + "non_idempotent", + "unknown" + ] + }, + "applies_to": { + "type": "object", + "additionalProperties": true + }, + "io_schema": { + "type": "object", + "additionalProperties": true + }, + "effects": { + "type": "array", + "description": "Optional: machine-readable impact annotations (what this action reads/writes/derives).", + "items": { + "$ref": "#/$defs/effect" + } + }, + "examples": { + "type": "array", + "items": { + "type": "string" + } + }, + "tool_hint": { + "type": "object", + "additionalProperties": true + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + } + }, + "deprecated": { + "type": "boolean" + }, + "version": { + "type": "string" + } + } + }, + "effect": { + "type": "object", + "additionalProperties": true, + "required": [ + "entity", + "mode" + ], + "properties": { + "entity": { + "type": "string", + "enum": [ + "dataset", + "field", + "metric", + "relationship" + ] + }, + "mode": { + "type": "string", + "enum": [ + "read", + "write", + "derive" + ] + }, + "impact_type": { + "type": "string", + "enum": [ + "state_transition", + "master_data_mutation", + "transactional_write", + "derived_metric_change", + "other" + ] + }, + "selectors": { + "type": "object", + "additionalProperties": true, + "properties": { + "dataset": { + "type": "string" + }, + "field_names": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "transition": { + "type": "object", + "additionalProperties": true, + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + "set_value": { + "type": "string" + }, + "confidence": { + "type": "string", + "enum": [ + "guaranteed", + "likely", + "unknown" + ] + }, + "notes": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "rule": { + "type": "object", + "additionalProperties": true, + "required": [ + "id", + "title", + "severity", + "when", + "constraint", + "message" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "severity": { + "type": "string", + "enum": [ + "error", + "warn", + "info" + ] + }, + "when": { + "type": "object", + "additionalProperties": true + }, + "if": { + "type": "object", + "additionalProperties": true + }, + "constraint": { + "type": "object", + "additionalProperties": true + }, + "message": { + "type": "string", + "minLength": 1 + }, + "remediation": { + "type": "string" + }, + "references": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } }, "custom_extensions": { "type": "array", diff --git a/core-spec/spec.md b/core-spec/spec.md index 908bd90c2..8cedde0ba 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -171,8 +171,10 @@ The optional root-level `behavior` property describes actions, rules, and effect for planning and attribution. It does not execute actions or enforce policies. [behavior-layer.schema.json](behavior-layer.schema.json) is the canonical -validation source. `ossie-schema.json` references it with `$ref`, including from -the reusable `SemanticModel` definition. The YAML reference describes the same +validation source. `ossie-schema.json` embeds a generated copy in its reusable +`SemanticModel` definition for single-file offline validation, with a CI check +preventing drift from the canonical source. The root property references that +same definition. The YAML reference describes the same contract. See [Behavior Layer](behavior-layer.md) for all fields and legacy embedded-payload validation. diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 80b78ced3..3c3023c73 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -96,8 +96,7 @@ relationships: [] # See Metrics section below for detailed structure metrics: [] -# Optional: Behavior metadata; canonical contract: behavior-layer.schema.json -behavior: {} +# Optional: Behavior metadata; see the Behavior Schema below for a valid block. # Optional: Vendor-specific attributes for extensibility # Allows vendors to add custom metadata without breaking core compatibility diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 056829ca8..7f1f81f85 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -18,10 +18,20 @@ import json import math from enum import Enum -from typing import Any, Literal, Optional, Union +from typing import Annotated, Any, Literal, Optional, Union import yaml -from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, StrictBool, model_serializer +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + StrictBool, + TypeAdapter, + field_validator, + model_serializer, +) +from typing_extensions import NotRequired, TypedDict _AI_CONTEXT_FIELDS = frozenset({"instructions", "synonyms", "examples"}) @@ -56,9 +66,9 @@ def _invalid_below( accepts the non-standard `NaN`, `Infinity` and `-Infinity`, none of which JSON can express -- a `NaN` reaching `to_ossie_json` comes back out as `null`. - `free_form` marks the parts of `ai_context` where a null is legal: its object form - sets `additionalProperties: true`, so an extra key may hold any value. A non-finite - number is rejected there too, since it still cannot be serialized. + `free_form` marks the parts of `ai_context` where a null is legal, and behavior, + whose declared fields are checked separately. Both allow arbitrary extension + values. A non-finite number is rejected there too, since it cannot be serialized. `active` holds the containers on the current branch, so a self-reference raises while an anchor reused on a sibling branch still walks. `walked` holds containers @@ -87,7 +97,9 @@ def _invalid_below( found.append((where, _NULL)) else: found += _invalid_below( - value, where, inner, walked, legal_null, key == "ai_context" + value, where, inner, walked, + legal_null or (not path and key == "behavior"), + key == "ai_context", ) else: for index, item in enumerate(node): @@ -283,6 +295,84 @@ class OssieMetric(BaseModel): custom_extensions: Optional[list[OssieCustomExtension]] = None +class _BehaviorObject(TypedDict): + __pydantic_config__ = ConfigDict(extra="allow") + + +_BehaviorString = Annotated[str, Field(min_length=1)] + + +class _BehaviorSelectors(_BehaviorObject): + dataset: NotRequired[str] + field_names: NotRequired[list[str]] + + +class _BehaviorTransition(_BehaviorObject): + from_state: NotRequired[Annotated[str, Field(alias="from")]] + to: NotRequired[str] + + +class _BehaviorEffect(_BehaviorObject): + entity: Literal["dataset", "field", "metric", "relationship"] + mode: Literal["read", "write", "derive"] + impact_type: NotRequired[Literal[ + "state_transition", "master_data_mutation", "transactional_write", + "derived_metric_change", "other", + ]] + selectors: NotRequired[_BehaviorSelectors] + transition: NotRequired[_BehaviorTransition] + set_value: NotRequired[str] + confidence: NotRequired[Literal["guaranteed", "likely", "unknown"]] + notes: NotRequired[str] + tags: NotRequired[list[str]] + + +class _BehaviorAction(_BehaviorObject): + id: _BehaviorString + title: _BehaviorString + description: NotRequired[str] + kind: NotRequired[Literal["command", "query"]] + operation: NotRequired[str] + aggregate: NotRequired[str] + entity_name: NotRequired[str] + idempotency: NotRequired[Literal["idempotent", "non_idempotent", "unknown"]] + applies_to: NotRequired[dict[str, Any]] + io_schema: NotRequired[dict[str, Any]] + effects: NotRequired[list[_BehaviorEffect]] + examples: NotRequired[list[str]] + tool_hint: NotRequired[dict[str, Any]] + tags: NotRequired[list[str]] + synonyms: NotRequired[list[str]] + deprecated: NotRequired[bool] + version: NotRequired[str] + + +class _BehaviorRule(_BehaviorObject): + id: _BehaviorString + title: _BehaviorString + description: NotRequired[str] + severity: Literal["error", "warn", "info"] + when: dict[str, Any] + condition: NotRequired[Annotated[dict[str, Any], Field(alias="if")]] + constraint: dict[str, Any] + message: _BehaviorString + remediation: NotRequired[str] + references: NotRequired[list[dict[str, Any]]] + tags: NotRequired[list[str]] + + +class _Behavior(_BehaviorObject): + namespace: _BehaviorString + behavior_layer_version: _BehaviorString + metadata: NotRequired[dict[str, Any]] + actions: NotRequired[list[_BehaviorAction]] + action_types: NotRequired[list[_BehaviorAction]] + rules: list[_BehaviorRule] + + +_BEHAVIOR_ADAPTER = TypeAdapter(_Behavior) + + class OssieSemanticModel(BaseModel): """Semantic model: the datasets, relationships, and metrics for one business domain.""" @@ -294,10 +384,20 @@ class OssieSemanticModel(BaseModel): datasets: list[OssieDataset] = Field(..., min_length=1) relationships: Optional[list[OssieRelationship]] = None metrics: Optional[list[OssieMetric]] = None - # Behavior field constraints are defined by the canonical JSON Schema. + # Keep the dictionary API and extension values while enforcing the canonical + # behavior-layer.schema.json constraints; regression tests check schema parity. behavior: Optional[dict[str, Any]] = None custom_extensions: Optional[list[OssieCustomExtension]] = None + @field_validator("behavior") + @classmethod + def _validate_behavior(cls, value: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: + if value is not None: + _BEHAVIOR_ADAPTER.validate_python(value, strict=True) + if "actions" not in value and "action_types" not in value: + raise ValueError("behavior requires actions or action_types") + return value + class OssieDocument(OssieSemanticModel): """A single semantic model with document metadata at the root.""" diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 3d5bf00d6..29a5eb7b4 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -17,6 +17,7 @@ import json from collections.abc import Callable +from copy import deepcopy from pathlib import Path import pytest @@ -277,6 +278,173 @@ def test_behavior_survives_document_and_embedded_model_serialization() -> None: assert embedded.model_dump(by_alias=True, exclude_none=True, mode="json") == model_data +@pytest.fixture(scope="module") +def behavior_schema() -> dict: + path = Path(__file__).parents[2] / "core-spec/behavior-layer.schema.json" + return json.loads(path.read_text()) + + +def _behavior_node(schema: dict, definitions: dict) -> dict: + if "$ref" in schema: + return definitions[schema["$ref"].rsplit("/", 1)[-1]] + return schema + + +def _behavior_example(schema: dict, definitions: dict): + """Populate every declared field, including free-form extension nulls.""" + schema = _behavior_node(schema, definitions) + if "enum" in schema: + return schema["enum"][0] + if schema["type"] == "object": + return { + **{ + name: _behavior_example(child, definitions) + for name, child in schema.get("properties", {}).items() + }, + "vendor_extension": {"value": None}, + } + if schema["type"] == "array": + return [_behavior_example(schema["items"], definitions)] + if schema["type"] == "boolean": + return False + assert schema["type"] == "string" + return "value" + + +def _behavior_nodes(schema: dict, definitions: dict, value, path: tuple = ()): + schema = _behavior_node(schema, definitions) + yield path, schema + if schema["type"] == "object": + for name, child in schema.get("properties", {}).items(): + if name in value: + yield from _behavior_nodes(child, definitions, value[name], (*path, name)) + elif schema["type"] == "array" and value: + yield from _behavior_nodes(schema["items"], definitions, value[0], (*path, 0)) + + +_BEHAVIOR_MISSING = object() + + +def _mutate_behavior(value: dict, path: tuple, replacement): + mutated = deepcopy(value) + if not path: + return replacement + target = mutated + for key in path[:-1]: + target = target[key] + if replacement is _BEHAVIOR_MISSING: + del target[path[-1]] + else: + target[path[-1]] = replacement + return mutated + + +@pytest.mark.parametrize("model_class", [OssieDocument, OssieSemanticModel]) +@pytest.mark.parametrize("alias", ["actions", "action_types", "both"]) +def test_behavior_validation_matches_canonical_schema( + document_data: dict, behavior_schema: dict, model_class: type[BaseModel], alias: str +) -> None: + """Check every canonical property, enum value, required field, and item type. + + The schema supplies the cases and expected acceptance, so SDK-only constraints + cannot silently become a different behavior contract. + """ + jsonschema = pytest.importorskip("jsonschema") + validator = jsonschema.Draft202012Validator(behavior_schema) + definitions = behavior_schema["$defs"] + behavior = _behavior_example(behavior_schema, definitions) + if alias != "both": + del behavior["action_types" if alias == "actions" else "actions"] + data = dict(document_data) + if model_class is OssieSemanticModel: + del data["version"] + + for path, schema in _behavior_nodes(behavior_schema, definitions, behavior): + replacements = [None, False, 1, "", [], {}] + if path: + replacements.append(_BEHAVIOR_MISSING) + if "enum" in schema: + replacements.extend([*schema["enum"], "NOT_A_VALID_VALUE"]) + if schema["type"] == "array": + replacements.extend([[None], [1]]) + if schema["type"] == "boolean": + replacements.extend([True, "false"]) + for replacement in replacements: + value = _mutate_behavior(behavior, path, replacement) + # Python construction permits top-level None to mean omission, as for + # the other optional model fields. Serialized loaders reject it. + if value is None: + continue + expected = validator.is_valid(value) + case = f"{model_class.__name__}, {alias}, {path}, {replacement!r}" + try: + model = model_class.model_validate({**data, "behavior": value}) + except ValidationError as error: + assert not expected, case + assert any(item["loc"][0] == "behavior" for item in error.errors()), case + else: + assert expected, case + assert model.behavior == value, case + + +@pytest.mark.parametrize("model_class", [OssieDocument, OssieSemanticModel]) +@pytest.mark.parametrize("alias", ["actions", "action_types", "both"]) +def test_behavior_extensions_aliases_and_round_trips( + document_data: dict, behavior_schema: dict, model_class: type[BaseModel], alias: str +) -> None: + behavior = _behavior_example(behavior_schema, behavior_schema["$defs"]) + if alias != "both": + del behavior["action_types" if alias == "actions" else "actions"] + # These are extension keys, even though the private validation types use them + # as Python names for the reserved words `from` and `if`. + for name in ("actions", "action_types"): + if name in behavior: + behavior[name][0]["effects"][0]["transition"]["from_state"] = None + behavior["rules"][0]["condition"] = None + data = {**document_data, "behavior": behavior} + if model_class is OssieSemanticModel: + del data["version"] + model = model_class.model_validate(data) + + assert isinstance(model.behavior, dict) + assert model.behavior == behavior + assert model.model_dump(mode="json", by_alias=True, exclude_none=True) == data + assert model_class.model_validate_json(model.model_dump_json(exclude_none=True)) == model + document = OssieDocument.model_validate({"version": "0.2.0.dev0", **data}) + assert OssieDocument.from_ossie_json(document.to_ossie_json()) == document + assert OssieDocument.from_ossie_yaml(document.to_ossie_yaml()) == document + assert json.loads(document.to_ossie_json())["behavior"] == behavior + assert yaml.safe_load(document.to_ossie_yaml())["behavior"] == behavior + + +@pytest.mark.parametrize("model_class", [OssieDocument, OssieSemanticModel]) +def test_behavior_rejects_invalid_blocks_without_schema_dependency( + document_data: dict, model_class: type[BaseModel] +) -> None: + data = dict(document_data) + if model_class is OssieSemanticModel: + del data["version"] + invalid_blocks = [ + {}, + {"namespace": "example", "behavior_layer_version": "0.1", "rules": []}, + { + "namespace": "example", "behavior_layer_version": "0.1", "rules": [], + "actions": [{"id": "action", "title": "Action", "effects": [ + {"entity": "dataset", "mode": "write", "impact_type": "invalid"} + ]}], + }, + ] + for behavior in invalid_blocks: + with pytest.raises(ValidationError): + model_class.model_validate({**data, "behavior": behavior}) + + for optional in ({}, {"behavior": None}): + assert model_class.model_validate({**data, **optional}).behavior is None + for load in (OssieDocument.from_ossie_json, OssieDocument.from_ossie_yaml): + with pytest.raises(ValueError, match="set to null"): + load(json.dumps({**document_data, "behavior": None})) + + def test_document_defaults_version_when_omitted(document_data: dict) -> None: del document_data["version"] diff --git a/python/tests/test_serialization.py b/python/tests/test_serialization.py index 62d8e2197..f9a9cda1c 100644 --- a/python/tests/test_serialization.py +++ b/python/tests/test_serialization.py @@ -293,6 +293,10 @@ def _fully_populated_document() -> dict: "description": "d", "ai_context": dict(ai_context), "custom_extensions": list(extensions), + "behavior": { + "namespace": "example", "behavior_layer_version": "0.1", + "actions": [], "rules": [], + }, "datasets": [ { "name": "orders", @@ -394,7 +398,10 @@ def test_every_property_in_a_document_rejects_null() -> None: for path in paths: document = _fully_populated_document() _set_null(document, path) - with pytest.raises(ValueError, match="set to null"): + # Behavior's typed validator checks declared fields; its free-form + # extension values may contain null and bypass the generic null walk. + diagnostic = None if path.startswith("behavior.") else "set to null" + with pytest.raises(ValueError, match=diagnostic): OssieDocument.from_ossie_yaml(yaml.safe_dump(document)) diff --git a/scripts/generate-behavior-schema.py b/scripts/generate-behavior-schema.py new file mode 100644 index 000000000..edda0377d --- /dev/null +++ b/scripts/generate-behavior-schema.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Embed the canonical Behavior resource so the core schema works offline. + +Use --apply after editing behavior-layer.schema.json, or --check in CI. +""" + +import argparse +import json +from pathlib import Path + +_CORE_SPEC = Path(__file__).resolve().parents[1] / "core-spec" + + +def render(core_text: str, behavior: dict) -> str: + # Replace only the generated member, preserving the rest of the core file. + marker = ' "behavior": ' + if core_text.count(marker) != 1: + raise ValueError("Expected one SemanticModel behavior member") + start = core_text.index(marker) + len(marker) + current, length = json.JSONDecoder().raw_decode(core_text[start:]) + core = json.loads(core_text) + if current != core["$defs"]["SemanticModel"]["properties"]["behavior"]: + raise ValueError("Behavior member is outside SemanticModel") + # Keep $id: local fragment references belong to the embedded resource. + embedded = json.dumps(behavior, indent=2, ensure_ascii=False).replace("\n", "\n ") + return core_text[:start] + embedded + core_text[start + length:] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--apply", action="store_true") + mode.add_argument("--check", action="store_true") + args = parser.parse_args() + core_path = _CORE_SPEC / "ossie-schema.json" + current = core_path.read_text(encoding="utf-8") + behavior = json.loads((_CORE_SPEC / "behavior-layer.schema.json").read_text(encoding="utf-8")) + expected = render(current, behavior) + if args.apply: + core_path.write_text(expected, encoding="utf-8") + return 0 + if current != expected: + print("Behavior schema is out of sync; run python3 scripts/generate-behavior-schema.py --apply") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index d0c97eef5..0f09ad0be 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -26,7 +26,7 @@ # validate.py exits at import time when its dependencies are missing, which # would abort the whole pytest session during collection — skip instead. yaml = pytest.importorskip("yaml") -pytest.importorskip("jsonschema") +jsonschema = pytest.importorskip("jsonschema") _VALIDATE_PATH = Path(__file__).parents[1] / "validate.py" _SPEC = spec_from_file_location("ossie_validate", _VALIDATE_PATH) @@ -110,6 +110,77 @@ def behavior_schema(): return json.loads(path.read_text()) +def test_behavior_schema_generation_and_drift_check( + core_schema, behavior_schema, tmp_path, monkeypatch +): + path = Path(__file__).parents[2] / "scripts/generate-behavior-schema.py" + spec = spec_from_file_location("generate_behavior_schema", path) + generator = module_from_spec(spec) + spec.loader.exec_module(generator) + monkeypatch.setattr(generator, "_CORE_SPEC", tmp_path) + core_path = tmp_path / "ossie-schema.json" + source_path = tmp_path / "behavior-layer.schema.json" + original = (path.parents[1] / "core-spec/ossie-schema.json").read_text() + core_path.write_text(original) + source_path.write_text(json.dumps(behavior_schema)) + + assert core_schema["$defs"]["SemanticModel"]["properties"]["behavior"] == behavior_schema + monkeypatch.setattr("sys.argv", [str(path), "--check"]) + assert generator.main() == 0 + + behavior_schema["properties"]["namespace"]["minLength"] = 2 + source_path.write_text(json.dumps(behavior_schema)) + assert generator.main() == 1 + assert core_path.read_text() == original + + monkeypatch.setattr("sys.argv", [str(path), "--apply"]) + assert generator.main() == 0 + core_schema["$defs"]["SemanticModel"]["properties"]["behavior"] = behavior_schema + assert json.loads(core_path.read_text()) == core_schema + monkeypatch.setattr("sys.argv", [str(path), "--check"]) + assert generator.main() == 0 + + +@pytest.mark.parametrize("alias", ["actions", "action_types", "both"]) +@pytest.mark.parametrize("embedded", [False, True]) +def test_single_file_behavior_schema_validates_offline( + offline, tmp_path, core_schema, behavior_document, alias, embedded +): + # No sibling file or custom registry is available to this consumer. + schema_path = tmp_path / "ossie-schema.json" + schema_path.write_text(json.dumps(core_schema)) + schema = json.loads(schema_path.read_text()) + if embedded: + schema = { + "$schema": schema["$schema"], + "$id": schema["$id"], + "$ref": "#/$defs/SemanticModel", + "$defs": schema["$defs"], + } + del behavior_document["version"] + behavior = behavior_document["behavior"] + if alias == "action_types": + behavior["action_types"] = behavior.pop("actions") + elif alias == "both": + behavior["action_types"] = deepcopy(behavior["actions"]) + validator = jsonschema.Draft202012Validator(schema) + validator.validate(behavior_document) + actions = behavior.get("actions", behavior.get("action_types")) + actions[0]["effects"][0]["impact_type"] = "invalid" + assert not validator.is_valid(behavior_document) + behavior_document["behavior"] = {} + assert not validator.is_valid(behavior_document) + + +def test_behavior_template_has_a_valid_optional_block(behavior_schema): + path = Path(__file__).parents[2] / "core-spec/spec.yaml" + documents = list(yaml.safe_load_all(path.read_text())) + root = next(document for document in documents if "name" in document and "datasets" in document) + assert "behavior" not in root + block = next(document["behavior"] for document in documents if "behavior" in document) + jsonschema.Draft202012Validator(behavior_schema).validate(block) + + @pytest.mark.parametrize("alias", ["actions", "action_types", "both"]) def test_behavior_aliases_resolve_offline( offline, core_schema, behavior_schema, behavior_document, alias