Enable PHP FFE evaluation system tests

docs/understand/weblogs/end-to-end_weblog.md

@@ -1026,6 +1026,20 @@ The endpoint must accept a query string parameter `code`, which should be an int
 This endpoint is used for client-stats tests to provide a separate "resource" via the endpoint path `stats-unique` to disambiguate those tests from other
 stats generating tests.
 
+### POST /ffe
+
+This endpoint is used by the Feature Flags & Experimentation scenario. It must
+accept a JSON body with these fields:
+
+- `flag`: the feature flag key to evaluate.
+- `variationType`: the expected variation type.
+- `defaultValue`: the value to return when evaluation cannot resolve the flag.
+- `targetingKey`: the evaluation subject key.
+- `attributes`: flat scalar targeting attributes.
+
+The response must be JSON and include at least `value` and `reason`. Error
+responses should also include `errorCode` and `errorMessage`.
+
 ### GET /healthcheck
 
 Returns a JSON dict, with those values :

manifests/dotnet.yml

@@ -810,7 +810,6 @@ manifest:
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2: v2.44.0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0'  # Modified by easy win activation script
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature  # Created by easy win activation script
-  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_of7_empty_targeting_key: missing_feature  # Created by easy win activation script
   tests/parametric/test_ffe/test_span_enrichment.py: missing_feature
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_valid: missing_feature (Need to remove b3=b3multi alias)
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_inject_valid: missing_feature (Need to remove b3=b3multi alias)

manifests/java.yml

@@ -3644,7 +3644,6 @@ manifest:
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV1_ServiceTargets::test_not_match_service_target: irrelevant (APMAPI-1003)
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2: v1.31.0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0
-  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_of7_empty_targeting_key: bug (FFL-1729)
   tests/parametric/test_ffe/test_span_enrichment.py: missing_feature
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_invalid:  # Modified by easy win activation script
     - declaration: missing_feature (Need to remove b3=b3multi alias)

manifests/nodejs.yml

@@ -2014,7 +2014,6 @@ manifest:
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV1_ServiceTargets::test_not_match_service_target: bug (APMAPI-865)
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2: *ref_4_23_0
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0
-  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_of7_empty_targeting_key: bug (FFL-1730)
   tests/parametric/test_ffe/test_span_enrichment.py: "missing_feature (dd-trace-js#8343)"
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_invalid: missing_feature (Need to remove b3=b3multi alias)
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_valid: missing_feature (Need to remove b3=b3multi alias)

manifests/php.yml

@@ -605,7 +605,7 @@ manifest:
       component_version: <1.12.0
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3
   tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme)
-  tests/ffe/test_dynamic_evaluation.py: missing_feature
+  tests/ffe/test_dynamic_evaluation.py: v1.21.0-dev
   tests/ffe/test_exposures.py: missing_feature
   tests/ffe/test_flag_eval_metrics.py: missing_feature
   tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234)
@@ -731,7 +731,7 @@ manifest:
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV1_ServiceTargets::test_not_match_service_target: missing_feature
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2: '>=1.16.0'
   tests/parametric/test_dynamic_configuration.py::TestDynamicConfigV2::test_tracing_client_tracing_tags: missing_feature
-  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature
+  tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev
   tests/parametric/test_ffe/test_span_enrichment.py: missing_feature
   tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_migrated_extract_invalid:
     - declaration: missing_feature (Need to remove b3=b3multi alias)
@@ -870,7 +870,7 @@ manifest:
   tests/parametric/test_parametric_endpoints.py::Test_Parametric_DDSpan_Start: v1.13.0+4663b2fa7c20c6920f347d059b57dc2a419cb7f7
   tests/parametric/test_parametric_endpoints.py::Test_Parametric_DDTrace_Baggage: missing_feature (baggage is not supported)
   tests/parametric/test_parametric_endpoints.py::Test_Parametric_DDTrace_Current_Span: bug (APMAPI-778)  # current span endpoint should return span and trace id of zero if no span is "active"
-  tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start: missing_feature
+  tests/parametric/test_parametric_endpoints.py::Test_Parametric_FFE_Start: v1.21.0-dev
   tests/parametric/test_parametric_endpoints.py::Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported)
   tests/parametric/test_parametric_endpoints.py::Test_Parametric_Otel_Current_Span: bug (APMAPI-778)  # otel current span endpoint should return a span and trace id of zero if no span is "active"
   tests/parametric/test_parametric_endpoints.py::Test_Parametric_Write_Log: missing_feature

tests/parametric/test_ffe/test_dynamic_evaluation.py

@@ -2,11 +2,11 @@
 
 import json
 import pytest
+import time
 from pathlib import Path
 from typing import Any
 
 from utils import (
-    context,
     features,
     scenarios,
 )
@@ -16,6 +16,8 @@
 
 RC_PRODUCT = "FFE_FLAGS"
 RC_PATH = f"datadog/2/{RC_PRODUCT}"
+FFE_READY_RETRY_ATTEMPTS = 10
+FFE_READY_RETRY_INTERVAL_SECONDS = 0.2
 
 parametrize = pytest.mark.parametrize
 
@@ -81,6 +83,44 @@ def _set_and_wait_ffe_rc(
     return test_agent.wait_for_rc_apply_state(RC_PRODUCT, state=RemoteConfigApplyState.ACKNOWLEDGED, clear=True)
 
 
+def _is_ffe_waiting_for_rc(result: dict[str, Any]) -> bool:
+    provider_state = result.get("providerState")
+    return result.get("errorCode") == "PROVIDER_NOT_READY" or (
+        isinstance(provider_state, dict) and provider_state.get("hasConfig") is False
+    )
+
+
+def _ffe_evaluate_with_rc_retry(
+    test_library: APMLibrary,
+    *,
+    flag: str,
+    variation_type: str,
+    default_value: bool | str | float | dict[str, Any],
+    targeting_key: str,
+    attributes: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    result = test_library.ffe_evaluate(
+        flag=flag,
+        variation_type=variation_type,
+        default_value=default_value,
+        targeting_key=targeting_key,
+        attributes=attributes,
+    )
+    for _ in range(FFE_READY_RETRY_ATTEMPTS - 1):
+        if not _is_ffe_waiting_for_rc(result):
+            return result
+        time.sleep(FFE_READY_RETRY_INTERVAL_SECONDS)
+        result = test_library.ffe_evaluate(
+            flag=flag,
+            variation_type=variation_type,
+            default_value=default_value,
+            targeting_key=targeting_key,
+            attributes=attributes,
+        )
+
+    return result
+
+
 @scenarios.parametric
 @features.feature_flags_dynamic_evaluation
 class Test_Feature_Flag_Dynamic_Evaluation:
@@ -111,13 +151,6 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI
         4. Handles user targeting, attribute matching, and rollout percentages
 
         """
-        # Skip OF.7 (empty targeting key) test for libraries with known bugs
-        # Java: FFL-1729 - OpenFeature Java SDK rejects empty targeting keys
-        # Node.js: FFL-1730 - OpenFeature JS SDK rejects empty targeting keys
-        if test_case_file == "test-case-of-7-empty-targeting-key.json":
-            if context.library.name in ("java", "nodejs"):
-                pytest.skip("OF.7 empty targeting key bug: FFL-1729 (java), FFL-1730 (nodejs)")
-
         # Load the test case file
         test_case_path = Path(__file__).parent / test_case_file
 
@@ -131,7 +164,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI
         _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA)
 
         # Initialize FFE provider
-        success = test_library.ffe_start()
+        success = test_library.ffe_start(UFC_FIXTURE_DATA)
         assert success, "Failed to start FFE provider"
 
         # Run each test case
@@ -143,13 +176,18 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI
             attributes = test_case.get("attributes", {})
             expected_result = test_case["result"]["value"]
 
-            result = test_library.ffe_evaluate(
+            result = _ffe_evaluate_with_rc_retry(
+                test_library,
                 flag=flag,
                 variation_type=variation_type,
                 default_value=default_value,
                 targeting_key=targeting_key,
                 attributes=attributes,
             )
+            assert not _is_ffe_waiting_for_rc(result), (
+                f"Test case {i} in {test_case_file} failed: FFE provider did not load RC data after "
+                f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}"
+            )
             actual_value = result.get("value")
 
             # Assert the evaluation result matches expected value
@@ -158,33 +196,3 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI
                 f"flag='{flag}', targetingKey='{targeting_key}', "
                 f"expected={expected_result}, actual={actual_value}"
             )
-
-    @parametrize("library_env", [{**DEFAULT_ENVVARS}])
-    def test_ffe_of7_empty_targeting_key(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None:
-        """OF.7: Empty string is a valid targeting key.
-
-        This test validates that flag evaluation succeeds when the targeting key
-        is an empty string. The flag should still match allocations and return
-        the expected value, not fail with TARGETING_KEY_MISSING.
-
-        Temporary dedicated test until FFL-1729 (Java) and FFL-1730 (Node.js) are resolved.
-        """
-        # Set up UFC Remote Config and wait for it to be applied
-        _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA)
-
-        # Initialize FFE provider
-        success = test_library.ffe_start()
-        assert success, "Failed to start FFE provider"
-
-        # Evaluate flag with empty targeting key
-        result = test_library.ffe_evaluate(
-            flag="empty-targeting-key-flag",
-            variation_type="STRING",
-            default_value="default",
-            targeting_key="",
-            attributes={},
-        )
-
-        assert result.get("value") == "on-value", (
-            f"OF.7 failed: empty targeting key should return 'on-value', got '{result.get('value')}'"
-        )