Initial integration tests

tests/integration/complex_monitoring/BUILD

@@ -0,0 +1,89 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files")
+load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
+load("//:defs.bzl", "launch_manager_config")
+load("//tests/utils/bazel:integration.bzl", "integration_test")
+
+launch_manager_config(
+    name = "lm_complex_monitoring_config",
+    config = "//tests/integration/complex_monitoring:complex_monitoring.json",
+    flatbuffer_out_dir = "etc",
+)
+
+cc_binary(
+    name = "control_client_mock",
+    srcs = ["control_client_mock.cpp"],
+    deps = [
+        "//src/control_client_lib",
+        "//src/launch_manager_daemon/lifecycle_client_lib:lifecycle_client",
+        "//tests/utils/test_helper",
+        "@googletest//:gtest_main",
+    ],
+)
+
+cc_binary(
+    name = "component_complex_monitoring",
+    srcs = ["component_complex_monitoring.cpp"],
+    linkopts = select({
+        "@platforms//os:qnx": [
+            "-lsocket",
+        ],
+        "@platforms//os:linux": [
+            "-lpthread",
+            "-lrt",
+        ],
+    }),
+    deps = [
+        "//src/health_monitoring_lib:health_monitoring_lib_cc",
+        "//src/launch_manager_daemon/health_monitor_lib:hm_shared_lib",
+        "//src/launch_manager_daemon/lifecycle_client_lib:lifecycle_client",
+        "//tests/utils/test_helper",
+        "@googletest//:gtest_main",
+        "@score_baselibs_rust//src/log/stdout_logger_cpp_init",
+    ],
+)
+
+pkg_files(
+    name = "complex_monitoring_main_files",
+    srcs = [
+        ":component_complex_monitoring",
+        ":control_client_mock",
+        "//src/launch_manager_daemon:launch_manager",
+        "//tests/utils/test_helper:verification_process",
+    ],
+    attributes = pkg_attributes(mode = "0755"),
+    prefix = "tests/complex_monitoring",
+)
+
+pkg_files(
+    name = "complex_monitoring_etc_files",
+    srcs = [":lm_complex_monitoring_config"],
+    prefix = "tests/complex_monitoring",
+)
+
+pkg_tar(
+    name = "complex_monitoring_binaries",
+    srcs = [
+        ":complex_monitoring_etc_files",
+        ":complex_monitoring_main_files",
+    ],
+)
+
+integration_test(
+    name = "complex_monitoring",
+    srcs = ["complex_monitoring.py"],
+    tags = ["integration"],
+    test_binaries = ":complex_monitoring_binaries",
+    deps = ["//tests/utils/testing_utils"],
+)

tests/integration/complex_monitoring/complex_monitoring.json

@@ -0,0 +1,120 @@
+{
+    "schema_version": 1,
+    "defaults": {
+        "deployment_config": {
+            "bin_dir": "/tmp/tests/complex_monitoring",
+            "ready_timeout": 2.0,
+            "shutdown_timeout": 0.2,
+            "ready_recovery_action": {
+                "restart": {
+                    "number_of_attempts": 0
+                }
+            },
+            "recovery_action": {
+                "switch_run_target": {
+                    "run_target": "fallback_run_target"
+                }
+            },
+            "environmental_variables": {
+                "LD_LIBRARY_PATH": "/opt/lib"
+            },
+            "sandbox": {
+                "uid": 0,
+                "gid": 0,
+                "scheduling_policy": "SCHED_OTHER",
+                "scheduling_priority": 0
+            }
+        },
+        "component_properties": {
+            "application_profile": {
+                "application_type": "Reporting",
+                "is_self_terminating": false
+            },
+            "ready_condition": {
+                "process_state": "Running"
+            }
+        }
+    },
+    "components": {
+        "control_client_mock": {
+            "component_properties": {
+                "binary_name": "control_client_mock",
+                "application_profile": {
+                    "application_type": "State_Manager",
+                    "alive_supervision": {
+                        "min_indications": 0
+                    }
+                }
+            },
+            "deployment_config": {
+                "environmental_variables": {
+                    "PROCESSIDENTIFIER": "control_client_mock"
+                }
+            }
+        },
+        "component_complex_monitoring": {
+            "component_properties": {
+                "binary_name": "component_complex_monitoring",
+                "application_profile": {
+                    "application_type": "Reporting_And_Supervised",
+                    "alive_supervision": {
+                        "reporting_cycle": 0.1,
+                        "min_indications": 1,
+                        "max_indications": 3,
+                        "failed_cycles_tolerance": 0
+                    }
+                }
+            },
+            "deployment_config": {
+                "environmental_variables": {
+                    "PROCESSIDENTIFIER": "component_complex_monitoring",
+                    "IDENTIFIER": "component_complex_monitoring",
+                    "CONFIG_PATH": "etc/hmproc_component_complex_monitoring.bin"
+                }
+            }
+        },
+        "verification_component": {
+            "component_properties": {
+                "binary_name": "verification_process",
+                "application_profile": {
+                    "application_type": "Native",
+                    "is_self_terminating": true
+                },
+                "ready_condition": {
+                    "process_state": "Terminated"
+                }
+            }
+        }
+    },
+    "run_targets": {
+        "Startup": {
+            "depends_on": [
+                "control_client_mock"
+            ]
+        },
+        "run_target_complex_monitoring": {
+            "depends_on": [
+                "control_client_mock",
+                "component_complex_monitoring"
+            ],
+            "recovery_action": {
+                "switch_run_target": {
+                    "run_target": "fallback_run_target"
+                }
+            }
+        },
+        "Off": {
+            "depends_on": []
+        }
+    },
+    "initial_run_target": "Startup",
+    "alive_supervision": {
+        "evaluation_cycle": 0.05
+    },
+    "fallback_run_target": {
+        "depends_on": [
+            "control_client_mock",
+            "verification_component"
+        ]
+    }
+}

tests/integration/complex_monitoring/complex_monitoring.py

@@ -0,0 +1,44 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed
+from tests.utils.testing_utils.setup_test import setup_test
+from tests.utils.testing_utils.test_results import (
+    check_for_failures,
+    download_xml_results,
+)
+from attribute_plugin import add_test_properties
+
+
+@add_test_properties(
+    partially_verifies=[
+        # Health monitoring requirements not yet ready
+        "feat_req__lifecycle__ext_monitor_notify"
+    ],
+    test_type="requirements-based",
+    derivation_technique="requirements-analysis",
+)
+def test_complex_monitoring(target, setup_test, test_output_dir, remote_test_dir):
+    """Integration test for recovery actions triggered by heartbeat monitor failure. See component_complex_monitoring.cpp for detailed spec"""
+
+    run_until_file_deployed(
+        target=target,
+        binary_path=str(remote_test_dir / "launch_manager"),
+        file_path=remote_test_dir.parent / "test_end",
+        cwd=str(remote_test_dir),
+        timeout_s=4.0,
+    )
+
+    download_xml_results(target, remote_test_dir, test_output_dir)
+    all_files, failing_files = check_for_failures(test_output_dir)
+    assert len(all_files) == 2, f"Didn't find the expected number of files {all_files}"
+    assert len(failing_files) == 0, f"Found failures in files {failing_files}"

tests/integration/complex_monitoring/component_complex_monitoring.cpp

@@ -0,0 +1,75 @@
+/********************************************************************************
+ * Copyright (c) 2026 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+#include <gtest/gtest.h>
+#include <thread>
+
+#include "score/lcm/lifecycle_client.h"
+#include "score/mw/log/rust/stdout_logger_init.h"
+#include "tests/utils/test_helper/test_helper.hpp"
+#include <score/hm/common.h>
+#include <score/hm/health_monitor.h>
+
+TEST(ComplexMonitoring, ComponentComplexMonitoring)
+{
+    // Rust logger required for hmon logs
+    score::mw::log::rust::StdoutLoggerBuilder builder;
+    builder.Context("APP").LogLevel(score::mw::log::rust::LogLevel::Verbose).SetAsDefaultLogger();
+
+    using namespace score::hm;
+    using namespace std::chrono_literals;
+
+    auto hm_result = HealthMonitorBuilder()
+                         .add_heartbeat_monitor(MonitorTag("complex_monitoring_monitor"),
+                                                heartbeat::HeartbeatMonitorBuilder(TimeRange(50ms, 150ms)))
+                         .with_internal_processing_cycle(50ms)
+                         .with_supervisor_api_cycle(50ms)
+                         .build();
+    ASSERT_TRUE(hm_result.has_value()) << "Failed to build HealthMonitor";
+
+    auto hm = std::move(*hm_result);
+
+    auto heartbeat_monitor_result = hm.get_heartbeat_monitor(MonitorTag("complex_monitoring_monitor"));
+    ASSERT_TRUE(heartbeat_monitor_result.has_value()) << "Failed to get heartbeat monitor";
+    auto heartbeat_monitor = std::move(*heartbeat_monitor_result);
+
+    hm.start();
+
+    TEST_STEP("Report kRunning")
+    {
+        auto result = score::lcm::LifecycleClient{}.ReportExecutionState(score::lcm::ExecutionState::kRunning);
+        ASSERT_TRUE(result.has_value()) << "ReportExecutionState() failed: " << result.error().Message();
+    }
+
+    auto time_to_report_checkpoints_until = std::chrono::steady_clock::now() + 1s;
+    // Given that we can send heartbeats successfully...
+    TEST_STEP("Send heartbeats for 1 second")
+    {
+        while (std::chrono::steady_clock::now() < time_to_report_checkpoints_until)
+        {
+            std::this_thread::sleep_for(100ms);
+            heartbeat_monitor.heartbeat();
+        }
+        EXPECT_FALSE(TestRunner::exitRequested) << "Process should not be terminated yet";
+    }
+    // When heartbeats are no longer sent...
+}
+
+int main(int argc, char** argv)
+{
+    TestRunner(__FILE__, false).RunTests();
+    // Then expect kill due to recovery action (verified by control client)
+    while (true)  // Stop reporting, wait for sigkill
+    {
+        pause();
+    }
+}

tests/integration/complex_monitoring/control_client_mock.cpp

@@ -0,0 +1,54 @@
+/********************************************************************************
+ * Copyright (c) 2026 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include "tests/utils/test_helper/test_helper.hpp"
+#include <score/lcm/control_client.h>
+#include <score/lcm/lifecycle_client.h>
+
+score::lcm::ControlClient client;
+
+TEST(ComplexMonitoring, ControlClientMock)
+{
+    ASSERT_TRUE(check_clean({test_end_location, fallback_file}));
+
+    TEST_STEP("Report kRunning")
+    {
+        auto result = score::lcm::LifecycleClient{}.ReportExecutionState(score::lcm::ExecutionState::kRunning);
+        ASSERT_TRUE(result.has_value()) << "ReportExecutionState() failed: " << result.error().Message();
+    }
+    TEST_STEP("Launch monitored process")
+    {
+        score::cpp::stop_token stop_token;
+        auto result = client.ActivateRunTarget("run_target_complex_monitoring").Get(stop_token);
+        EXPECT_TRUE(result.has_value()) << "Activating target run_target_complex_monitoring failed: "
+                                        << result.error().Message();
+    }
+    // Wait for health monitoring to fail and recovery to trigger
+    sleep(2);
+    TEST_STEP("Verify state changed to fallback run target")
+    {
+        // workaround to detect we're in fallback
+        EXPECT_TRUE(std::filesystem::exists(fallback_file)) << "Fallback run target was not activated";
+    }
+    TEST_STEP("Activate Off run target")
+    {
+        client.ActivateRunTarget("Off");
+    }
+}
+
+int main(int argc, char** argv)
+{
+    return TestRunner(__FILE__, true, true).RunTests();
+}