diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3b613bf77..767cd57f6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -115,6 +115,9 @@ jobs: env: # Fail instead of silently skipping if the schema files above are missing (see test_helpers.py). REQUIRE_SCHEMA_TESTS: "1" + # A CouchDB service container is always started for this job (see `services:` above), so a + # missing or misconfigured CouchDB should fail the build, not silently skip the CouchDB tests. + REQUIRE_COUCHDB_INTEGRATION_TESTS: "1" run: | python -m coverage run --source=basyx -m unittest - name: Report test coverage diff --git a/sdk/test/_helper/setup_testdb.py b/sdk/test/_helper/setup_testdb.py index e5f15d0ae..c093b726f 100755 --- a/sdk/test/_helper/setup_testdb.py +++ b/sdk/test/_helper/setup_testdb.py @@ -13,8 +13,10 @@ setup_testdb.py -u admin -p admin_password -If no CouchDB server at the configured URL, the script will exit with exit code 1. To avoid the error exit code (for use -in CI), provide the ``--failsafe`` option. +If no CouchDB server is reachable at the configured URL, or the test database already exists, the script exits with +exit code 1 by default. CI relies on this to fail fast if the CouchDB service container isn't healthy, rather than +silently continuing into a test run that would just skip the CouchDB-backed tests. For idempotent/local re-runs +where an already-existing database is expected and fine, pass ``--failsafe`` to exit with code 0 instead. """ import argparse diff --git a/sdk/test/_helper/test_helpers.py b/sdk/test/_helper/test_helpers.py index 71975d4d8..12fac4985 100644 --- a/sdk/test/_helper/test_helpers.py +++ b/sdk/test/_helper/test_helpers.py @@ -21,13 +21,22 @@ ) +def env_flag(name: str) -> bool: + """Return whether the environment variable ``name`` is set to "1", "true" or "yes" (case-insensitive).""" + return os.environ.get(name, "false").lower() in {"1", "true", "yes"} + + # By default, the schema validation tests are skipped when the AAS schema files are missing, so that a plain local # `python -m unittest` run doesn't require downloading them first. Set this environment variable to "1"/"true" # (e.g. in CI) to instead make those tests fail if the files are missing, so a broken download can't silently cause # the tests to be skipped without anyone noticing. -REQUIRE_SCHEMAS = os.environ.get("REQUIRE_SCHEMA_TESTS", "false").lower() in {"1", "true", "yes"} +REQUIRE_SCHEMAS = env_flag("REQUIRE_SCHEMA_TESTS") + +# Likewise, the CouchDB tests are skipped when no CouchDB is reachable. Set this environment variable to fail them +# instead. +REQUIRE_COUCHDB_INTEGRATION_TESTS = env_flag("REQUIRE_COUCHDB_INTEGRATION_TESTS") -# Check if CouchDB database is available. Otherwise, skip tests. +# Check if CouchDB database is available. Otherwise, skip tests (unless REQUIRE_COUCHDB_INTEGRATION_TESTS is set). try: request = urllib.request.Request( "{}/{}".format( @@ -53,3 +62,7 @@ except urllib.error.URLError as e: COUCHDB_OKAY = False COUCHDB_ERROR = e + +COUCHDB_UNREACHABLE_MESSAGE = "No CouchDB is reachable at {}/{}: {}".format( + TEST_CONFIG["couchdb"]["url"], TEST_CONFIG["couchdb"]["database"], COUCHDB_ERROR +) diff --git a/sdk/test/backend/test_couchdb.py b/sdk/test/backend/test_couchdb.py index 829ccb889..6d1b73e00 100644 --- a/sdk/test/backend/test_couchdb.py +++ b/sdk/test/backend/test_couchdb.py @@ -10,7 +10,12 @@ from basyx.aas.backend import couchdb from basyx.aas.examples.data.example_aas import * -from test._helper.test_helpers import COUCHDB_ERROR, COUCHDB_OKAY, TEST_CONFIG +from test._helper.test_helpers import ( + COUCHDB_OKAY, + COUCHDB_UNREACHABLE_MESSAGE, + REQUIRE_COUCHDB_INTEGRATION_TESTS, + TEST_CONFIG, +) source_core: str = ( "couchdb://" @@ -21,13 +26,14 @@ ) -@unittest.skipUnless( - COUCHDB_OKAY, - "No CouchDB is reachable at {}/{}: {}".format( - TEST_CONFIG["couchdb"]["url"], TEST_CONFIG["couchdb"]["database"], COUCHDB_ERROR - ), -) +@unittest.skipUnless(COUCHDB_OKAY or REQUIRE_COUCHDB_INTEGRATION_TESTS, COUCHDB_UNREACHABLE_MESSAGE) class CouchDBBackendTest(unittest.TestCase): + + @classmethod + def setUpClass(cls) -> None: + if not COUCHDB_OKAY: + raise RuntimeError(f"REQUIRE_COUCHDB_INTEGRATION_TESTS is set. {COUCHDB_UNREACHABLE_MESSAGE}") + def setUp(self) -> None: self.couch_identifiable_store = couchdb.CouchDBIdentifiableStore( TEST_CONFIG["couchdb"]["url"], TEST_CONFIG["couchdb"]["database"] diff --git a/sdk/test/examples/test_tutorials.py b/sdk/test/examples/test_tutorials.py index e2b7e0277..0660f8351 100644 --- a/sdk/test/examples/test_tutorials.py +++ b/sdk/test/examples/test_tutorials.py @@ -17,7 +17,7 @@ from basyx.aas import model -from .._helper.test_helpers import COUCHDB_ERROR, COUCHDB_OKAY, TEST_CONFIG +from .._helper.test_helpers import COUCHDB_OKAY, COUCHDB_UNREACHABLE_MESSAGE, REQUIRE_COUCHDB_INTEGRATION_TESTS class TutorialTest(unittest.TestCase): @@ -35,16 +35,10 @@ def test_tutorial_storage(self): pass # The tutorial already includes assert statements for the relevant points. So no further checks are required. - @unittest.skipUnless( - COUCHDB_OKAY, - "No CouchDB is reachable at {}/{}: {}".format( - TEST_CONFIG["couchdb"]["url"], - TEST_CONFIG["couchdb"]["database"], - COUCHDB_ERROR, - ), - ) + @unittest.skipUnless(COUCHDB_OKAY or REQUIRE_COUCHDB_INTEGRATION_TESTS, COUCHDB_UNREACHABLE_MESSAGE) def test_tutorial_backend_couchdb(self): - pass + if not COUCHDB_OKAY: + self.fail(f"REQUIRE_COUCHDB_INTEGRATION_TESTS is set. {COUCHDB_UNREACHABLE_MESSAGE}") # The tutorial already includes assert statements for the relevant points. So no further checks are required. def test_tutorial_serialization_deserialization_json(self):