Fix flaky test caused by detector name collision in parallel CI jobs

test/conftest.py

@@ -1,10 +1,23 @@
 from datetime import datetime
+from typing import Callable
+from uuid import uuid4
 
 import pytest
 from groundlight import ExperimentalApi, Groundlight
 from model import Detector, ImageQuery, ImageQueryTypeEnum, ResultTypeEnum
 
 
+def _generate_unique_detector_name(prefix: str = "Test") -> str:
+    """Generates a detector name with a timestamp and random suffix to ensure uniqueness."""
+    return f"{prefix} {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}_{uuid4().hex[:8]}"
+
+
+@pytest.fixture(name="detector_name")
+def fixture_detector_name() -> Callable[..., str]:
+    """Fixture that provides a callable to generate unique detector names."""
+    return _generate_unique_detector_name
+
+
 def pytest_configure(config):  # pylint: disable=unused-argument
     # Run environment check before tests
     gl = Groundlight()
@@ -25,20 +38,18 @@ def fixture_gl() -> Groundlight:
 @pytest.fixture(name="detector")
 def fixture_detector(gl: Groundlight) -> Detector:
     """Creates a new Test detector."""
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
     query = "Is there a dog?"
     pipeline_config = "never-review"
-    return gl.create_detector(name=name, query=query, pipeline_config=pipeline_config)
+    return gl.create_detector(name=_generate_unique_detector_name(), query=query, pipeline_config=pipeline_config)
 
 
 @pytest.fixture(name="count_detector")
 def fixture_count_detector(gl_experimental: ExperimentalApi) -> Detector:
     """Creates a new Test detector."""
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
     query = "How many dogs?"
     pipeline_config = "never-review-multi"  # always predicts 0
     return gl_experimental.create_counting_detector(
-        name=name, query=query, class_name="dog", pipeline_config=pipeline_config
+        name=_generate_unique_detector_name(), query=query, class_name="dog", pipeline_config=pipeline_config
     )
 
 

test/integration/test_groundlight.py

@@ -5,8 +5,7 @@
 import random
 import string
 import time
-from datetime import datetime
-from typing import Any, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union
 
 import pytest
 from groundlight import Groundlight
@@ -96,8 +95,8 @@ def test_create_groundlight_with_retries():
     assert gl.api_client.configuration.retries.total == retries.total
 
 
-def test_create_detector(gl: Groundlight):
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
+def test_create_detector(gl: Groundlight, detector_name: Callable):
+    name = detector_name()
     query = "Is there a dog?"
     _detector = gl.create_detector(name=name, query=query)
     assert str(_detector)
@@ -107,29 +106,27 @@ def test_create_detector(gl: Groundlight):
     ), "We expected the default confidence threshold to be used."
 
     # Test creating dectors with other modes
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
-    count_detector = gl.create_detector(name=name, query=query, mode=ModeEnum.COUNT, class_names="dog")
+    count_detector = gl.create_detector(name=detector_name(), query=query, mode=ModeEnum.COUNT, class_names="dog")
     assert str(count_detector)
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
     multiclass_detector = gl.create_detector(
-        name=name, query=query, mode=ModeEnum.MULTI_CLASS, class_names=["dog", "cat"]
+        name=detector_name(), query=query, mode=ModeEnum.MULTI_CLASS, class_names=["dog", "cat"]
     )
     assert str(multiclass_detector)
 
 
-def test_create_detector_with_pipeline_config(gl: Groundlight):
+def test_create_detector_with_pipeline_config(gl: Groundlight, detector_name: Callable):
     # "never-review" is a special model that always returns the same result with 100% confidence.
     # It's useful for testing.
-    name = f"Test never-review {datetime.utcnow()}"  # Need a unique name
+    name = detector_name("Test never-review")
     query = "Is there a dog (always-pass)?"
     pipeline_config = "never-review"
     _detector = gl.create_detector(name=name, query=query, pipeline_config=pipeline_config)
     assert str(_detector)
     assert isinstance(_detector, Detector)
 
 
-def test_create_detector_with_edge_pipeline_config(gl: Groundlight):
-    name = f"Test edge-pipeline-config {datetime.utcnow()}"
+def test_create_detector_with_edge_pipeline_config(gl: Groundlight, detector_name: Callable):
+    name = detector_name("Test edge-pipeline-config")
     query = "Is there a dog (edge-config)?"
     _detector = gl.create_detector(
         name=name,
@@ -141,10 +138,10 @@ def test_create_detector_with_edge_pipeline_config(gl: Groundlight):
     assert isinstance(_detector, Detector)
 
 
-def test_create_detector_with_confidence_threshold(gl: Groundlight):
+def test_create_detector_with_confidence_threshold(gl: Groundlight, detector_name: Callable):
     # "never-review" is a special model that always returns the same result with 100% confidence.
     # It's useful for testing.
-    name = f"Test with confidence {datetime.utcnow()}"  # Need a unique name
+    name = detector_name("Test with confidence")
     query = "Is there a dog in the image?"
     pipeline_config = "never-review"
     confidence_threshold = 0.825
@@ -197,8 +194,8 @@ def test_create_detector_with_confidence_threshold(gl: Groundlight):
 
 
 @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing detector metadata.")
-def test_create_detector_with_everything(gl: Groundlight):
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
+def test_create_detector_with_everything(gl: Groundlight, detector_name: Callable):
+    name = detector_name()
     query = "Is there a dog?"
     group_name = "Test group"
     confidence_threshold = 0.825
@@ -232,9 +229,9 @@ def test_list_detectors(gl: Groundlight):
     assert isinstance(detectors, PaginatedDetectorList)
 
 
-def test_get_or_create_detector(gl: Groundlight):
+def test_get_or_create_detector(gl: Groundlight, detector_name: Callable):
     # With a unique name, we should be creating a new detector.
-    unique_name = f"Unique name {datetime.utcnow()}"
+    unique_name = detector_name()
     query = "Is there a dog?"
     detector = gl.get_or_create_detector(name=unique_name, query=query)
     assert str(detector)
@@ -410,8 +407,8 @@ def test_submit_image_query_with_low_request_timeout(gl: Groundlight, detector:
 
 
 @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing detector metadata.")
-def test_create_detector_with_metadata(gl: Groundlight):
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
+def test_create_detector_with_metadata(gl: Groundlight, detector_name: Callable):
+    name = detector_name()
     query = "Is there a dog?"
     metadata = generate_random_dict(target_size_bytes=200)
     detector = gl.create_detector(name=name, query=query, metadata=metadata)
@@ -422,8 +419,8 @@ def test_create_detector_with_metadata(gl: Groundlight):
 
 
 @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing detector metadata.")
-def test_get_or_create_detector_with_metadata(gl: Groundlight):
-    unique_name = f"Unique name {datetime.utcnow()}"
+def test_get_or_create_detector_with_metadata(gl: Groundlight, detector_name: Callable):
+    unique_name = detector_name()
     query = "Is there a dog?"
     metadata = generate_random_dict(target_size_bytes=200)
     detector = gl.get_or_create_detector(name=unique_name, query=query, metadata=metadata)
@@ -443,8 +440,8 @@ def test_get_or_create_detector_with_metadata(gl: Groundlight):
         [""],
     ],
 )
-def test_create_detector_with_invalid_metadata(gl: Groundlight, metadata_list: Any):
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
+def test_create_detector_with_invalid_metadata(gl: Groundlight, metadata_list: Any, detector_name: Callable):
+    name = detector_name()
     query = "Is there a dog?"
 
     for metadata in metadata_list:
@@ -627,9 +624,9 @@ def test_list_image_queries(gl: Groundlight):
             assert is_valid_display_result(image_query.result)
 
 
-def test_list_image_queries_with_filter(gl: Groundlight):
+def test_list_image_queries_with_filter(gl: Groundlight, detector_name: Callable):
     # We want a fresh detector so we know exactly what image queries are associated with it
-    detector = gl.create_detector(name=f"Test {datetime.utcnow()}", query="Is there a dog?")
+    detector = gl.create_detector(name=detector_name(), query="Is there a dog?")
     image_query_yes = gl.ask_async(detector=detector.id, image="test/assets/dog.jpeg", human_review="NEVER")
     image_query_no = gl.ask_async(detector=detector.id, image="test/assets/cat.jpeg", human_review="NEVER")
     iq_ids = [image_query_yes.id, image_query_no.id]
@@ -855,33 +852,33 @@ def test_submit_image_query_with_empty_inspection_id(gl: Groundlight, detector:
     )
 
 
-def test_binary_detector(gl: Groundlight):
+def test_binary_detector(gl: Groundlight, detector_name: Callable):
     """
     verify that we can create and submit to a binary detector
     """
-    name = f"Test {datetime.utcnow()}"
+    name = detector_name()
     created_detector = gl.create_binary_detector(name, "Is there a dog", confidence_threshold=0.0)
     assert created_detector is not None
     binary_iq = gl.submit_image_query(created_detector, "test/assets/dog.jpeg")
     assert binary_iq.result.label is not None
 
 
-def test_counting_detector(gl: Groundlight):
+def test_counting_detector(gl: Groundlight, detector_name: Callable):
     """
     verify that we can create and submit to a counting detector
     """
-    name = f"Test {datetime.utcnow()}"
+    name = detector_name()
     created_detector = gl.create_counting_detector(name, "How many dogs", "dog", confidence_threshold=0.0)
     assert created_detector is not None
     count_iq = gl.submit_image_query(created_detector, "test/assets/dog.jpeg")
     assert count_iq.result.count is not None
 
 
-def test_counting_detector_async(gl: Groundlight):
+def test_counting_detector_async(gl: Groundlight, detector_name: Callable):
     """
     verify that we can create and submit to a counting detector
     """
-    name = f"Test {datetime.utcnow()}"
+    name = detector_name()
     created_detector = gl.create_counting_detector(name, "How many dogs", "dog", confidence_threshold=0.0)
     assert created_detector is not None
     async_iq = gl.ask_async(created_detector, "test/assets/dog.jpeg")
@@ -896,11 +893,11 @@ def test_counting_detector_async(gl: Groundlight):
     assert _image_query.result is not None
 
 
-def test_multiclass_detector(gl: Groundlight):
+def test_multiclass_detector(gl: Groundlight, detector_name: Callable):
     """
     verify that we can create and submit to a multi-class detector
     """
-    name = f"Test {datetime.utcnow()}"
+    name = detector_name()
     class_names = ["Golden Retriever", "Labrador Retriever", "Poodle"]
     created_detector = gl.create_multiclass_detector(
         name, "What kind of dog is this?", class_names=class_names, confidence_threshold=0.0
@@ -911,12 +908,12 @@ def test_multiclass_detector(gl: Groundlight):
     assert mc_iq.result.label in class_names
 
 
-def test_delete_detector(gl: Groundlight):
+def test_delete_detector(gl: Groundlight, detector_name: Callable):
     """
     Test deleting a detector by both ID and object, and verify proper error handling.
     """
     # Create a detector to delete
-    name = f"Test delete detector {datetime.utcnow()}"
+    name = detector_name("Test delete detector")
     query = "Is there a dog to delete?"
     pipeline_config = "never-review"
     detector = gl.create_detector(name=name, query=query, pipeline_config=pipeline_config)
@@ -929,7 +926,7 @@ def test_delete_detector(gl: Groundlight):
         gl.get_detector(detector.id)
 
     # Create another detector to test deletion by ID string and that an attached image query is deleted
-    name2 = f"Test delete detector 2 {datetime.utcnow()}"
+    name2 = detector_name("Test delete detector 2")
     detector2 = gl.create_detector(name=name2, query=query, pipeline_config=pipeline_config)
     gl.submit_image_query(detector2, "test/assets/dog.jpeg")
 
@@ -952,14 +949,14 @@ def test_delete_detector(gl: Groundlight):
         gl.delete_detector(fake_detector_id)  # type: ignore
 
 
-def test_create_detector_with_invalid_priming_group_id(gl: Groundlight):
+def test_create_detector_with_invalid_priming_group_id(gl: Groundlight, detector_name: Callable):
     """
     Test that creating a detector with a non-existent priming_group_id returns an appropriate error.
 
     Note: PrimingGroup IDs are provided by Groundlight representatives. If you would like to
     use a priming_group_id, please reach out to your Groundlight representative.
     """
-    name = f"Test invalid priming {datetime.utcnow()}"
+    name = detector_name("Test invalid priming")
     query = "Is there a dog?"
     pipeline_config = "never-review"
     priming_group_id = "prgrp_nonexistent12345678901234567890"

test/integration/test_groundlight_expensive.py

@@ -7,7 +7,7 @@
 # pylint: disable=wildcard-import,unused-wildcard-import,redefined-outer-name,import-outside-toplevel
 import random
 import time
-from datetime import datetime
+from typing import Callable
 
 import pytest
 from groundlight import Groundlight
@@ -30,8 +30,8 @@ def fixture_gl() -> Groundlight:
 
 
 @pytest.mark.skip(reason="This test requires a human labeler who does not need to be in the testing loop")
-def test_human_label(gl: Groundlight):
-    detector = gl.create_detector(name=f"Test {datetime.utcnow()}", query="Is there a dog?")
+def test_human_label(gl: Groundlight, detector_name: Callable):
+    detector = gl.create_detector(name=detector_name(), query="Is there a dog?")
     img_query = gl.submit_image_query(
         detector=detector.id, image="test/assets/dog.jpeg", wait=60, human_review="ALWAYS"
     )
@@ -52,7 +52,7 @@ def test_human_label(gl: Groundlight):
 
 @pytest.mark.skip(reason="This test can block development depending on the state of the service")
 @pytest.mark.skipif(MISSING_PIL, reason="Needs pillow")  # type: ignore
-def test_detector_improvement(gl: Groundlight):
+def test_detector_improvement(gl: Groundlight, detector_name: Callable):
     # test that we get confidence improvement after sending images in
     # Pass two of each type of image in
     import time
@@ -61,7 +61,7 @@ def test_detector_improvement(gl: Groundlight):
 
     random.seed(2741)
 
-    name = f"Test test_detector_improvement {datetime.utcnow()}"  # Need a unique name
+    name = detector_name("Test test_detector_improvement")
     query = "Is there a dog?"
     detector = gl.create_detector(name=name, query=query)
 
@@ -122,11 +122,11 @@ def submit_noisy_image(image, label=None):
 @pytest.mark.skip(
     reason="We don't yet have an SLA level to test ask_confident against, and the test is flakey as a result"
 )
-def test_ask_method_quality(gl: Groundlight, detector: Detector):
+def test_ask_method_quality(gl: Groundlight, detector: Detector, detector_name: Callable):
     # asks for some level of quality on how fast ask_ml is and that we will get a confident result from ask_confident
     fast_always_yes_iq = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=0)
     assert iq_is_answered(fast_always_yes_iq)
-    name = f"Test {datetime.utcnow()}"  # Need a unique name
+    name = detector_name()
     query = "Is there a dog?"
     detector = gl.create_detector(name=name, query=query, confidence_threshold=0.8)
     fast_iq = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=0)

test/unit/test_cli.py

@@ -1,7 +1,7 @@
 import os
 import re
 import subprocess
-from datetime import datetime
+from typing import Callable
 from unittest.mock import patch
 
 
@@ -23,9 +23,9 @@ def test_list_detector():
     assert completed_process.returncode == 0
 
 
-def test_detector_and_image_queries():
+def test_detector_and_image_queries(detector_name: Callable):
     # test creating a detector
-    test_detector_name = f"testdetector {datetime.utcnow()}"
+    test_detector_name = detector_name("testdetector")
     completed_process = subprocess.run(
         [
             "groundlight",
@@ -41,7 +41,9 @@ def test_detector_and_image_queries():
         check=False,
     )
     assert completed_process.returncode == 0
-    det_id_on_create = re.search("id='([^']+)'", completed_process.stdout).group(1)
+    match = re.search("id='([^']+)'", completed_process.stdout)
+    assert match is not None
+    det_id_on_create = match.group(1)
     # The output of the create-detector command looks something like:
     # id='det_abc123'
     # type=<DetectorTypeEnum.detector: 'detector'>
@@ -59,7 +61,9 @@ def test_detector_and_image_queries():
         check=False,
     )
     assert completed_process.returncode == 0
-    det_id_on_get = re.search("id='([^']+)'", completed_process.stdout).group(1)
+    match = re.search("id='([^']+)'", completed_process.stdout)
+    assert match is not None
+    det_id_on_get = match.group(1)
     assert det_id_on_create == det_id_on_get
     completed_process = subprocess.run(
         ["groundlight", "get-detector", det_id_on_create],