diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 65589050e4..97213ec2fe 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -53206,6 +53206,8 @@ components: description: S3 bucket name. example: "my-bucket" type: string + buffer: + $ref: "#/components/schemas/ObservabilityPipelineBufferOptions" compression: $ref: "#/components/schemas/ObservabilityPipelineAmazonS3GenericCompression" encoding: @@ -53455,6 +53457,8 @@ components: **Supported pipeline types:** logs properties: + buffer: + $ref: "#/components/schemas/ObservabilityPipelineBufferOptions" endpoint_url_key: description: Name of the environment variable or secret that holds the CloudPrem endpoint URL. example: CLOUDPREM_ENDPOINT_URL @@ -55105,6 +55109,8 @@ components: properties: auth_strategy: $ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationAuthStrategy" + buffer: + $ref: "#/components/schemas/ObservabilityPipelineBufferOptions" compression: $ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationCompression" custom_key: @@ -55389,6 +55395,8 @@ components: description: Name of the environment variable or secret that holds the Kafka bootstrap servers list. example: KAFKA_BOOTSTRAP_SERVERS type: string + buffer: + $ref: "#/components/schemas/ObservabilityPipelineBufferOptions" compression: $ref: "#/components/schemas/ObservabilityPipelineKafkaDestinationCompression" encoding: diff --git a/examples/v2/observability-pipelines/ValidatePipeline_1070998129.py b/examples/v2/observability-pipelines/ValidatePipeline_1070998129.py new file mode 100644 index 0000000000..d192027839 --- /dev/null +++ b/examples/v2/observability-pipelines/ValidatePipeline_1070998129.py @@ -0,0 +1,95 @@ +""" +Validate an observability pipeline with cloud_prem destination buffer returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi +from datadog_api_client.v2.model.observability_pipeline_buffer_options_disk_type import ( + ObservabilityPipelineBufferOptionsDiskType, +) +from datadog_api_client.v2.model.observability_pipeline_buffer_options_when_full import ( + ObservabilityPipelineBufferOptionsWhenFull, +) +from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination import ( + ObservabilityPipelineCloudPremDestination, +) +from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination_type import ( + ObservabilityPipelineCloudPremDestinationType, +) +from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) +from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( + ObservabilityPipelineDatadogAgentSource, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import ( + ObservabilityPipelineDatadogAgentSourceType, +) +from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import ( + ObservabilityPipelineDiskBufferOptions, +) +from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor +from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import ( + ObservabilityPipelineFilterProcessorType, +) +from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec +from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData + +body = ObservabilityPipelineSpec( + data=ObservabilityPipelineSpecData( + attributes=ObservabilityPipelineDataAttributes( + config=ObservabilityPipelineConfig( + destinations=[ + ObservabilityPipelineCloudPremDestination( + id="cloud-prem-destination", + inputs=[ + "my-processor-group", + ], + type=ObservabilityPipelineCloudPremDestinationType.CLOUD_PREM, + endpoint_url_key="CLOUDPREM_ENDPOINT_URL", + buffer=ObservabilityPipelineDiskBufferOptions( + type=ObservabilityPipelineBufferOptionsDiskType.DISK, + max_size=1073741824, + when_full=ObservabilityPipelineBufferOptionsWhenFull.BLOCK, + ), + ), + ], + processor_groups=[ + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", + include="service:my-service", + inputs=[ + "datadog-agent-source", + ], + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="status:error", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], + ), + ], + sources=[ + ObservabilityPipelineDatadogAgentSource( + id="datadog-agent-source", + type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT, + ), + ], + ), + name="Pipeline with CloudPrem Buffer", + ), + type="pipelines", + ), +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = ObservabilityPipelinesApi(api_client) + response = api_instance.validate_pipeline(body=body) + + print(response) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_generic_destination.py b/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_generic_destination.py index 2c7e740af7..136307b4e2 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_generic_destination.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_generic_destination.py @@ -18,6 +18,7 @@ from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_batch_settings import ( ObservabilityPipelineAmazonS3GenericBatchSettings, ) + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_compression import ( ObservabilityPipelineAmazonS3GenericCompression, ) @@ -30,6 +31,15 @@ from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_destination_type import ( ObservabilityPipelineAmazonS3GenericDestinationType, ) + from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import ( + ObservabilityPipelineDiskBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import ( + ObservabilityPipelineMemoryBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import ( + ObservabilityPipelineMemoryBufferSizeOptions, + ) from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_compression_zstd import ( ObservabilityPipelineAmazonS3GenericCompressionZstd, ) @@ -54,6 +64,7 @@ def openapi_types(_): from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_batch_settings import ( ObservabilityPipelineAmazonS3GenericBatchSettings, ) + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_compression import ( ObservabilityPipelineAmazonS3GenericCompression, ) @@ -71,6 +82,7 @@ def openapi_types(_): "auth": (ObservabilityPipelineAwsAuth,), "batch_settings": (ObservabilityPipelineAmazonS3GenericBatchSettings,), "bucket": (str,), + "buffer": (ObservabilityPipelineBufferOptions,), "compression": (ObservabilityPipelineAmazonS3GenericCompression,), "encoding": (ObservabilityPipelineAmazonS3GenericEncoding,), "id": (str,), @@ -85,6 +97,7 @@ def openapi_types(_): "auth": "auth", "batch_settings": "batch_settings", "bucket": "bucket", + "buffer": "buffer", "compression": "compression", "encoding": "encoding", "id": "id", @@ -116,6 +129,13 @@ def __init__( type: ObservabilityPipelineAmazonS3GenericDestinationType, auth: Union[ObservabilityPipelineAwsAuth, UnsetType] = unset, batch_settings: Union[ObservabilityPipelineAmazonS3GenericBatchSettings, UnsetType] = unset, + buffer: Union[ + ObservabilityPipelineBufferOptions, + ObservabilityPipelineDiskBufferOptions, + ObservabilityPipelineMemoryBufferOptions, + ObservabilityPipelineMemoryBufferSizeOptions, + UnsetType, + ] = unset, key_prefix: Union[str, UnsetType] = unset, **kwargs, ): @@ -134,6 +154,9 @@ def __init__( :param bucket: S3 bucket name. :type bucket: str + :param buffer: Configuration for buffer settings on destination components. + :type buffer: ObservabilityPipelineBufferOptions, optional + :param compression: Compression algorithm applied to encoded logs. :type compression: ObservabilityPipelineAmazonS3GenericCompression @@ -162,6 +185,8 @@ def __init__( kwargs["auth"] = auth if batch_settings is not unset: kwargs["batch_settings"] = batch_settings + if buffer is not unset: + kwargs["buffer"] = buffer if key_prefix is not unset: kwargs["key_prefix"] = key_prefix super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_cloud_prem_destination.py b/src/datadog_api_client/v2/model/observability_pipeline_cloud_prem_destination.py index b26caa73a5..6fdf1196a4 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_cloud_prem_destination.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_cloud_prem_destination.py @@ -14,19 +14,31 @@ if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination_type import ( ObservabilityPipelineCloudPremDestinationType, ) + from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import ( + ObservabilityPipelineDiskBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import ( + ObservabilityPipelineMemoryBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import ( + ObservabilityPipelineMemoryBufferSizeOptions, + ) class ObservabilityPipelineCloudPremDestination(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination_type import ( ObservabilityPipelineCloudPremDestinationType, ) return { + "buffer": (ObservabilityPipelineBufferOptions,), "endpoint_url_key": (str,), "id": (str,), "inputs": ([str],), @@ -34,6 +46,7 @@ def openapi_types(_): } attribute_map = { + "buffer": "buffer", "endpoint_url_key": "endpoint_url_key", "id": "id", "inputs": "inputs", @@ -45,6 +58,13 @@ def __init__( id: str, inputs: List[str], type: ObservabilityPipelineCloudPremDestinationType, + buffer: Union[ + ObservabilityPipelineBufferOptions, + ObservabilityPipelineDiskBufferOptions, + ObservabilityPipelineMemoryBufferOptions, + ObservabilityPipelineMemoryBufferSizeOptions, + UnsetType, + ] = unset, endpoint_url_key: Union[str, UnsetType] = unset, **kwargs, ): @@ -53,6 +73,9 @@ def __init__( **Supported pipeline types:** logs + :param buffer: Configuration for buffer settings on destination components. + :type buffer: ObservabilityPipelineBufferOptions, optional + :param endpoint_url_key: Name of the environment variable or secret that holds the CloudPrem endpoint URL. :type endpoint_url_key: str, optional @@ -65,6 +88,8 @@ def __init__( :param type: The destination type. The value should always be ``cloud_prem``. :type type: ObservabilityPipelineCloudPremDestinationType """ + if buffer is not unset: + kwargs["buffer"] = buffer if endpoint_url_key is not unset: kwargs["endpoint_url_key"] = endpoint_url_key super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_http_client_destination.py b/src/datadog_api_client/v2/model/observability_pipeline_http_client_destination.py index 0eba7b7edf..34bb1cea18 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_http_client_destination.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_http_client_destination.py @@ -17,6 +17,7 @@ from datadog_api_client.v2.model.observability_pipeline_http_client_destination_auth_strategy import ( ObservabilityPipelineHttpClientDestinationAuthStrategy, ) + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_http_client_destination_compression import ( ObservabilityPipelineHttpClientDestinationCompression, ) @@ -27,6 +28,15 @@ from datadog_api_client.v2.model.observability_pipeline_http_client_destination_type import ( ObservabilityPipelineHttpClientDestinationType, ) + from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import ( + ObservabilityPipelineDiskBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import ( + ObservabilityPipelineMemoryBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import ( + ObservabilityPipelineMemoryBufferSizeOptions, + ) class ObservabilityPipelineHttpClientDestination(ModelNormal): @@ -35,6 +45,7 @@ def openapi_types(_): from datadog_api_client.v2.model.observability_pipeline_http_client_destination_auth_strategy import ( ObservabilityPipelineHttpClientDestinationAuthStrategy, ) + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_http_client_destination_compression import ( ObservabilityPipelineHttpClientDestinationCompression, ) @@ -48,6 +59,7 @@ def openapi_types(_): return { "auth_strategy": (ObservabilityPipelineHttpClientDestinationAuthStrategy,), + "buffer": (ObservabilityPipelineBufferOptions,), "compression": (ObservabilityPipelineHttpClientDestinationCompression,), "custom_key": (str,), "encoding": (ObservabilityPipelineHttpClientDestinationEncoding,), @@ -63,6 +75,7 @@ def openapi_types(_): attribute_map = { "auth_strategy": "auth_strategy", + "buffer": "buffer", "compression": "compression", "custom_key": "custom_key", "encoding": "encoding", @@ -83,6 +96,13 @@ def __init__( inputs: List[str], type: ObservabilityPipelineHttpClientDestinationType, auth_strategy: Union[ObservabilityPipelineHttpClientDestinationAuthStrategy, UnsetType] = unset, + buffer: Union[ + ObservabilityPipelineBufferOptions, + ObservabilityPipelineDiskBufferOptions, + ObservabilityPipelineMemoryBufferOptions, + ObservabilityPipelineMemoryBufferSizeOptions, + UnsetType, + ] = unset, compression: Union[ObservabilityPipelineHttpClientDestinationCompression, UnsetType] = unset, custom_key: Union[str, UnsetType] = unset, password_key: Union[str, UnsetType] = unset, @@ -100,6 +120,9 @@ def __init__( :param auth_strategy: HTTP authentication strategy. :type auth_strategy: ObservabilityPipelineHttpClientDestinationAuthStrategy, optional + :param buffer: Configuration for buffer settings on destination components. + :type buffer: ObservabilityPipelineBufferOptions, optional + :param compression: Compression configuration for HTTP requests. :type compression: ObservabilityPipelineHttpClientDestinationCompression, optional @@ -135,6 +158,8 @@ def __init__( """ if auth_strategy is not unset: kwargs["auth_strategy"] = auth_strategy + if buffer is not unset: + kwargs["buffer"] = buffer if compression is not unset: kwargs["compression"] = compression if custom_key is not unset: diff --git a/src/datadog_api_client/v2/model/observability_pipeline_kafka_destination.py b/src/datadog_api_client/v2/model/observability_pipeline_kafka_destination.py index 123a158197..415cdfb1f9 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_kafka_destination.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_kafka_destination.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_kafka_destination_compression import ( ObservabilityPipelineKafkaDestinationCompression, ) @@ -28,6 +29,15 @@ from datadog_api_client.v2.model.observability_pipeline_kafka_destination_type import ( ObservabilityPipelineKafkaDestinationType, ) + from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import ( + ObservabilityPipelineDiskBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import ( + ObservabilityPipelineMemoryBufferOptions, + ) + from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import ( + ObservabilityPipelineMemoryBufferSizeOptions, + ) class ObservabilityPipelineKafkaDestination(ModelNormal): @@ -49,6 +59,7 @@ class ObservabilityPipelineKafkaDestination(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions from datadog_api_client.v2.model.observability_pipeline_kafka_destination_compression import ( ObservabilityPipelineKafkaDestinationCompression, ) @@ -66,6 +77,7 @@ def openapi_types(_): return { "bootstrap_servers_key": (str,), + "buffer": (ObservabilityPipelineBufferOptions,), "compression": (ObservabilityPipelineKafkaDestinationCompression,), "encoding": (ObservabilityPipelineKafkaDestinationEncoding,), "headers_key": (str,), @@ -85,6 +97,7 @@ def openapi_types(_): attribute_map = { "bootstrap_servers_key": "bootstrap_servers_key", + "buffer": "buffer", "compression": "compression", "encoding": "encoding", "headers_key": "headers_key", @@ -110,6 +123,13 @@ def __init__( topic: str, type: ObservabilityPipelineKafkaDestinationType, bootstrap_servers_key: Union[str, UnsetType] = unset, + buffer: Union[ + ObservabilityPipelineBufferOptions, + ObservabilityPipelineDiskBufferOptions, + ObservabilityPipelineMemoryBufferOptions, + ObservabilityPipelineMemoryBufferSizeOptions, + UnsetType, + ] = unset, compression: Union[ObservabilityPipelineKafkaDestinationCompression, UnsetType] = unset, headers_key: Union[str, UnsetType] = unset, key_field: Union[str, UnsetType] = unset, @@ -130,6 +150,9 @@ def __init__( :param bootstrap_servers_key: Name of the environment variable or secret that holds the Kafka bootstrap servers list. :type bootstrap_servers_key: str, optional + :param buffer: Configuration for buffer settings on destination components. + :type buffer: ObservabilityPipelineBufferOptions, optional + :param compression: Compression codec for Kafka messages. :type compression: ObservabilityPipelineKafkaDestinationCompression, optional @@ -177,6 +200,8 @@ def __init__( """ if bootstrap_servers_key is not unset: kwargs["bootstrap_servers_key"] = bootstrap_servers_key + if buffer is not unset: + kwargs["buffer"] = buffer if compression is not unset: kwargs["compression"] = compression if headers_key is not unset: diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_cloud_prem_destination_buffer_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_cloud_prem_destination_buffer_returns_ok_response.frozen new file mode 100644 index 0000000000..48640770ec --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_cloud_prem_destination_buffer_returns_ok_response.frozen @@ -0,0 +1 @@ +2026-05-21T09:58:44.439Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_cloud_prem_destination_buffer_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_cloud_prem_destination_buffer_returns_ok_response.yaml new file mode 100644 index 0000000000..cb6b6fbce8 --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_cloud_prem_destination_buffer_returns_ok_response.yaml @@ -0,0 +1,23 @@ +interactions: +- request: + body: '{"data":{"attributes":{"config":{"destinations":[{"buffer":{"max_size":1073741824,"type":"disk","when_full":"block"},"id":"cloud-prem-destination","inputs":["my-processor-group"],"type":"cloud_prem"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Pipeline + with CloudPrem Buffer"},"type":"pipelines"}}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate + response: + body: + string: '{"errors":[]} + + ' + headers: + content-type: + - application/vnd.api+json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v2/features/observability_pipelines.feature b/tests/v2/features/observability_pipelines.feature index 24890aaddb..487d6971f8 100644 --- a/tests/v2/features/observability_pipelines.feature +++ b/tests/v2/features/observability_pipelines.feature @@ -247,6 +247,14 @@ Feature: Observability Pipelines Then the response status is 200 OK And the response "errors" has length 0 + @skip @team:DataDog/observability-pipelines + Scenario: Validate an observability pipeline with cloud_prem destination buffer returns "OK" response + Given new "ValidatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "cloud-prem-destination", "inputs": ["my-processor-group"], "type": "cloud_prem", "endpoint_url_key": "CLOUDPREM_ENDPOINT_URL", "buffer": {"type": "disk", "max_size": 1073741824, "when_full": "block"}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with CloudPrem Buffer"}, "type": "pipelines"}} + When the request is sent + Then the response status is 200 OK + And the response "errors" has length 0 + @team:DataDog/observability-pipelines Scenario: Validate an observability pipeline with destination secret key returns "OK" response Given new "ValidatePipeline" request