From 5878970a5d01027ba89472e491a9e1d26528b79d Mon Sep 17 00:00:00 2001 From: Malak El Kouri Date: Fri, 17 Apr 2026 13:51:42 +0200 Subject: [PATCH 1/3] Adapt sync-cli to support synthetics PLs replication for DDR --- datadog_sync/commands/shared/options.py | 7 +++ datadog_sync/constants.py | 1 + .../model/synthetics_private_locations.py | 63 +++++++++++++++++-- datadog_sync/utils/configuration.py | 3 + 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/datadog_sync/commands/shared/options.py b/datadog_sync/commands/shared/options.py index 34eb7927e..2a551d643 100644 --- a/datadog_sync/commands/shared/options.py +++ b/datadog_sync/commands/shared/options.py @@ -244,6 +244,13 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non "Disables progress bar.", cls=CustomOptionClass, ), + option( + "--datadog-host-override", + envvar=constants.DD_DATADOG_HOST_OVERRIDE, + required=False, + help="Optional CNAME override for the Datadog host used in DDR private location replication.", + cls=CustomOptionClass, + ), ] _storage_options = [ diff --git a/datadog_sync/constants.py b/datadog_sync/constants.py index 90297938f..61b450f94 100644 --- a/datadog_sync/constants.py +++ b/datadog_sync/constants.py @@ -27,6 +27,7 @@ DD_VERIFY_SSL_CERTIFICATES = "DD_VERIFY_SSL_CERTIFICATES" DD_ALLOW_PARTIAL_PERMISSIONS_ROLES = "DD_ALLOW_PARTIAL_PERMISSIONS_ROLES" DD_SYNC_JSON = "DD_SYNC_JSON" +DD_DATADOG_HOST_OVERRIDE = "DD_DATADOG_HOST_OVERRIDE" LOCAL_STORAGE_TYPE = "local" S3_STORAGE_TYPE = "s3" diff --git a/datadog_sync/model/synthetics_private_locations.py b/datadog_sync/model/synthetics_private_locations.py index 1f7d3ed1a..14cd12425 100644 --- a/datadog_sync/model/synthetics_private_locations.py +++ b/datadog_sync/model/synthetics_private_locations.py @@ -4,6 +4,8 @@ # Copyright 2019 Datadog, Inc. from __future__ import annotations +import json +import os import re from typing import TYPE_CHECKING, List, Dict, Optional, Tuple @@ -14,6 +16,9 @@ if TYPE_CHECKING: from datadog_sync.utils.custom_client import CustomClient +# Fields returned by include_pl_info=true that should not be stored in source state +_PL_INFO_FIELDS = ["pl_id", "org_id", "datacenter", "public_key_test", "public_key_result"] + class SyntheticsPrivateLocations(BaseResource): resource_type = "synthetics_private_locations" @@ -28,6 +33,7 @@ class SyntheticsPrivateLocations(BaseResource): "secrets", "config", "result_encryption", + "ddr_metadata", ], tagging_config=TaggingConfig(path="tags"), ) @@ -35,6 +41,11 @@ class SyntheticsPrivateLocations(BaseResource): base_locations_path: str = "/api/v1/synthetics/locations" pl_id_regex: re.Pattern = re.compile("^pl:.*") + def __init__(self, config): + super().__init__(config) + # In-memory store for include_pl_info data, keyed by source PL ID + self._pl_info: Dict[str, Dict] = {} + async def get_resources(self, client: CustomClient) -> List[Dict]: resp = await client.get(self.base_locations_path) @@ -47,10 +58,21 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di if not self.pl_id_regex.match(import_id): raise SkipResource(import_id, self.resource_type, "Managed location.") - pl = await source_client.get(self.resource_config.base_path + f"/{import_id}") - self.config.state.source[self.resource_type][import_id] = pl + resp = await source_client.get( + self.resource_config.base_path + f"/{import_id}", + params={"include_pl_info": "true"}, + ) + + # Extract pl_info fields for use during create, keep source state clean + pl_info = {} + for field in _PL_INFO_FIELDS: + if field in resp: + pl_info[field] = resp.pop(field) + self._pl_info[import_id] = pl_info + + self.config.state.source[self.resource_type][import_id] = resp - return import_id, pl + return import_id, resp async def pre_resource_action_hook(self, _id, resource: Dict) -> None: pass @@ -60,12 +82,32 @@ async def pre_apply_hook(self) -> None: async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: destination_client = self.config.destination_client + pl_info = self._pl_info.get(_id, {}) + + resource["ddr_metadata"] = { + "disaster_recovery": { + "source_pl_id": pl_info["pl_id"], + "source_name": _id, + "source_dc": pl_info["datacenter"], + "source_org_id": pl_info["org_id"], + } + } + resource["test_encryption_public_key"] = pl_info["public_key_test"] + resource["result_encryption_public_key"] = pl_info["public_key_result"] + if self.config.datadog_host_override: + resource["datadog_host_override"] = self.config.datadog_host_override resp = await destination_client.post(self.resource_config.base_path, resource) + # DDR response: {"private_location": {...}, "publicKeysByMainDC": {...}} pl = resp["private_location"] - pl["config"] = resp.get("config") - pl["result_encryption"] = resp.get("result_encryption") + + # Save PL config to file for later use running the PL + pl_config = { + "publicKeysByMainDC": resp.get("publicKeysByMainDC"), + "datadogHostOverride": self.config.datadog_host_override, + } + self._save_pl_config(pl.get("name", _id), pl_config) return _id, pl @@ -88,3 +130,14 @@ async def delete_resource(self, _id: str) -> None: def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]: return super(SyntheticsPrivateLocations, self).connect_id(key, r_obj, resource_to_connect) + + def _save_pl_config(self, pl_name: str, config: Dict) -> None: + destination_path = self.config.state._storage.destination_resources_path + config_dir = os.path.join(destination_path, "synthetics_private_locations_config") + os.makedirs(config_dir, exist_ok=True) + + sanitized_name = re.sub(r"[^\w\-]", "_", pl_name) + config_file = os.path.join(config_dir, f"{sanitized_name}.json") + + with open(config_file, "w", encoding="utf-8") as f: + json.dump(config, f, indent=2) diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index e5bbd99de..e9d9ac02c 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -66,6 +66,7 @@ class Configuration(object): backup_before_reset: bool show_progress_bar: bool allow_self_lockout: bool + datadog_host_override: Optional[str] = None emit_json: bool = False command: str = "" allow_partial_permissions_roles: List[str] = field(default_factory=list) @@ -208,6 +209,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: if emit_json: show_progress_bar = False allow_self_lockout = kwargs.get("allow_self_lockout", False) + datadog_host_override = kwargs.get("datadog_host_override") # Parse allow_partial_permissions_roles allow_partial_permissions_roles = [] @@ -334,6 +336,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration: backup_before_reset=backup_before_reset, show_progress_bar=show_progress_bar, allow_self_lockout=allow_self_lockout, + datadog_host_override=datadog_host_override, emit_json=emit_json, command=cmd.value, allow_partial_permissions_roles=allow_partial_permissions_roles, From d9e4212916fb6f778c233c41129ec4fddafb3e47 Mon Sep 17 00:00:00 2001 From: Malak El Kouri Date: Fri, 17 Apr 2026 15:31:53 +0200 Subject: [PATCH 2/3] fix datadogHostOverride --- .../model/synthetics_private_locations.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/datadog_sync/model/synthetics_private_locations.py b/datadog_sync/model/synthetics_private_locations.py index 14cd12425..8e0dc928e 100644 --- a/datadog_sync/model/synthetics_private_locations.py +++ b/datadog_sync/model/synthetics_private_locations.py @@ -16,10 +16,6 @@ if TYPE_CHECKING: from datadog_sync.utils.custom_client import CustomClient -# Fields returned by include_pl_info=true that should not be stored in source state -_PL_INFO_FIELDS = ["pl_id", "org_id", "datacenter", "public_key_test", "public_key_result"] - - class SyntheticsPrivateLocations(BaseResource): resource_type = "synthetics_private_locations" resource_config = ResourceConfig( @@ -41,11 +37,6 @@ class SyntheticsPrivateLocations(BaseResource): base_locations_path: str = "/api/v1/synthetics/locations" pl_id_regex: re.Pattern = re.compile("^pl:.*") - def __init__(self, config): - super().__init__(config) - # In-memory store for include_pl_info data, keyed by source PL ID - self._pl_info: Dict[str, Dict] = {} - async def get_resources(self, client: CustomClient) -> List[Dict]: resp = await client.get(self.base_locations_path) @@ -60,16 +51,8 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di resp = await source_client.get( self.resource_config.base_path + f"/{import_id}", - params={"include_pl_info": "true"}, ) - # Extract pl_info fields for use during create, keep source state clean - pl_info = {} - for field in _PL_INFO_FIELDS: - if field in resp: - pl_info[field] = resp.pop(field) - self._pl_info[import_id] = pl_info - self.config.state.source[self.resource_type][import_id] = resp return import_id, resp @@ -82,7 +65,17 @@ async def pre_apply_hook(self) -> None: async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: destination_client = self.config.destination_client - pl_info = self._pl_info.get(_id, {}) + source_client = self.config.source_client + + # Fetch pl_info from source API for DDR metadata + pl_info = await source_client.get( + self.resource_config.base_path + f"/{_id}", + params={"include_pl_info": "true"}, + ) + + # Strip null metadata — DDR endpoint requires it to be an object + if resource.get("metadata") is None: + resource.pop("metadata", None) resource["ddr_metadata"] = { "disaster_recovery": { @@ -92,8 +85,14 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: "source_org_id": pl_info["org_id"], } } - resource["test_encryption_public_key"] = pl_info["public_key_test"] - resource["result_encryption_public_key"] = pl_info["public_key_result"] + # test_encryption_public_key expects the JSON-stringified public_key_test object + resource["test_encryption_public_key"] = json.dumps(pl_info["public_key_test"]) + # result_encryption_public_key expects {"pem": ..., "fingerprint": ...} + pub_key_result = pl_info["public_key_result"] + resource["result_encryption_public_key"] = { + "pem": pub_key_result["key"], + "fingerprint": pub_key_result["id"], + } if self.config.datadog_host_override: resource["datadog_host_override"] = self.config.datadog_host_override @@ -105,8 +104,9 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: # Save PL config to file for later use running the PL pl_config = { "publicKeysByMainDC": resp.get("publicKeysByMainDC"), - "datadogHostOverride": self.config.datadog_host_override, } + if self.config.datadog_host_override: + pl_config["datadogHostOverride"] = self.config.datadog_host_override self._save_pl_config(pl.get("name", _id), pl_config) return _id, pl From 4d4cda3a823ff1974e3dca215e389b50083a2c8f Mon Sep 17 00:00:00 2001 From: Malak El Kouri Date: Wed, 22 Apr 2026 10:20:35 +0200 Subject: [PATCH 3/3] fix reset --- datadog_sync/utils/resources_handler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datadog_sync/utils/resources_handler.py b/datadog_sync/utils/resources_handler.py index 07930cb63..a9695d693 100644 --- a/datadog_sync/utils/resources_handler.py +++ b/datadog_sync/utils/resources_handler.py @@ -81,6 +81,9 @@ async def init_async(self) -> None: self.worker: Workers = Workers(self.config) async def reset(self) -> None: + # Save existing destination state — only contains resources managed by sync-cli + managed_destination = self.config.state._data.destination + if self.config.backup_before_reset: await self.import_resources() else: @@ -89,8 +92,9 @@ async def reset(self) -> None: sleep(5) await self.import_resources_without_saving() - # move the import data from source to destination - self.config.state._data.destination = self.config.state._data.source + # Restore the original destination state so only managed resources are deleted, + # not all resources fetched from the destination API during backup. + self.config.state._data.destination = managed_destination for resource_type in self.config.resources_arg: resources = {}