chore: add provenance and candidates to phone sampling data

.gitattributes

@@ -0,0 +1,4 @@
+# Preserve bytes used by the person-data provenance checks.
+scripts/person_data/*.py text eol=lf
+scripts/person_data/*.json text eol=lf
+scripts/person_data/*.csv text eol=lf

.github/workflows/ci.yml

@@ -96,7 +96,7 @@ jobs:
       - name: Run engine tests
         run: |
           uv run --with pytest --with pytest-asyncio --with pytest-httpx --with pytest-env \
-            pytest packages/data-designer-engine/tests
+            pytest packages/data-designer-engine/tests scripts/person_data
 
   test-interface:
     name: Test Interface (Python ${{ matrix.python-version }} on ${{ matrix.os }})

Makefile

@@ -27,7 +27,7 @@ ALL_PKG_PATHS := packages/ scripts/ tests_e2e/
 
 # Test directories
 CONFIG_TESTS := $(CONFIG_PKG)/tests
-ENGINE_TESTS := $(ENGINE_PKG)/tests
+ENGINE_TESTS := $(ENGINE_PKG)/tests scripts/person_data
 SLURM_TESTS := $(SLURM_PKG)/tests
 INTERFACE_TESTS := $(INTERFACE_PKG)/tests
 

packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE

@@ -0,0 +1,33 @@
+zip_area_code_map.parquet
+
+Population and geography: U.S. Census Bureau, 2023 ACS five-year table B01003,
+2023 national ZCTA Gazetteer, and 2023 TIGER/Line state shapefiles.
+U.S. government works are public domain in the United States. TIGER/Line
+metadata permits use in products/publications with Census source acknowledgment.
+TIGER/Line is a registered trademark of the U.S. Census Bureau. This product is
+not endorsed or certified by the Census Bureau. Boundaries are statistical
+representations, not legal land descriptions.
+https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2023/TGRSHP2023_TechDoc.pdf
+
+Locality, postal-code, and dialing-code statements: Wikidata contributors,
+structured-data snapshots retrieved 2026-11-06, CC0 1.0 Universal.
+CC0 permits copying, modification, and redistribution, including commercially,
+without attribution or share-alike requirements.
+https://www.wikidata.org/wiki/Wikidata:Licensing
+https://creativecommons.org/publicdomain/zero/1.0/
+
+Transformation: normalize locality statements, remove curated exclusions,
+prefer nearby same-state literal postal-code associations, and otherwise use
+the nearest same-state locality point. Retain the candidate codes at that point.
+Sample ZIPs by Census population, then choose uniformly among their candidates.
+Candidates are locality-level approximations, not exact telephone boundaries or
+measured telephone market shares. Each ZIP's population weight is stored once.
+
+NANPA's 2026-11-06 NPA report was consulted only as a screening reference.
+No explicit open-data license was established for that report. It is not
+redistributed and is not required to rebuild from the pinned snapshots.
+
+Source snapshots, queries, exclusions, and permissions (immutable input revision):
+https://github.com/NVIDIA-NeMo/DataDesigner/tree/eae83186b32d7c62ee880c6f538bda46a6831c72/scripts/person_data
+The parquet key-value metadata contains the source manifest, its SHA-256,
+the rebuild script SHA-256, and the sampling policy under data_designer.provenance.

packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/phone_number.py

@@ -13,45 +13,46 @@
 
 
 @functools.lru_cache(maxsize=1)
-def _load_zip_area_code_data() -> tuple[dict[str, str], dict[str, int]]:
+def _load_zip_area_code_data() -> tuple[dict[str, tuple[str, ...]], dict[str, float]]:
     """Load and cache the ZIP-to-area-code mapping from the parquet asset.
 
     Returns:
         A tuple of (area_code_map, population_map).
     """
     data = lazy.pd.read_parquet(Path(__file__).parent / "assets" / "zip_area_code_map.parquet")
-    area_code_map = dict(zip(data["zipcode"], data["area_code"]))
+    candidates = data["area_codes"] if "area_codes" in data else data["area_code"].map(lambda value: [value])
+    area_code_map = dict(zip(data["zipcode"], (tuple(str(code) for code in values) for values in candidates)))
     population_map = dict(zip(data["zipcode"], data["count"]))
     return area_code_map, population_map
 
 
 def get_area_code(zip_prefix: str | None = None) -> str:
     """
-    Sample an area code for the given ZIP code prefix, population-weighted.
+    Sample a population-weighted ZIP, then one of its area-code candidates uniformly.
 
     Args:
         zip_prefix: The prefix of a ZIP code, 5 digits or fewer. If None, sample from all ZIP codes.
 
     Returns:
-        A sampled area code matching the prefix, population-weighted.
+        An area-code candidate for a matching ZIP.
     """
     area_code_map, population_map = _load_zip_area_code_data()
 
     if zip_prefix is None:
         zipcodes, weights = zip(*population_map.items())
         zipcode = random.choices(zipcodes, weights=weights, k=1)[0]
-        return str(area_code_map[zipcode])
+        return random.choice(area_code_map[zipcode])
     if len(zip_prefix) == 5:
         try:
-            return str(area_code_map[zip_prefix])
+            return random.choice(area_code_map[zip_prefix])
         except KeyError:
             raise ValueError(f"ZIP code {zip_prefix} not found.")
     matching_zipcodes = [[z, c] for z, c in population_map.items() if z.startswith(zip_prefix)]
     zipcodes, weights = zip(*matching_zipcodes)
     if not zipcodes:
         raise ValueError(f"No ZIP codes found with prefix {zip_prefix}.")
     zipcode = random.choices(zipcodes, weights=weights, k=1)[0]
-    return str(area_code_map[zipcode])
+    return random.choice(area_code_map[zipcode])
 
 
 class PhoneNumber(BaseModel):
@@ -113,9 +114,11 @@ def format(self, style: str = "dash") -> str:
 
     @classmethod
     def from_area_code(cls, area_code: str) -> "PhoneNumber":
-        prefix = str(random.randint(200, 1000))
-        line_number = str(random.randint(0, 10000)).zfill(4)
-        return PhoneNumber(area_code=area_code, prefix=prefix, line_number=line_number)
+        prefix = random.randint(200, 999)
+        while prefix % 100 == 11:
+            prefix = random.randint(200, 999)
+        line_number = str(random.randint(0, 9999)).zfill(4)
+        return PhoneNumber(area_code=area_code, prefix=str(prefix), line_number=line_number)
 
     @classmethod
     def from_zip_prefix(cls, zip_prefix: str) -> "PhoneNumber":

packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import annotations
+
 from unittest.mock import patch
 
 import pytest
@@ -63,13 +65,30 @@ def test_from_area_code():
         assert phone.line_number == "1234"
 
 
+@pytest.mark.parametrize(
+    ("upper_bound", "expected"),
+    [(False, "202-200-0000"), (True, "202-999-9999")],
+)
+def test_from_area_code_boundaries(upper_bound: bool, expected: str) -> None:
+    with patch("random.randint", side_effect=lambda low, high: high if upper_bound else low):
+        phone = PhoneNumber.from_area_code("202")
+    assert str(phone) == expected
+
+
+@pytest.mark.parametrize("reserved_prefix", range(211, 912, 100))
+def test_from_area_code_skips_n11_exchanges(reserved_prefix: int) -> None:
+    with patch("random.randint", side_effect=[reserved_prefix, 500, 42]):
+        phone = PhoneNumber.from_area_code("202")
+    assert str(phone) == "202-500-0042"
+
+
 def test_str_and_repr_representations(stub_sample_phone_number):
     assert str(stub_sample_phone_number) == "555-123-4567"
     assert repr(stub_sample_phone_number) == "PhoneNumber(555-123-4567)"
 
 
 def test_get_area_code_exact_zipcode():
-    with patch(_LOAD_TARGET, return_value=({"12345": "555"}, {"12345": 100})):
+    with patch(_LOAD_TARGET, return_value=({"12345": ("555",)}, {"12345": 100})):
         result = get_area_code("12345")
         assert result == "555"
 
@@ -83,10 +102,27 @@ def test_get_area_code_exact_zipcode_not_found():
 def test_get_area_code_prefix_match():
     with patch(
         _LOAD_TARGET,
-        return_value=({"12345": "555", "12346": "666"}, {"12345": 100, "12346": 200}),
+        return_value=({"12345": ("555",), "12346": ("666",)}, {"12345": 100, "12346": 200}),
     ):
         with patch("random.choices") as mock_choices:
             mock_choices.return_value = ["12345"]
 
             result = get_area_code("123")
             assert result == "555"
+
+
+@pytest.mark.parametrize("prefix", [None, "123", "12345"])
+def test_get_area_code_samples_all_candidates_without_changing_zip_weights(prefix: str | None) -> None:
+    area_codes = {"12345": ("555", "777"), "12346": ("888",)}
+    population = {"12345": 100, "12346": 200}
+    with (
+        patch(_LOAD_TARGET, return_value=(area_codes, population)),
+        patch("random.choices", return_value=["12345"]) as select_zip,
+        patch("random.choice", return_value="777") as select_code,
+    ):
+        assert get_area_code(prefix) == "777"
+    select_code.assert_called_once_with(("555", "777"))
+    if prefix == "12345":
+        select_zip.assert_not_called()
+    else:
+        select_zip.assert_called_once_with(("12345", "12346"), weights=(100, 200), k=1)

packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py

@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+from unittest.mock import patch
+
+import numpy as np
+import pandas as pd
+import pyarrow.parquet as pq
+import pytest
+
+from data_designer.engine.sampling_gen.entities import phone_number
+from data_designer.engine.sampling_gen.entities.phone_number import get_area_code
+
+ROOT = Path(__file__).parents[6]
+ASSET = Path(phone_number.__file__).parent / "assets/zip_area_code_map.parquet"
+PROVENANCE = ROOT / "scripts/person_data"
+
+
+@pytest.fixture(scope="module")
+def phone_data() -> pd.DataFrame:
+    return pd.read_parquet(ASSET)
+
+
+def test_phone_data_provenance() -> None:
+    report = json.loads((PROVENANCE / "build_report.json").read_text())
+    assert hashlib.sha256(ASSET.read_bytes()).hexdigest() == report["output_sha256"]
+    lineage = json.loads(pq.read_schema(ASSET).metadata[b"data_designer.provenance"])
+    manifest_bytes = (PROVENANCE / "sources.json").read_bytes()
+    assert lineage["format_version"] == 2
+    assert lineage["candidate_selection"] == "population_weighted_zip_then_uniform_area_code"
+    assert lineage["source_manifest"] == json.loads(manifest_bytes)
+    assert lineage["source_manifest_sha256"] == hashlib.sha256(manifest_bytes).hexdigest()
+    assert (
+        lineage["source_manifest"]["excluded_localities_sha256"]
+        == hashlib.sha256((PROVENANCE / "excluded_localities.csv").read_bytes()).hexdigest()
+    )
+    assert lineage["rebuild_script_sha256"] == hashlib.sha256((PROVENANCE / "rebuild.py").read_bytes()).hexdigest()
+    assert re.fullmatch(
+        r"https://github.com/NVIDIA-NeMo/DataDesigner/tree/[0-9a-f]{40}/scripts/person_data",
+        lineage["source_revision_url"],
+    )
+    assert lineage["source_revision_url"] in ASSET.with_suffix(".NOTICE").read_text()
+
+
+def test_phone_data_area_code_reconciliation(phone_data: pd.DataFrame) -> None:
+    report = json.loads((PROVENANCE / "build_report.json").read_text())["area_code_coverage"]
+    csv = (PROVENANCE / "area_code_reconciliation.csv").read_bytes()
+    assert hashlib.sha256(csv).hexdigest() == report["reconciliation_sha256"]
+    coverage = pd.read_csv(PROVENANCE / "area_code_reconciliation.csv").set_index("area_code")
+    assert len(coverage) == report["input_codes"]
+    assignments = phone_data[["zipcode", "area_codes", "count"]].copy()
+    assignments["count"] /= assignments.area_codes.str.len()
+    assignments = assignments.explode("area_codes").rename(columns={"area_codes": "area_code"})
+    assert set(assignments.area_code) <= set(coverage.index)
+    expected = assignments.groupby("area_code").agg(
+        output_zipcodes=("zipcode", "size"), output_population=("count", "sum")
+    )
+    pd.testing.assert_frame_equal(coverage.loc[expected.index, expected.columns], expected)
+    assert np.isclose(coverage.output_population.sum(), phone_data["count"].sum())
+    assert (coverage.loc[coverage.status != "selected", "output_zipcodes"] == 0).all()
+
+
+def test_phone_data_coverage_and_weights(phone_data: pd.DataFrame) -> None:
+    assert list(phone_data.columns) == ["zipcode", "area_code", "count", "area_codes"]
+    assert len(phone_data) == 33772
+    assert phone_data.zipcode.is_unique
+    assert phone_data.zipcode.str.fullmatch(r"[0-9]{5}").all()
+    assert phone_data.area_code.dtype == np.dtype("int64")
+    assert phone_data.area_code.between(200, 999).all()
+    assert phone_data["count"].dtype == np.dtype("float64")
+    assert np.isfinite(phone_data["count"]).all()
+    assert phone_data["count"].ge(0).all()
+    assert phone_data["count"].sum() == 335555510.0
+    for row in phone_data.itertuples():
+        assert list(row.area_codes) == sorted(set(row.area_codes))
+        assert row.area_code in row.area_codes
+
+
+def test_all_packaged_zip_codes_resolve(phone_data: pd.DataFrame) -> None:
+    for row in phone_data.itertuples():
+        assert int(get_area_code(row.zipcode)) in row.area_codes
+
+
+@pytest.mark.parametrize(
+    ("zipcode", "expected"),
+    [("00901", "787"), ("02108", "617"), ("06360", "860"), ("10001", "212"), ("90210", "310"), ("98101", "206")],
+)
+def test_representative_regional_area_codes(zipcode: str, expected: str) -> None:
+    with patch("random.choice", side_effect=lambda values: values[0]):
+        assert get_area_code(zipcode) == expected
+
+
+@pytest.mark.parametrize(
+    ("zipcode", "expected"), [("60629", {312, 773, 872}), ("10001", {212, 332, 646, 917}), ("78201", {210, 726})]
+)
+def test_packaged_zip_retains_alternatives(phone_data: pd.DataFrame, zipcode: str, expected: set[int]) -> None:
+    candidates = set(phone_data.set_index("zipcode").loc[zipcode, "area_codes"])
+    assert expected <= candidates
+    for code in expected:
+        with patch("random.choice", side_effect=lambda values: str(code) if str(code) in values else values[0]):
+            assert get_area_code(zipcode) == str(code)
+
+
+@pytest.mark.parametrize("candidates", [None, [212, 646]])
+def test_phone_sampler_reads_legacy_and_candidate_assets(candidates: list[int] | None) -> None:
+    data = pd.DataFrame({"zipcode": ["10001"], "area_code": [212], "count": [100.0]})
+    if candidates is not None:
+        data["area_codes"] = [candidates]
+    phone_number._load_zip_area_code_data.cache_clear()
+    try:
+        with (
+            patch.object(phone_number.lazy.pd, "read_parquet", return_value=data),
+            patch("random.choice", side_effect=lambda values: values[-1]),
+        ):
+            assert get_area_code("10001") == ("212" if candidates is None else "646")
+    finally:
+        phone_number._load_zip_area_code_data.cache_clear()