diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..20721e8d2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Preserve bytes used by the person-data provenance checks. +scripts/person_data/*.py text eol=lf +scripts/person_data/*.json text eol=lf +scripts/person_data/*.csv text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d89fe807b..69b39575f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: - name: Run engine tests run: | uv run --with pytest --with pytest-asyncio --with pytest-httpx --with pytest-env \ - pytest packages/data-designer-engine/tests + pytest packages/data-designer-engine/tests scripts/person_data test-interface: name: Test Interface (Python ${{ matrix.python-version }} on ${{ matrix.os }}) diff --git a/Makefile b/Makefile index 73a1cd873..b6e6007c7 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ ALL_PKG_PATHS := packages/ scripts/ tests_e2e/ # Test directories CONFIG_TESTS := $(CONFIG_PKG)/tests -ENGINE_TESTS := $(ENGINE_PKG)/tests +ENGINE_TESTS := $(ENGINE_PKG)/tests scripts/person_data SLURM_TESTS := $(SLURM_PKG)/tests INTERFACE_TESTS := $(INTERFACE_PKG)/tests diff --git a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE new file mode 100644 index 000000000..aca1a9dc6 --- /dev/null +++ b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE @@ -0,0 +1,33 @@ +zip_area_code_map.parquet + +Population and geography: U.S. Census Bureau, 2023 ACS five-year table B01003, +2023 national ZCTA Gazetteer, and 2023 TIGER/Line state shapefiles. +U.S. government works are public domain in the United States. TIGER/Line +metadata permits use in products/publications with Census source acknowledgment. +TIGER/Line is a registered trademark of the U.S. Census Bureau. This product is +not endorsed or certified by the Census Bureau. Boundaries are statistical +representations, not legal land descriptions. +https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2023/TGRSHP2023_TechDoc.pdf + +Locality, postal-code, and dialing-code statements: Wikidata contributors, +structured-data snapshots retrieved 2026-11-06, CC0 1.0 Universal. +CC0 permits copying, modification, and redistribution, including commercially, +without attribution or share-alike requirements. +https://www.wikidata.org/wiki/Wikidata:Licensing +https://creativecommons.org/publicdomain/zero/1.0/ + +Transformation: normalize locality statements, remove curated exclusions, +prefer nearby same-state literal postal-code associations, and otherwise use +the nearest same-state locality point. Retain the candidate codes at that point. +Sample ZIPs by Census population, then choose uniformly among their candidates. +Candidates are locality-level approximations, not exact telephone boundaries or +measured telephone market shares. Each ZIP's population weight is stored once. + +NANPA's 2026-11-06 NPA report was consulted only as a screening reference. +No explicit open-data license was established for that report. It is not +redistributed and is not required to rebuild from the pinned snapshots. + +Source snapshots, queries, exclusions, and permissions (immutable input revision): +https://github.com/NVIDIA-NeMo/DataDesigner/tree/eae83186b32d7c62ee880c6f538bda46a6831c72/scripts/person_data +The parquet key-value metadata contains the source manifest, its SHA-256, +the rebuild script SHA-256, and the sampling policy under data_designer.provenance. diff --git a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet index d7c682715..1388ab213 100644 Binary files a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet and b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet differ diff --git a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/phone_number.py b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/phone_number.py index 6041c7e88..b06859630 100644 --- a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/phone_number.py +++ b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/phone_number.py @@ -13,37 +13,38 @@ @functools.lru_cache(maxsize=1) -def _load_zip_area_code_data() -> tuple[dict[str, str], dict[str, int]]: +def _load_zip_area_code_data() -> tuple[dict[str, tuple[str, ...]], dict[str, float]]: """Load and cache the ZIP-to-area-code mapping from the parquet asset. Returns: A tuple of (area_code_map, population_map). """ data = lazy.pd.read_parquet(Path(__file__).parent / "assets" / "zip_area_code_map.parquet") - area_code_map = dict(zip(data["zipcode"], data["area_code"])) + candidates = data["area_codes"] if "area_codes" in data else data["area_code"].map(lambda value: [value]) + area_code_map = dict(zip(data["zipcode"], (tuple(str(code) for code in values) for values in candidates))) population_map = dict(zip(data["zipcode"], data["count"])) return area_code_map, population_map def get_area_code(zip_prefix: str | None = None) -> str: """ - Sample an area code for the given ZIP code prefix, population-weighted. + Sample a population-weighted ZIP, then one of its area-code candidates uniformly. Args: zip_prefix: The prefix of a ZIP code, 5 digits or fewer. If None, sample from all ZIP codes. Returns: - A sampled area code matching the prefix, population-weighted. + An area-code candidate for a matching ZIP. """ area_code_map, population_map = _load_zip_area_code_data() if zip_prefix is None: zipcodes, weights = zip(*population_map.items()) zipcode = random.choices(zipcodes, weights=weights, k=1)[0] - return str(area_code_map[zipcode]) + return random.choice(area_code_map[zipcode]) if len(zip_prefix) == 5: try: - return str(area_code_map[zip_prefix]) + return random.choice(area_code_map[zip_prefix]) except KeyError: raise ValueError(f"ZIP code {zip_prefix} not found.") matching_zipcodes = [[z, c] for z, c in population_map.items() if z.startswith(zip_prefix)] @@ -51,7 +52,7 @@ def get_area_code(zip_prefix: str | None = None) -> str: if not zipcodes: raise ValueError(f"No ZIP codes found with prefix {zip_prefix}.") zipcode = random.choices(zipcodes, weights=weights, k=1)[0] - return str(area_code_map[zipcode]) + return random.choice(area_code_map[zipcode]) class PhoneNumber(BaseModel): @@ -113,9 +114,11 @@ def format(self, style: str = "dash") -> str: @classmethod def from_area_code(cls, area_code: str) -> "PhoneNumber": - prefix = str(random.randint(200, 1000)) - line_number = str(random.randint(0, 10000)).zfill(4) - return PhoneNumber(area_code=area_code, prefix=prefix, line_number=line_number) + prefix = random.randint(200, 999) + while prefix % 100 == 11: + prefix = random.randint(200, 999) + line_number = str(random.randint(0, 9999)).zfill(4) + return PhoneNumber(area_code=area_code, prefix=str(prefix), line_number=line_number) @classmethod def from_zip_prefix(cls, zip_prefix: str) -> "PhoneNumber": diff --git a/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number.py b/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number.py index 05cb800c9..bf3f6348f 100644 --- a/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number.py +++ b/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from unittest.mock import patch import pytest @@ -63,13 +65,30 @@ def test_from_area_code(): assert phone.line_number == "1234" +@pytest.mark.parametrize( + ("upper_bound", "expected"), + [(False, "202-200-0000"), (True, "202-999-9999")], +) +def test_from_area_code_boundaries(upper_bound: bool, expected: str) -> None: + with patch("random.randint", side_effect=lambda low, high: high if upper_bound else low): + phone = PhoneNumber.from_area_code("202") + assert str(phone) == expected + + +@pytest.mark.parametrize("reserved_prefix", range(211, 912, 100)) +def test_from_area_code_skips_n11_exchanges(reserved_prefix: int) -> None: + with patch("random.randint", side_effect=[reserved_prefix, 500, 42]): + phone = PhoneNumber.from_area_code("202") + assert str(phone) == "202-500-0042" + + def test_str_and_repr_representations(stub_sample_phone_number): assert str(stub_sample_phone_number) == "555-123-4567" assert repr(stub_sample_phone_number) == "PhoneNumber(555-123-4567)" def test_get_area_code_exact_zipcode(): - with patch(_LOAD_TARGET, return_value=({"12345": "555"}, {"12345": 100})): + with patch(_LOAD_TARGET, return_value=({"12345": ("555",)}, {"12345": 100})): result = get_area_code("12345") assert result == "555" @@ -83,10 +102,27 @@ def test_get_area_code_exact_zipcode_not_found(): def test_get_area_code_prefix_match(): with patch( _LOAD_TARGET, - return_value=({"12345": "555", "12346": "666"}, {"12345": 100, "12346": 200}), + return_value=({"12345": ("555",), "12346": ("666",)}, {"12345": 100, "12346": 200}), ): with patch("random.choices") as mock_choices: mock_choices.return_value = ["12345"] result = get_area_code("123") assert result == "555" + + +@pytest.mark.parametrize("prefix", [None, "123", "12345"]) +def test_get_area_code_samples_all_candidates_without_changing_zip_weights(prefix: str | None) -> None: + area_codes = {"12345": ("555", "777"), "12346": ("888",)} + population = {"12345": 100, "12346": 200} + with ( + patch(_LOAD_TARGET, return_value=(area_codes, population)), + patch("random.choices", return_value=["12345"]) as select_zip, + patch("random.choice", return_value="777") as select_code, + ): + assert get_area_code(prefix) == "777" + select_code.assert_called_once_with(("555", "777")) + if prefix == "12345": + select_zip.assert_not_called() + else: + select_zip.assert_called_once_with(("12345", "12346"), weights=(100, 200), k=1) diff --git a/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py b/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py new file mode 100644 index 000000000..1b8534720 --- /dev/null +++ b/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +import pytest + +from data_designer.engine.sampling_gen.entities import phone_number +from data_designer.engine.sampling_gen.entities.phone_number import get_area_code + +ROOT = Path(__file__).parents[6] +ASSET = Path(phone_number.__file__).parent / "assets/zip_area_code_map.parquet" +PROVENANCE = ROOT / "scripts/person_data" + + +@pytest.fixture(scope="module") +def phone_data() -> pd.DataFrame: + return pd.read_parquet(ASSET) + + +def test_phone_data_provenance() -> None: + report = json.loads((PROVENANCE / "build_report.json").read_text()) + assert hashlib.sha256(ASSET.read_bytes()).hexdigest() == report["output_sha256"] + lineage = json.loads(pq.read_schema(ASSET).metadata[b"data_designer.provenance"]) + manifest_bytes = (PROVENANCE / "sources.json").read_bytes() + assert lineage["format_version"] == 2 + assert lineage["candidate_selection"] == "population_weighted_zip_then_uniform_area_code" + assert lineage["source_manifest"] == json.loads(manifest_bytes) + assert lineage["source_manifest_sha256"] == hashlib.sha256(manifest_bytes).hexdigest() + assert ( + lineage["source_manifest"]["excluded_localities_sha256"] + == hashlib.sha256((PROVENANCE / "excluded_localities.csv").read_bytes()).hexdigest() + ) + assert lineage["rebuild_script_sha256"] == hashlib.sha256((PROVENANCE / "rebuild.py").read_bytes()).hexdigest() + assert re.fullmatch( + r"https://github.com/NVIDIA-NeMo/DataDesigner/tree/[0-9a-f]{40}/scripts/person_data", + lineage["source_revision_url"], + ) + assert lineage["source_revision_url"] in ASSET.with_suffix(".NOTICE").read_text() + + +def test_phone_data_area_code_reconciliation(phone_data: pd.DataFrame) -> None: + report = json.loads((PROVENANCE / "build_report.json").read_text())["area_code_coverage"] + csv = (PROVENANCE / "area_code_reconciliation.csv").read_bytes() + assert hashlib.sha256(csv).hexdigest() == report["reconciliation_sha256"] + coverage = pd.read_csv(PROVENANCE / "area_code_reconciliation.csv").set_index("area_code") + assert len(coverage) == report["input_codes"] + assignments = phone_data[["zipcode", "area_codes", "count"]].copy() + assignments["count"] /= assignments.area_codes.str.len() + assignments = assignments.explode("area_codes").rename(columns={"area_codes": "area_code"}) + assert set(assignments.area_code) <= set(coverage.index) + expected = assignments.groupby("area_code").agg( + output_zipcodes=("zipcode", "size"), output_population=("count", "sum") + ) + pd.testing.assert_frame_equal(coverage.loc[expected.index, expected.columns], expected) + assert np.isclose(coverage.output_population.sum(), phone_data["count"].sum()) + assert (coverage.loc[coverage.status != "selected", "output_zipcodes"] == 0).all() + + +def test_phone_data_coverage_and_weights(phone_data: pd.DataFrame) -> None: + assert list(phone_data.columns) == ["zipcode", "area_code", "count", "area_codes"] + assert len(phone_data) == 33772 + assert phone_data.zipcode.is_unique + assert phone_data.zipcode.str.fullmatch(r"[0-9]{5}").all() + assert phone_data.area_code.dtype == np.dtype("int64") + assert phone_data.area_code.between(200, 999).all() + assert phone_data["count"].dtype == np.dtype("float64") + assert np.isfinite(phone_data["count"]).all() + assert phone_data["count"].ge(0).all() + assert phone_data["count"].sum() == 335555510.0 + for row in phone_data.itertuples(): + assert list(row.area_codes) == sorted(set(row.area_codes)) + assert row.area_code in row.area_codes + + +def test_all_packaged_zip_codes_resolve(phone_data: pd.DataFrame) -> None: + for row in phone_data.itertuples(): + assert int(get_area_code(row.zipcode)) in row.area_codes + + +@pytest.mark.parametrize( + ("zipcode", "expected"), + [("00901", "787"), ("02108", "617"), ("06360", "860"), ("10001", "212"), ("90210", "310"), ("98101", "206")], +) +def test_representative_regional_area_codes(zipcode: str, expected: str) -> None: + with patch("random.choice", side_effect=lambda values: values[0]): + assert get_area_code(zipcode) == expected + + +@pytest.mark.parametrize( + ("zipcode", "expected"), [("60629", {312, 773, 872}), ("10001", {212, 332, 646, 917}), ("78201", {210, 726})] +) +def test_packaged_zip_retains_alternatives(phone_data: pd.DataFrame, zipcode: str, expected: set[int]) -> None: + candidates = set(phone_data.set_index("zipcode").loc[zipcode, "area_codes"]) + assert expected <= candidates + for code in expected: + with patch("random.choice", side_effect=lambda values: str(code) if str(code) in values else values[0]): + assert get_area_code(zipcode) == str(code) + + +@pytest.mark.parametrize("candidates", [None, [212, 646]]) +def test_phone_sampler_reads_legacy_and_candidate_assets(candidates: list[int] | None) -> None: + data = pd.DataFrame({"zipcode": ["10001"], "area_code": [212], "count": [100.0]}) + if candidates is not None: + data["area_codes"] = [candidates] + phone_number._load_zip_area_code_data.cache_clear() + try: + with ( + patch.object(phone_number.lazy.pd, "read_parquet", return_value=data), + patch("random.choice", side_effect=lambda values: values[-1]), + ): + assert get_area_code("10001") == ("212" if candidates is None else "646") + finally: + phone_number._load_zip_area_code_data.cache_clear() diff --git a/scripts/person_data/README.md b/scripts/person_data/README.md new file mode 100644 index 000000000..91463460f --- /dev/null +++ b/scripts/person_data/README.md @@ -0,0 +1,157 @@ +# ZIP/area-code lookup provenance + +This directory rebuilds the bundled `zip_area_code_map.parquet` used by the US +person and phone samplers. It contains 33,772 Census ZIP Code Tabulation Areas +(ZCTAs), including Puerto Rico. ZCTAs approximate postal ZIP areas. Each row has +`zipcode` (five-character string), `count` (float64 population estimate), and +`area_codes` (a sorted, unique list of int64 candidates). The existing int64 +`area_code` column remains as a deterministic representative for older readers; +the sampler uses `area_codes`. + +## Sources and permissions + +`sources.json` records exact download URLs, retrieval dates, and SHA-256 hashes +of the uncompressed inputs. All source snapshots were retrieved on 2026-11-06. + +| Source | Fields used | Published terms and reuse | +| --- | --- | --- | +| U.S. Census Bureau, 2023 ACS five-year table B01003 | ZCTA identifiers and population estimates | U.S. government work, public domain in the United States; may be reproduced. Source acknowledgment is provided below. | +| U.S. Census Bureau, 2023 national ZCTA Gazetteer | ZCTA representative coordinates | Same Census public-domain status. | +| U.S. Census Bureau, 2023 TIGER/Line state boundaries | State/territory containing each representative point | Bundled metadata states that the products are not copyrighted and are free to use in products/publications with Census acknowledgment. | +| Wikidata structured statements, two pinned query results | Item identifiers, coordinates, dialing codes (P473), postal codes (P281) | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/): public-domain dedication with fallback permission to copy, modify, and redistribute, including commercially, without attribution or share-alike requirements. | + +The Census Bureau's [2023 TIGER/Line technical documentation, sections 1.1-1.2](https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2023/TGRSHP2023_TechDoc.pdf) +explains government-work copyright status, source citation, and trademark limits. +[Wikidata's licensing policy](https://www.wikidata.org/wiki/Wikidata:Licensing) +applies CC0 to structured data. Wikipedia articles and Wikidata project-page text +are not used. These data permissions impose no noncommercial or share-alike +restriction on the generated lookup. The repository's Apache-2.0 license covers +the rebuild code; it does not replace the source-data terms. + +The [NANPA NPA database](https://www.nanpa.com/reports/npa-reports) was consulted +on 2026-11-06 to screen locality associations against country, in-service status, +and state. No explicit open-data license was established for that report, so +it is **not described as CC0 or public domain**. The report is not redistributed +and is not an input required by the rebuild. Its URL, date, and checksum are +recorded as a validation reference. `excluded_localities.csv` records the 92 +rejected Wikidata rows using Wikidata fields and Census-derived states, rather +than copying the NANPA database. Regeneration reproduces this pinned curation; +it does not recheck current numbering assignments. + +No third-party GitHub ZIP/area-code compilation is used. + +## Transformation + +1. Keep ACS ZCTA population rows and join Gazetteer representative coordinates. +2. Assign states using the full Census state polygons. A reader for the pinned + Polygon/DBF format and a NumPy point-in-polygon calculation handle multipart + polygons, holes, and boundaries without additional geometry packages. Normalize + Wikidata three-digit dialing codes and coordinates, then remove the exclusions. +3. Prefer literal five-digit Wikidata postal-code associations when the locality + lies in the same state and within 50 km of the ZCTA point. Postal-code ranges + and wildcards are not expanded. +4. Select the closest qualifying literal-postcode locality point, or otherwise + the nearest same-state locality point by spherical distance. Retain codes from + all accepted same-state localities at that point, including those without a + literal postcode statement. + Ties between points use the existing deterministic ordering; code alternatives + at the chosen point are deduplicated, not discarded. +5. Write one row per ZCTA with its candidate list and unchanged population weight. + The representative `area_code` still follows the previous ordering. +6. At runtime, sample a ZIP with its Census population weight, then choose uniformly + among its candidates. An exact-ZIP request skips the ZIP draw. Having more + candidates does not increase a ZIP's probability of being sampled. + +The checked-in build has 18,749 explicit postal-code associations and 15,023 +nearest-locality fallbacks. All ZIP identifiers, representative codes, and population +weights match the initial rebuild in this PR. Candidate lists contain 37,239 +ZIP/code associations, and 2,322 ZIPs have more than one candidate. Agreement with +the original repository asset is not an accuracy measure. + +This is a regional approximation for synthetic data, not a telephone service-area +or address lookup. A city's candidate list can contain both overlays and geographic +splits. Candidates are plausible at the chosen locality point, not verified for +each postal address or exact ZIP boundary. Uniform candidate selection does not +estimate actual telephone market shares. There are 348 fallback ZIPs +more than 50 km from their donor locality; the largest distance is about 231 km. +State-level screening does not establish ZIP-level accuracy. Population estimates +and numbering statements reflect the pinned source dates. + +The code-level reconciliation counts 386 distinct three-digit source values, +358 codes with retained localities, 355 with localities in the covered ZCTA +states/territories, and 333 represented in candidate lists. Retaining alternatives +recovers 56 codes previously discarded, including 332, 726, and Chicago's 773 and +872. There are still 22 eligible but unselected codes. They have no selected +locality point under this method. These omissions are reported, not treated as +evidence that the codes are invalid. + +## Rebuild + +From the repository root, using the development tools required by the project: + +```sh +make install-dev +.venv/bin/python scripts/person_data/rebuild.py +cp .cache/person_data/report/build_report.json scripts/person_data/build_report.json +cp .cache/person_data/report/area_code_reconciliation.csv scripts/person_data/area_code_reconciliation.csv +.venv/bin/pytest -q scripts/person_data/test_rebuild.py +``` + +The rebuild uses the Python standard library and the project's existing NumPy, +pandas, SciPy, and PyArrow dependencies. It requires no additional build or runtime +packages, native geometry libraries, or downloaded extensions. The state reader +supports the pinned Census Polygon/DBF layout described by the published +[shapefile specification](https://www.esri.com/content/dam/esrisites/sitecore-archive/Files/Pdfs/library/whitepapers/pdfs/shapefile.pdf). +Unsupported layouts are rejected. Rebuild unit tests run with the engine suite +through both `make test-engine` and CI. + +The script verifies every input hash before building. It downloads about 30 MB +of immutable Census inputs into `.cache/person_data` on the first run. The two +compressed Wikidata snapshots are checked in because live query results change. +After populating the cache, `--offline` rebuilds without network access. + +The default output replaces the packaged parquet. For a comparison build: + +```sh +.venv/bin/python scripts/person_data/rebuild.py --offline \ + --output .cache/person_data/rebuilt.parquet \ + --report-dir .cache/person_data/report +``` + +The report directory contains `build_report.json`, `area_code_reconciliation.csv`, +and compressed per-candidate lineage (`mapping_provenance.csv.gz`): ZIP, code, +Wikidata item, method, state, and distance. The reconciliation lists each source +code's input/locality counts, candidate ZIP count, expected population weight, and status: +`selected`, `no_retained_locality`, `outside_zcta_states`, or `not_selected`. +Each ZIP's weight is divided equally between its candidates for this report. +Those expected weights describe the sampler, not measured telephone usage. +The build rejects any output code absent from the pinned input. + +The checked-in report and reconciliation record the committed build. Standard +engine tests compare the packaged parquet to its recorded output hash and verify +the per-code counts and conserved population weights. Parquet byte +hashes can vary with writer versions; compare decoded tables across environments. +The recorded build used Python 3.12, NumPy 2.4.3, pandas 3.0.5, SciPy 1.17.1, +and PyArrow 24.0.0. + +The parquet's `data_designer.provenance` key-value metadata embeds the complete +source manifest (URLs, hashes, permissions, and screening reference), the manifest +SHA-256, the rebuild script SHA-256, and the ZIP/candidate sampling policy. The packaged NOTICE links to the immutable +input revision, so an older wheel retains a stable source reference. Refresh that +link and `SOURCE_REVISION_URL` when adopting a new committed input revision. + +To update sources, rerun the two `.sparql` queries, replace their compressed +snapshots and hashes, review the exclusions against a newly recorded numbering +reference, and update the Census URLs/hashes if changing vintage. Rebuild, +inspect lineage and coverage, and update the build report and data notice. + +## Source acknowledgment + +Population and geography: U.S. Census Bureau, 2023 ACS five-year estimates, +2023 Gazetteer Files, and 2023 TIGER/Line state shapefiles. This product uses +Census Bureau data but is not endorsed or certified by the Census Bureau. +TIGER/Line is a registered trademark of the U.S. Census Bureau. Census boundaries +are statistical representations, not legal land descriptions. + +Locality and dialing-code statements: Wikidata contributors, structured-data +query snapshots retrieved 2026-11-06, CC0 1.0. diff --git a/scripts/person_data/area_code_reconciliation.csv b/scripts/person_data/area_code_reconciliation.csv new file mode 100644 index 000000000..436ba55af --- /dev/null +++ b/scripts/person_data/area_code_reconciliation.csv @@ -0,0 +1,387 @@ +area_code,input_records,retained_locality_records,eligible_locality_records,output_zipcodes,output_population,status +201,30,29,29,98,1695304.0,selected +202,30,30,30,57,672079.0,selected +203,68,68,68,87,1393767.0,selected +205,168,168,168,188,1534246.0,selected +206,41,41,41,42,567041.5,selected +207,852,852,852,427,1377734.0,selected +208,249,247,247,280,1893304.0,selected +209,101,101,101,137,1880553.0,selected +210,9,9,9,69,1156174.0,selected +212,55,55,55,70,638189.25,selected +213,3,3,3,63,377714.5,selected +214,7,7,7,73,850862.6666666666,selected +215,18,18,18,171,2108491.0,selected +216,29,28,28,41,995011.0,selected +217,136,136,136,350,1048108.5,selected +218,232,232,232,292,797314.0,selected +219,72,72,72,68,840925.0,selected +220,1,1,1,4,6160.5,selected +223,4,4,4,61,222153.0,selected +224,4,4,4,13,161519.5,selected +225,44,43,43,89,864103.5,selected +228,29,29,29,27,403017.0,selected +229,109,109,109,128,714067.0,selected +231,48,48,48,136,687894.0,selected +234,2,2,2,25,150196.5,selected +235,1,1,1,2,1246.0,selected +236,1,0,0,0,0.0,no_retained_locality +239,62,62,62,58,1206864.0,selected +240,91,91,91,172,1261527.8,selected +248,30,30,30,73,1374096.75,selected +250,1,0,0,0,0.0,no_retained_locality +251,84,84,84,104,749718.0,selected +252,147,147,147,198,1070963.0,selected +253,62,62,62,58,1269771.0,selected +254,89,89,89,131,1003984.0,selected +256,137,137,137,179,1564460.0,selected +257,1,0,0,0,0.0,no_retained_locality +260,66,66,66,93,732688.0,selected +262,85,85,85,103,1216795.0,selected +267,3,3,3,55,830644.0,selected +269,31,31,31,98,857116.0,selected +270,70,70,70,221,941466.5,selected +272,2,2,2,13,61779.0,selected +274,1,1,1,0,0.0,not_selected +276,51,51,51,125,465373.0,selected +279,1,1,1,0,0.0,not_selected +281,47,47,47,157,3072186.7,selected +283,1,1,1,0,0.0,not_selected +301,121,121,121,231,1811124.8,selected +302,78,77,77,68,1006247.0,selected +303,26,26,26,118,2077404.6666666667,selected +304,217,217,217,738,1705155.0,selected +305,74,74,74,95,2029129.0,selected +307,203,203,203,176,564413.0,selected +308,71,71,71,193,360503.0,selected +309,80,80,80,205,975700.5,selected +310,16,16,16,107,1448462.1666666667,selected +312,4,3,3,56,897075.3333333334,selected +313,18,18,18,49,1161381.0,selected +314,56,56,56,54,1115598.0,selected +315,204,204,204,314,1440713.0,selected +316,26,26,26,58,612536.0,selected +317,53,53,53,82,1827586.0,selected +318,169,168,168,209,1125622.0,selected +319,64,64,64,175,793459.0,selected +320,162,162,162,160,581609.0,selected +321,29,29,29,63,1154504.0,selected +323,16,16,16,87,1252435.1666666667,selected +325,55,54,54,103,461764.0,selected +326,1,1,1,1,284.0,selected +327,1,1,1,0,0.0,not_selected +329,1,1,1,0,0.0,not_selected +330,25,25,25,205,1631919.5,selected +331,1,1,1,15,235152.0,selected +332,1,1,1,56,326286.25,selected +334,150,149,149,185,1205713.0,selected +336,71,71,71,145,1847267.0,selected +337,70,70,70,112,949012.0,selected +339,4,4,4,19,172263.5,selected +340,2,2,0,0,0.0,outside_zcta_states +341,1,1,1,0,0.0,not_selected +346,3,3,3,97,658946.7,selected +347,7,7,7,57,812461.3333333334,selected +349,1,0,0,0,0.0,no_retained_locality +350,1,1,1,1,2253.0,selected +352,120,120,120,126,1677904.0,selected +357,1,1,1,0,0.0,not_selected +360,525,524,524,216,2249084.5,selected +361,62,62,62,116,774012.0,selected +363,6,6,6,19,164028.5,selected +364,7,7,7,27,213848.5,selected +380,1,1,1,0,0.0,not_selected +384,1,0,0,0,0.0,no_retained_locality +385,19,19,19,53,682646.0,selected +386,58,58,58,70,950185.0,selected +393,2,0,0,0,0.0,no_retained_locality +401,43,43,43,81,1095391.0,selected +402,135,133,133,393,1454988.0,selected +404,9,9,9,51,628465.3333333334,selected +405,116,116,116,183,1912177.0,selected +406,184,183,183,369,1105740.0,selected +407,65,65,65,66,1631275.0,selected +408,16,16,16,49,1149360.0,selected +409,45,45,45,67,642668.0,selected +410,55,55,55,249,2113906.6333333333,selected +412,27,27,27,108,797524.6666666666,selected +413,115,115,115,153,1097415.0,selected +414,14,14,14,37,949011.0,selected +415,28,28,28,55,679961.5,selected +417,107,107,107,218,1054259.0,selected +419,35,35,35,280,1765658.0,selected +423,81,81,81,149,1430346.0,selected +424,3,3,3,74,468134.1666666667,selected +425,76,76,76,42,1182789.0,selected +430,4,4,4,6,33844.0,selected +432,33,33,33,60,479969.0,selected +434,46,46,46,140,721434.0,selected +435,189,189,189,215,850174.0,selected +436,1,1,1,3,12930.5,selected +440,26,26,26,74,1127433.5,selected +442,2,2,2,1,0.0,selected +443,6,6,6,73,561859.6333333333,selected +445,2,2,2,5,39226.0,selected +447,2,2,2,11,72250.5,selected +448,36,36,36,14,22591.0,selected +453,1,0,0,0,0.0,no_retained_locality +456,1,0,0,0,0.0,no_retained_locality +457,1,1,1,0,0.0,not_selected +458,550,548,548,119,426747.5,selected +463,1,1,1,0,0.0,not_selected +464,1,1,1,0,0.0,not_selected +469,5,5,5,69,652393.6666666666,selected +470,3,3,3,5,74570.33333333333,selected +475,23,23,23,39,429985.5,selected +478,75,75,75,88,818988.0,selected +479,108,108,108,114,935987.0,selected +480,22,22,22,97,2301780.25,selected +483,1,1,1,0,0.0,not_selected +484,2,2,2,21,157124.83333333334,selected +486,1,0,0,0,0.0,no_retained_locality +501,111,110,110,121,990765.0,selected +502,55,55,55,101,1260079.0,selected +503,418,416,416,155,1482247.0,selected +504,13,13,13,52,689834.5,selected +505,41,41,41,157,1370689.0,selected +507,237,236,236,240,839753.0,selected +508,11,11,11,164,1540428.0,selected +509,462,461,461,248,1862724.0,selected +510,31,31,31,55,1686242.0,selected +512,33,33,33,91,2309709.0,selected +513,18,18,18,93,1606474.0,selected +515,53,53,53,140,892496.0,selected +516,20,20,20,75,1398673.5,selected +517,28,28,28,109,961533.75,selected +518,183,183,183,322,1379116.0,selected +519,1,0,0,0,0.0,no_retained_locality +520,173,173,173,147,1909131.25,selected +530,134,133,133,238,1352133.5,selected +531,2,2,2,25,151362.0,selected +534,1,1,1,33,76022.5,selected +539,11,11,11,76,375014.0,selected +540,76,76,76,274,2094068.0,selected +541,769,767,767,273,1301254.5,selected +551,2,2,2,7,52179.0,selected +557,1,1,1,0,0.0,not_selected +559,98,98,98,111,1777565.0,selected +561,61,61,61,65,2089576.0,selected +562,18,18,18,37,1339643.0,selected +563,46,46,46,135,483055.0,selected +564,8,8,8,48,609463.0,selected +567,1,1,1,0,0.0,not_selected +570,96,96,96,366,1510942.0,selected +571,5,5,5,65,886610.5,selected +572,1,1,1,0,0.0,not_selected +573,168,168,168,315,1196797.0,selected +574,64,64,64,86,722612.0,selected +575,45,44,44,216,746106.0,selected +580,270,270,270,353,1050387.0,selected +582,2,2,2,19,25907.0,selected +585,92,92,92,141,1062260.0,selected +586,14,14,14,40,864477.0,selected +590,1,0,0,0,0.0,no_retained_locality +601,113,113,113,165,1358123.0,selected +602,3,2,2,43,363748.25,selected +603,216,216,216,246,1387500.0,selected +605,378,371,371,375,897979.0,selected +606,97,97,97,364,897133.0,selected +607,138,138,138,206,831766.0,selected +608,360,360,360,318,1749167.0,selected +609,44,43,43,112,1304381.5,selected +610,29,28,28,226,2581691.8333333335,selected +611,2,0,0,0,0.0,no_retained_locality +612,6,5,5,34,797400.0,selected +614,25,25,25,69,1723175.0,selected +615,55,55,55,107,1850092.0,selected +616,17,17,17,52,1029901.0,selected +617,7,7,7,72,1458474.5,selected +618,127,127,127,318,1191996.5,selected +619,27,27,27,73,1578285.0,selected +620,221,220,220,265,609661.0,selected +621,1,1,1,94,616910.2,selected +622,1,0,0,0,0.0,no_retained_locality +623,18,18,18,80,1573589.25,selected +624,1,1,1,1,1046.5,selected +626,35,35,35,42,1403788.0,selected +627,1,0,0,0,0.0,no_retained_locality +628,2,2,2,29,418160.5,selected +629,1,1,1,6,113114.0,selected +630,9,8,8,58,1379779.0,selected +631,23,23,23,110,1525226.0,selected +635,1,0,0,0,0.0,no_retained_locality +636,42,42,42,65,976996.0,selected +639,1,0,0,0,0.0,no_retained_locality +640,1,1,1,9,31807.5,selected +641,72,72,72,250,470730.0,selected +642,1,0,0,0,0.0,no_retained_locality +646,17,17,17,58,333885.25,selected +650,36,36,36,39,954535.0,selected +651,70,70,70,69,1181181.0,selected +655,1,0,0,0,0.0,no_retained_locality +657,1,1,1,4,87538.0,selected +659,1,1,1,0,0.0,not_selected +660,167,167,167,229,434415.0,selected +661,55,55,55,60,1512997.0,selected +662,181,181,181,235,1158482.0,selected +667,4,4,4,54,421944.13333333336,selected +669,3,3,3,32,553094.0,selected +670,2,0,0,0,0.0,no_retained_locality +671,1,1,0,0,0.0,outside_zcta_states +672,1,0,0,0,0.0,no_retained_locality +678,4,4,4,45,420698.6666666667,selected +679,2,2,2,0,0.0,not_selected +680,3,3,3,2,8683.0,selected +681,6,6,6,26,78977.0,selected +682,2,2,2,46,573491.1666666666,selected +684,2,2,0,0,0.0,outside_zcta_states +689,1,1,1,0,0.0,not_selected +701,401,400,400,388,778556.0,selected +702,21,20,20,76,2141300.0,selected +703,7,7,7,79,1293539.5,selected +704,71,71,71,122,2467616.0,selected +706,134,132,132,197,1998223.0,selected +707,118,118,118,151,1473463.0,selected +708,82,82,82,83,1601918.0,selected +712,87,87,87,270,556654.0,selected +713,15,15,15,110,1251121.7,selected +714,9,9,9,44,1712647.0,selected +715,7,7,7,109,644854.5,selected +716,64,64,64,149,1304519.5,selected +717,19,19,19,196,1508027.0,selected +718,13,13,13,121,4335015.333333333,selected +719,119,119,119,168,1148368.0,selected +720,7,6,6,62,1175307.6666666667,selected +722,1,0,0,0,0.0,no_retained_locality +724,39,39,39,353,1515985.6666666667,selected +725,5,4,4,12,152483.0,selected +726,1,1,1,59,927090.0,selected +727,45,45,45,56,1197710.0,selected +730,1,1,1,4,1828.5,selected +731,85,85,85,112,518297.0,selected +732,30,29,29,109,1948107.5,selected +734,26,26,26,55,1106111.0,selected +737,1,1,1,0,0.0,not_selected +740,38,38,38,291,1319218.5,selected +747,5,5,5,88,843052.5,selected +748,1,1,1,1,512.0,selected +754,1,1,1,7,172985.0,selected +757,39,39,39,142,1778864.0,selected +760,79,79,79,135,1989949.0,selected +762,3,3,3,9,105422.0,selected +763,55,55,55,37,674605.0,selected +765,201,201,201,213,1127447.0,selected +769,3,3,3,4,31695.0,selected +770,86,85,85,203,4998632.666666667,selected +772,29,29,29,31,672502.0,selected +773,3,3,3,56,897075.3333333334,selected +774,3,2,2,35,250527.0,selected +775,57,56,56,104,846876.0,selected +778,1,0,0,0,0.0,no_retained_locality +779,3,3,3,9,10350.5,selected +781,7,7,7,59,978996.5,selected +785,231,231,231,307,780902.0,selected +786,5,3,3,61,977257.0,selected +787,9,9,9,132,3006915.5,selected +800,1,0,0,0,0.0,no_retained_locality +801,52,52,52,85,1811644.0,selected +802,256,256,256,265,645254.0,selected +803,110,110,110,149,1756437.0,selected +804,26,26,26,143,1417880.0,selected +805,59,59,59,75,1544123.5,selected +806,121,121,121,180,881959.0,selected +808,148,148,148,97,1445635.0,selected +810,22,22,22,62,670089.75,selected +812,190,190,190,265,1345239.5,selected +813,48,48,48,65,1780976.0,selected +814,96,96,96,436,1297365.0,selected +815,59,58,58,259,1663801.5,selected +816,87,87,87,154,1389068.0,selected +817,44,44,44,93,1767122.1666666667,selected +818,14,14,14,126,1823383.5,selected +820,4,1,1,3,29686.5,selected +821,2,2,2,2,3900.5,selected +828,89,89,89,130,1186881.0,selected +830,82,82,82,122,866123.0,selected +831,28,28,28,46,749649.0,selected +832,4,4,4,100,765113.7,selected +835,1,1,1,10,67881.33333333333,selected +837,1,1,1,1,256.5,selected +838,1,1,1,3,6573.0,selected +839,1,1,1,1,3970.0,selected +840,2,2,2,7,207458.33333333334,selected +843,96,96,96,154,1901653.0,selected +845,97,97,97,231,1383834.0,selected +847,25,25,25,67,1592547.5,selected +848,3,3,3,9,105658.5,selected +850,157,157,157,128,1539275.0,selected +854,1,1,1,2,3288.0,selected +856,18,18,18,99,1252771.0,selected +857,1,1,1,5,76656.5,selected +858,8,8,8,49,792788.0,selected +859,76,76,76,93,1186719.0,selected +860,170,170,170,202,1653884.0,selected +861,2,2,2,10,66422.5,selected +862,2,2,2,10,163822.5,selected +863,72,72,72,65,1136410.0,selected +864,65,65,65,121,1543525.5,selected +865,32,32,32,72,1028755.0,selected +870,329,328,328,379,1106081.0,selected +872,2,2,2,56,897075.3333333334,selected +877,1,0,0,0,0.0,no_retained_locality +878,2,2,2,60,261775.66666666666,selected +885,1,0,0,0,0.0,no_retained_locality +888,1,0,0,0,0.0,no_retained_locality +901,25,25,25,55,1026506.0,selected +903,153,153,153,222,1428434.0,selected +904,39,39,39,63,1650501.0,selected +906,26,26,26,135,310975.0,selected +907,98,97,97,245,733346.0,selected +908,23,23,23,79,1009658.0,selected +909,34,34,34,68,1904719.3333333333,selected +910,136,136,136,158,1682177.0,selected +912,107,107,107,124,1063545.0,selected +913,45,45,45,74,934321.0,selected +914,39,39,39,84,1470283.0,selected +915,25,25,25,55,882756.0,selected +916,17,17,17,63,1803153.0,selected +917,12,12,12,112,1103007.5833333333,selected +918,14,14,14,114,657650.0,selected +919,49,49,49,100,1949247.5,selected +920,218,218,218,216,1256190.0,selected +924,1,1,1,0,0.0,not_selected +925,46,46,46,36,1127028.0,selected +927,1,0,0,0,0.0,no_retained_locality +928,393,393,393,178,1118343.0,selected +929,3,3,3,8,212049.0,selected +930,6,6,6,20,215254.5,selected +931,72,72,72,143,1030631.0,selected +934,1,1,1,4,52147.0,selected +936,58,58,58,101,890746.0,selected +937,30,30,30,179,1441585.0,selected +938,1,1,1,0,0.0,not_selected +939,2,2,2,20,247969.5,selected +940,74,74,74,95,701461.0,selected +941,55,55,55,59,1081304.0,selected +945,1,1,1,14,82482.66666666667,selected +947,2,2,2,4,12379.75,selected +949,17,17,17,42,1252751.0,selected +951,22,22,22,53,2064580.3333333333,selected +952,40,40,40,49,842300.0,selected +954,3,3,3,29,957846.0,selected +956,70,70,70,72,1681656.0,selected +959,8,8,8,15,120711.5,selected +970,145,144,144,212,1334473.0,selected +971,352,350,350,104,1028588.0,selected +972,68,68,68,158,3500046.6666666665,selected +973,27,25,25,116,1703324.5,selected +975,1,1,1,0,0.0,not_selected +977,1,0,0,0,0.0,no_retained_locality +978,9,9,9,91,1417452.0,selected +979,58,58,58,91,726456.0,selected +980,2,2,2,2,16174.0,selected +983,1,1,1,6,75113.66666666667,selected +984,3,3,3,21,364074.5,selected +985,41,41,41,101,992006.0,selected +986,1,1,1,1,18.0,selected +989,52,52,52,192,1015622.0,selected diff --git a/scripts/person_data/build_report.json b/scripts/person_data/build_report.json new file mode 100644 index 000000000..f6f252c54 --- /dev/null +++ b/scripts/person_data/build_report.json @@ -0,0 +1,91 @@ +{ + "rows": 33772, + "population_total": 335555510.0, + "methods": { + "explicit_postcode_association": 18749, + "nearest_locality_same_state": 15023 + }, + "candidate_assignments": 37239, + "zipcodes_with_multiple_area_codes": 2322, + "raw_locality_rows": 22913, + "noncanonical_rows_excluded": 116, + "curated_locality_rows_excluded": 92, + "accepted_locality_records": 22705, + "accepted_locality_items": 20860, + "area_code_coverage": { + "input_codes": 386, + "retained_codes": 358, + "eligible_codes": 355, + "output_codes": 333, + "no_retained_locality": [ + 236, + 250, + 257, + 349, + 384, + 393, + 453, + 456, + 486, + 519, + 590, + 611, + 622, + 627, + 635, + 639, + 642, + 655, + 670, + 672, + 722, + 778, + 800, + 877, + 885, + 888, + 927, + 977 + ], + "outside_zcta_states": [ + 340, + 671, + 684 + ], + "not_selected": [ + 274, + 279, + 283, + 327, + 329, + 341, + 357, + 380, + 457, + 463, + 464, + 483, + 557, + 567, + 572, + 659, + 679, + 689, + 737, + 924, + 938, + 975 + ], + "reconciliation_sha256": "6b18cc988b83ef3a2de4c4954a39dc52267b536ecda11aaf230e5299b220954a" + }, + "distant_explicit_associations_excluded": 46, + "nearest_locality_fallbacks_over_50km": 348, + "locality_distance_km_quantiles": { + "0.5": 6.5035926030747735, + "0.9": 19.572534448154848, + "0.95": 25.88603157156294, + "0.99": 50.42410233449431, + "1.0": 230.84417705249874 + }, + "output_sha256": "b6fc7c8c27b008b8f0da89dfa75a4e2e5693cce18ae0c8a0d3527e326e17e5dd" +} diff --git a/scripts/person_data/excluded_localities.csv b/scripts/person_data/excluded_localities.csv new file mode 100644 index 000000000..652c45e99 --- /dev/null +++ b/scripts/person_data/excluded_localities.csv @@ -0,0 +1,93 @@ +qid,area_code,longitude,latitude,state +Q881120,201,-67.868611111,46.504722222,ME +Q4788559,250,-124.0,54.0, +Q1952785,216,-94.20638889,47.92527778,MN +Q3261921,360,-84.760555555,33.550555555,GA +Q4788610,236,-124.0,54.0, +Q436898,349,-89.58139,37.63111,MO +Q684406,393,-75.483333333,44.7,NY +Q2272332,393,-75.435555555,44.5525,NY +Q27988976,509,-121.77638889,45.69805556,OR +Q2227968,770,-92.39797,46.66355,MN +Q518466,725,-69.963055555,43.910833333,ME +Q7580837,970,-122.329722222,45.518888888,OR +Q2431419,501,-101.281944444,48.971388888,ND +Q342064,605,-107.623611111,35.065833333,NM +Q1783975,605,-89.931666666,44.121666666,WI +Q1887524,605,-89.3643323,45.2478878,WI +Q2304624,605,-71.558611111,44.654166666,NH +Q5817097,605,-89.638333333,42.601111111,WI +Q7589473,605,-89.052777777,44.454166666,WI +Q2582333,605,-91.2522,36.0067,AR +Q2381270,384,-82.340833333,30.021666666,FL +Q286148,627,-73.693333333,40.792777777,NY +Q136503722,639,-123.6,39.8,CA +Q2270773,870,-97.168611111,35.134722222,OK +Q584340,722,-80.335555555,27.438888888,FL +Q2218020,318,-91.796666666,44.4575,WI +Q370523,620,-96.870833333,35.894166666,OK +Q1592949,610,-73.99715,40.722263888,NY +Q2200776,630,-83.83051,42.95725,MI +Q951376,774,-84.5383,32.0608,GA +Q518097,519,-73.65,43.05,NY +Q2200776,635,-83.83051,42.95725,MI +Q6646528,530,-120.297222222,41.994166666,OR +Q1886236,402,-87.869444444,44.029722222,WI +Q3709190,402,-75.098055555,43.096111111,NY +Q7410310,973,-122.974166666,42.491666666,OR +Q7970633,973,-122.016,45.6126,OR +Q3708399,325,-76.341666666,43.123055555,NY +Q602784,701,-88.714166666,45.743611111,WI +Q600237,720,-95.1708,37.9161,KS +Q114222598,971,-122.3337357,47.4631546,WA +Q114358755,971,-122.6507302,48.5728826,WA +Q14682133,458,-121.51944444,41.99805556,CA +Q2572411,458,-117.718333333,41.9975,NV +Q979819,885,-83.700443,36.150854,TN +Q2535145,702,-104.917,39.8389,CO +Q4788610,257,-124.0,54.0, +Q735981,977,-91.861111111,41.994722222,IA +Q67077,208,-87.907178,32.993811,AL +Q67870,208,-86.943126,34.241338,AL +Q1898553,312,-118.253,33.9175,CA +Q51679,670,145.75,15.183333333,MP +Q93022,706,-99.015833333,26.405,TX +Q3150256,670,145.75,15.18333333,MP +Q9049890,706,-76.370277777,42.022222222,NY +Q1630163,732,-73.981086,40.702553,NY +Q2527718,786,-97.7597,38.9694,KS +Q4532801,786,-73.988211111,40.718497222,NY +Q3287686,456,-88.245,36.037222222,TN +Q647127,225,-100.018333333,37.759722222,KS +Q6346,486,-86.158055555,39.768611111,IN +Q2418283,602,-71.768888888,42.743611111,NH +Q1264177,453,-112.751388888,40.231111111,UT +Q14623578,800,-74.010694444,40.710972222,NY +Q1592949,877,-73.99715,40.722263888,NY +Q14682133,541,-121.51944444,41.99805556,CA +Q114222598,503,-122.3337357,47.4631546,WA +Q114358755,503,-122.6507302,48.5728826,WA +Q1976579,334,-85.0669,36.9847,KY +Q2572411,541,-117.718333333,41.9975,NV +Q1506783,590,-120.543,46.4042,WA +Q968951,655,-91.5006,40.0422,MO +Q1842438,507,-98.105,40.431944444,NE +Q483932,406,-116.6,48.29,ID +Q7231843,907,-107.701,38.081,CO +Q5127590,820,-85.3478,39.4333,IN +Q5969554,820,-85.3075,39.41,IN +Q5969554,820,-85.3078,39.4103,IN +Q2303201,622,-90.1364,34.1731,MS +Q1887361,815,-89.742935,45.3180179,WI +Q7546719,302,-75.570738,42.687292,NY +Q3128858,612,-122.428,37.7728,CA +Q549475,575,-123.0,45.5,OR +Q6160569,775,-120.048,39.5094,CA +Q14931631,888,-74.013888888,40.706111111,NY +Q1966409,611,-117.833611111,34.617777777,CA +Q3458813,611,-119.389166666,35.85,CA +Q2673377,642,-97.7664,39.2747,KS +Q949651,927,-83.8292,39.4458,OH +Q4788610,778,-124.0,54.0, +Q4788610,672,-124.0,54.0, +Q2738898,609,-88.333888888,46.098055555,MI diff --git a/scripts/person_data/rebuild.py b/scripts/person_data/rebuild.py new file mode 100644 index 000000000..f1d83b6af --- /dev/null +++ b/scripts/person_data/rebuild.py @@ -0,0 +1,387 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Rebuild the bundled ZIP/area-code lookup from documented source snapshots.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import re +import struct +from pathlib import Path +from urllib.request import urlopen +from zipfile import ZipFile + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[1] +ASSET = ( + REPO + / "packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet" +) +SOURCE_REVISION_URL = ( + "https://github.com/NVIDIA-NeMo/DataDesigner/tree/eae83186b32d7c62ee880c6f538bda46a6831c72/scripts/person_data" +) + + +def read_source(name: str, source: dict[str, str], cache: Path, offline: bool = False) -> bytes: + """Read a pinned snapshot, downloading immutable Census files when needed.""" + if "snapshot" in source: + data = gzip.decompress((ROOT / source["snapshot"]).read_bytes()) + elif (cache / name).exists(): + data = (cache / name).read_bytes() + elif offline: + raise FileNotFoundError(f"Missing cached source: {cache / name}") + else: + with urlopen(source["url"], timeout=120) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != source["sha256"]: + raise ValueError(f"Input checksum mismatch: {name}") + if "snapshot" not in source: + cache.mkdir(parents=True, exist_ok=True) + (cache / name).write_bytes(data) + return data + + +def sphere(coordinates: np.ndarray) -> np.ndarray: + radians = np.radians(coordinates) + return np.column_stack( + ( + np.cos(radians[:, 0]) * np.cos(radians[:, 1]), + np.cos(radians[:, 0]) * np.sin(radians[:, 1]), + np.sin(radians[:, 0]), + ) + ) + + +def load_state_polygons(data: bytes) -> list[tuple[str, list[np.ndarray]]]: + """Read the pinned Census Polygon/DBF pair using its fixed field layout.""" + with ZipFile(io.BytesIO(data)) as archive: + shp = archive.read("tl_2023_us_state.shp") + dbf = archive.read("tl_2023_us_state.dbf") + if len(shp) < 100 or struct.unpack_from(">i", shp)[0] != 9994 or struct.unpack_from("i", shp, 24)[0] * 2 != len(shp) or len(dbf) < 33: + raise ValueError("Invalid Census geometry file lengths") + count, header_size, record_size = struct.unpack_from(" len(shp): + raise ValueError("Missing Census state record") + number, words = struct.unpack_from(">ii", shp, offset) + end = offset + 8 + words * 2 + content = memoryview(shp)[offset + 8 : end] + if number != index + 1 or words < 22 or end > len(shp) or struct.unpack_from(" np.ndarray: + """Assign planar point coverage, including ring boundaries and excluding holes.""" + result = np.full(len(frame), "", dtype=object) + points = frame[[x, y]].to_numpy(dtype="float64") + for abbreviation, rings in states: + inside = np.zeros(len(points), dtype=bool) + boundary = np.zeros(len(points), dtype=bool) + for ring in rings: + minimum, maximum = ring.min(axis=0), ring.max(axis=0) + selected = np.flatnonzero(((points >= minimum) & (points <= maximum)).all(axis=1)) + selected = selected[np.argsort(points[selected, 1], kind="stable")] + start, stop = ring[:-1], ring[1:] + low_y = np.minimum(start[:, 1], stop[:, 1]) + high_y = np.maximum(start[:, 1], stop[:, 1]) + for offset in range(0, len(selected), 32): + indices = selected[offset : offset + 32] + px, py = points[indices, 0, None], points[indices, 1, None] + active = (low_y <= py.max()) & (high_y >= py.min()) + x0, y0 = start[active].T + x1, y1 = stop[active].T + dx, dy = x1 - x0, y1 - y0 + cross_x = x0 + (py - y0) * dx / np.where(dy == 0, 1, dy) + crossings = ((y0 > py) != (y1 > py)) & (px < cross_x) + inside[indices] ^= np.count_nonzero(crossings, axis=1) % 2 == 1 + on_segment = ( + ((px - x0) * dy == (py - y0) * dx) + & (px >= np.minimum(x0, x1)) + & (px <= np.maximum(x0, x1)) + & (py >= low_y[active]) + & (py <= high_y[active]) + ) + boundary[indices] |= np.any(on_segment, axis=1) + result[inside | boundary] = abbreviation + return result + + +def assign_area_codes( + data: pd.DataFrame, localities: pd.DataFrame, raw_postcodes: list[dict] +) -> tuple[pd.DataFrame, pd.DataFrame, int]: + """Retain all code candidates at the best supported same-state locality point.""" + parts = [] + for state, group in data.groupby("state"): + supplied = localities[localities.state == state].sort_values(["qid", "area_code"]) + points = supplied.drop_duplicates(["latitude", "longitude"]) + distance, position = cKDTree(sphere(points[["latitude", "longitude"]].to_numpy())).query( + sphere(group[["INTPTLAT", "INTPTLONG"]].to_numpy()) + ) + donor = points.iloc[position].reset_index(drop=True) + group = group.reset_index(drop=True) + group["latitude"] = donor.latitude + group["longitude"] = donor.longitude + group["distance_km"] = 2 * 6371.0088 * np.arcsin(np.minimum(distance / 2, 1)) + group["method"] = "nearest_locality_same_state" + parts.append(group.merge(supplied, on=["state", "latitude", "longitude"], validate="many_to_many")) + provenance = pd.concat(parts).rename(columns={"qid": "donor_qid"}) + + # Prefer literal ZIP associations. Broad ranges and wildcards are not expanded. + pairs = [ + { + "zipcode": row["postcode"]["value"], + "area_code": int(row["code"]["value"]), + "qid": row["place"]["value"].rsplit("/", 1)[-1], + } + for row in raw_postcodes + if re.fullmatch(r"\d{5}", row["postcode"]["value"]) and re.fullmatch(r"[2-9]\d{2}", row["code"]["value"]) + ] + explicit = ( + pd.DataFrame(pairs, columns=["zipcode", "area_code", "qid"]) + .merge(localities, on=["qid", "area_code"]) + .drop_duplicates() + ) + explicit = explicit.merge(data[["zipcode", "state", "INTPTLAT", "INTPTLONG"]], on=["zipcode", "state"]) + chord = np.linalg.norm( + sphere(explicit[["latitude", "longitude"]].to_numpy()) - sphere(explicit[["INTPTLAT", "INTPTLONG"]].to_numpy()), + axis=1, + ) + explicit["distance_km"] = 2 * 6371.0088 * np.arcsin(np.minimum(chord / 2, 1)) + distant_explicit_rows = int((explicit.distance_km > 50).sum()) + explicit = explicit[explicit.distance_km <= 50] + closest = explicit.sort_values(["zipcode", "distance_km", "qid", "area_code"]).drop_duplicates("zipcode") + chosen = closest.drop(columns=["qid", "area_code"]).merge( + localities, on=["state", "latitude", "longitude"], validate="many_to_many" + ) + chosen = chosen.merge(data[["zipcode", "count"]], on="zipcode", validate="many_to_one") + chosen["method"] = "explicit_postcode_association" + chosen = chosen.rename(columns={"qid": "donor_qid"}) + if not chosen.empty: + provenance = pd.concat([provenance[~provenance.zipcode.isin(chosen.zipcode)], chosen]) + provenance = ( + provenance.sort_values(["zipcode", "distance_km", "donor_qid", "area_code"]) + .drop_duplicates(["zipcode", "area_code"]) + .reset_index(drop=True) + ) + result = provenance.drop_duplicates("zipcode")[["zipcode", "area_code", "count"]].astype( + {"area_code": "int64", "count": "float64"} + ) + codes = provenance.groupby("zipcode").area_code.agg(lambda values: sorted(set(values))) + result = result.assign(area_codes=result.zipcode.map(codes)).reset_index(drop=True) + return result, provenance, distant_explicit_rows + + +def reconcile_area_codes( + input_codes: pd.Series, localities: pd.DataFrame, zip_states: set[str], result: pd.DataFrame +) -> pd.DataFrame: + """Account for each source code, including retained codes never selected.""" + assignments = result[["zipcode", "area_codes", "count"]].copy() + assignments["count"] /= assignments.area_codes.str.len() + assignments = assignments.explode("area_codes").rename(columns={"area_codes": "area_code"}) + if not set(assignments.area_code) <= set(input_codes): + raise ValueError("Output area codes are not a subset of the pinned input") + coverage = input_codes.value_counts().rename("input_records").to_frame() + coverage.index.name = "area_code" + coverage = coverage.join(localities.area_code.value_counts().rename("retained_locality_records")) + eligible = localities[localities.state.isin(zip_states)] + coverage = coverage.join(eligible.area_code.value_counts().rename("eligible_locality_records")) + coverage = coverage.join( + assignments.groupby("area_code").agg(output_zipcodes=("zipcode", "size"), output_population=("count", "sum")) + ).fillna(0) + for column in ("input_records", "retained_locality_records", "eligible_locality_records", "output_zipcodes"): + coverage[column] = coverage[column].astype("int64") + coverage["status"] = np.select( + [ + coverage.output_zipcodes > 0, + coverage.retained_locality_records == 0, + coverage.eligible_locality_records == 0, + ], + ["selected", "no_retained_locality", "outside_zcta_states"], + default="not_selected", + ) + return coverage.sort_index().reset_index() + + +def rebuild(cache: Path, output: Path, report_dir: Path, offline: bool = False) -> dict: + """Rebuild the asset and write row-level lineage and a validation summary.""" + manifest = json.loads((ROOT / "sources.json").read_text()) + excluded_bytes = (ROOT / "excluded_localities.csv").read_bytes() + if hashlib.sha256(excluded_bytes).hexdigest() != manifest["excluded_localities_sha256"]: + raise ValueError("Exclusion snapshot checksum mismatch") + inputs = {name: read_source(name, source, cache, offline) for name, source in manifest["sources"].items()} + states = load_state_polygons(inputs["census-state-boundaries-full-2023.zip"]) + + raw = json.loads(inputs["wikidata-us-dialing-codes.json"])["results"]["bindings"] + input_codes = pd.Series( + [int(row["code"]["value"]) for row in raw if re.fullmatch(r"[2-9]\d{2}", row["code"]["value"])], + dtype="int64", + ) + rows = [] + for row in raw: + code = row["code"]["value"] + match = re.fullmatch(r"Point\((-?[\d.]+) (-?[\d.]+)\)", row["coord"]["value"]) + if not re.fullmatch(r"[2-9]\d{2}", code) or match is None: + continue + rows.append( + { + "qid": row["place"]["value"].rsplit("/", 1)[-1], + "area_code": int(code), + "longitude": float(match[1]), + "latitude": float(match[2]), + } + ) + localities = pd.DataFrame(rows).drop_duplicates().reset_index(drop=True) + if not localities.latitude.between(-90, 90).all() or not localities.longitude.between(-180, 180).all(): + raise ValueError("Invalid locality coordinates") + localities["state"] = states_for(localities, "longitude", "latitude", states) + excluded = pd.read_csv(io.BytesIO(excluded_bytes)) + excluded_pairs = pd.MultiIndex.from_frame(excluded[["qid", "area_code"]]) + accepted = ~pd.MultiIndex.from_frame(localities[["qid", "area_code"]]).isin(excluded_pairs) + rejected = localities[~accepted] + localities = localities[accepted].copy() + + population = pd.read_csv(io.BytesIO(inputs["census-acs5-2023-b01003.dat"]), sep="|", dtype={"GEO_ID": str}) + population = population[population.GEO_ID.str.fullmatch(r"860Z200US\d{5}")].copy() + population["zipcode"] = population.GEO_ID.str[-5:] + population = population.rename(columns={"B01003_E001": "count"})[["zipcode", "count"]] + if len(population) != 33772 or population.zipcode.duplicated().any() or (population["count"] < 0).any(): + raise ValueError("Unexpected Census population records") + with ZipFile(io.BytesIO(inputs["gazetteer-2023-zcta.zip"])) as archive: + with archive.open("2023_Gaz_zcta_national.txt") as stream: + geography = pd.read_csv(stream, sep="\t", dtype={"GEOID": str}) + geography.columns = geography.columns.str.strip() + data = ( + population.merge( + geography[["GEOID", "INTPTLAT", "INTPTLONG"]], + left_on="zipcode", + right_on="GEOID", + how="left", + validate="one_to_one", + ) + .sort_values("zipcode") + .reset_index(drop=True) + ) + if data[["INTPTLAT", "INTPTLONG"]].isna().any().any(): + raise ValueError("Missing ZIP coordinates") + data["state"] = states_for(data, "INTPTLONG", "INTPTLAT", states) + if (data.state == "").any() or set(data.state) - set(localities.state): + raise ValueError("Incomplete state/locality coverage") + + postcodes = json.loads(inputs["wikidata-us-postcode-dialing-codes.json"])["results"]["bindings"] + result, provenance, distant_explicit_rows = assign_area_codes(data, localities, postcodes) + coverage = reconcile_area_codes(input_codes, localities, set(data.state), result) + lineage = { + "format_version": 2, + "candidate_selection": "population_weighted_zip_then_uniform_area_code", + "source_revision_url": SOURCE_REVISION_URL, + "source_manifest_sha256": hashlib.sha256((ROOT / "sources.json").read_bytes()).hexdigest(), + "source_manifest": manifest, + "rebuild_script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + } + table = pa.Table.from_pandas(result, preserve_index=False) + table = table.replace_schema_metadata( + {**table.schema.metadata, b"data_designer.provenance": json.dumps(lineage, sort_keys=True).encode()} + ) + output.parent.mkdir(parents=True, exist_ok=True) + pq.write_table(table, output, compression="snappy") + pd.testing.assert_frame_equal(result, pd.read_parquet(output)) + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / "mapping_provenance.csv.gz").write_bytes( + gzip.compress(provenance.to_csv(index=False).encode(), mtime=0) + ) + reconciliation = coverage.to_csv(index=False).encode() + (report_dir / "area_code_reconciliation.csv").write_bytes(reconciliation) + report = { + "rows": len(result), + "population_total": float(result["count"].sum()), + "methods": provenance.drop_duplicates("zipcode").method.value_counts().to_dict(), + "candidate_assignments": len(provenance), + "zipcodes_with_multiple_area_codes": int((result.area_codes.str.len() > 1).sum()), + "raw_locality_rows": len(raw), + "noncanonical_rows_excluded": len(raw) - len(rows), + "curated_locality_rows_excluded": len(rejected), + "accepted_locality_records": len(localities), + "accepted_locality_items": int(localities.qid.nunique()), + "area_code_coverage": { + "input_codes": len(coverage), + "retained_codes": int((coverage.retained_locality_records > 0).sum()), + "eligible_codes": int((coverage.eligible_locality_records > 0).sum()), + "output_codes": int((coverage.output_zipcodes > 0).sum()), + "no_retained_locality": coverage.loc[coverage.status == "no_retained_locality", "area_code"].tolist(), + "outside_zcta_states": coverage.loc[coverage.status == "outside_zcta_states", "area_code"].tolist(), + "not_selected": coverage.loc[coverage.status == "not_selected", "area_code"].tolist(), + "reconciliation_sha256": hashlib.sha256(reconciliation).hexdigest(), + }, + "distant_explicit_associations_excluded": distant_explicit_rows, + "nearest_locality_fallbacks_over_50km": int( + provenance.loc[ + (provenance.method == "nearest_locality_same_state") & (provenance.distance_km > 50), "zipcode" + ].nunique() + ), + "locality_distance_km_quantiles": provenance.drop_duplicates("zipcode") + .distance_km.quantile([0.5, 0.9, 0.95, 0.99, 1.0]) + .to_dict(), + "output_sha256": hashlib.sha256(output.read_bytes()).hexdigest(), + } + (report_dir / "build_report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache-dir", type=Path, default=REPO / ".cache/person_data") + parser.add_argument("--output", type=Path, default=ASSET) + parser.add_argument("--report-dir", type=Path, default=REPO / ".cache/person_data/report") + parser.add_argument("--offline", action="store_true", help="Require previously cached Census inputs") + args = parser.parse_args() + print(json.dumps(rebuild(args.cache_dir, args.output, args.report_dir, args.offline), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/person_data/snapshots/wikidata-us-dialing-codes.json.gz b/scripts/person_data/snapshots/wikidata-us-dialing-codes.json.gz new file mode 100644 index 000000000..2d902ebd7 Binary files /dev/null and b/scripts/person_data/snapshots/wikidata-us-dialing-codes.json.gz differ diff --git a/scripts/person_data/snapshots/wikidata-us-postcode-dialing-codes.json.gz b/scripts/person_data/snapshots/wikidata-us-postcode-dialing-codes.json.gz new file mode 100644 index 000000000..7376b21b3 Binary files /dev/null and b/scripts/person_data/snapshots/wikidata-us-postcode-dialing-codes.json.gz differ diff --git a/scripts/person_data/sources.json b/scripts/person_data/sources.json new file mode 100644 index 000000000..5a887f667 --- /dev/null +++ b/scripts/person_data/sources.json @@ -0,0 +1,50 @@ +{ + "sources": { + "census-acs5-2023-b01003.dat": { + "url": "https://www2.census.gov/programs-surveys/acs/summary_file/2023/table-based-SF/data/5YRData/acsdt5y2023-b01003.dat", + "sha256": "24ae3f523b4c54332ce0a71ba534569685a8a729056f975915d860d1eb943565", + "retrieved": "2026-11-06", + "license": "U.S. government work (public domain in the United States)", + "license_url": "https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2023/TGRSHP2023_TechDoc.pdf" + }, + "gazetteer-2023-zcta.zip": { + "url": "https://www2.census.gov/geo/docs/maps-data/data/gazetteer/2023_Gazetteer/2023_Gaz_zcta_national.zip", + "sha256": "75191ab2d3df4b668fb511d2726d97c521d6d1ff555725704a5d9bb3bc2077ce", + "retrieved": "2026-11-06", + "license": "U.S. government work (public domain in the United States)", + "license_url": "https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2023/TGRSHP2023_TechDoc.pdf" + }, + "census-state-boundaries-full-2023.zip": { + "url": "https://www2.census.gov/geo/tiger/TIGER2023/STATE/tl_2023_us_state.zip", + "sha256": "8ded0ef036e205e246ae4b03a66873f7e1eed50285b993702ce5aca674283a2e", + "retrieved": "2026-11-06", + "license": "U.S. government work (public domain in the United States)", + "license_url": "https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2023/TGRSHP2023_TechDoc.pdf" + }, + "wikidata-us-dialing-codes.json": { + "url": "https://query.wikidata.org/sparql?query=SELECT+%3Fplace+%3Fcode+%3Fcoord+WHERE+%7B%0A++%3Fplace+wdt%3AP17+wd%3AQ30%3B+wdt%3AP473+%3Fcode%3B+wdt%3AP625+%3Fcoord.%0A%7D&format=json", + "sha256": "5560d9fadd3316f9e230c24c8bf531c56f778ad0fbb934ade9bbdee0926fd8fb", + "retrieved": "2026-11-06", + "snapshot": "snapshots/wikidata-us-dialing-codes.json.gz", + "license": "CC0-1.0", + "license_url": "https://www.wikidata.org/wiki/Wikidata:Licensing" + }, + "wikidata-us-postcode-dialing-codes.json": { + "url": "https://query.wikidata.org/sparql?query=SELECT+DISTINCT+%3Fplace+%3Fpostcode+%3Fcode+WHERE+%7B%0A++VALUES+%3Fcountry+%7B+wd%3AQ30+wd%3AQ1183+%7D%0A++%3Fplace+wdt%3AP17+%3Fcountry%3B+wdt%3AP473+%3Fcode%3B+wdt%3AP281+%3Fpostcode.%0A%7D&format=json", + "sha256": "930af36cac300bac78e4ec0689245b293c0e79878504a88b57eb5b386470cd61", + "retrieved": "2026-11-06", + "snapshot": "snapshots/wikidata-us-postcode-dialing-codes.json.gz", + "license": "CC0-1.0", + "license_url": "https://www.wikidata.org/wiki/Wikidata:Licensing" + } + }, + "excluded_localities_sha256": "dc1d1ae22bc8ee4ecd2641a8d19b760f7612a1e6e4950c1fb0649b21f8a01e2b", + "validation_reference": { + "name": "NANPA NPA database", + "url": "https://reports.nanpa.com/public/npa_report.csv", + "retrieved": "2026-11-06", + "sha256": "dcbbcd9143d91a19c8573d2ffbb1777cfff10f7f16e86cd52cf2b8930c26b5bb", + "role": "Screening reference for foreign, inactive, and wrong-state Wikidata locality associations; report not redistributed or required for rebuilding.", + "license": "No explicit open-data license established; not claimed to be CC0 or public domain." + } +} diff --git a/scripts/person_data/test_rebuild.py b/scripts/person_data/test_rebuild.py new file mode 100644 index 000000000..6762a0e1c --- /dev/null +++ b/scripts/person_data/test_rebuild.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import io +import struct +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd +import pytest +from rebuild import assign_area_codes, load_state_polygons, read_source, reconcile_area_codes, states_for + + +@pytest.fixture +def geography() -> tuple[pd.DataFrame, pd.DataFrame]: + data = pd.DataFrame( + [("01000", "MA", 42.0, -72.0, 100.0), ("02000", "NY", 42.0, -72.01, 200.0)], + columns=["zipcode", "state", "INTPTLAT", "INTPTLONG", "count"], + ) + localities = pd.DataFrame( + [ + ("Q1", 413, 42.0, -72.1, "MA"), + ("Q2", 212, 42.0, -72.0, "NY"), + ("Q3", 617, 42.0, -72.2, "MA"), + ("Q4", 781, 42.0, -74.0, "MA"), + ], + columns=["qid", "area_code", "latitude", "longitude", "state"], + ) + return data, localities + + +@pytest.mark.parametrize( + ("postcode", "qid", "code", "expected", "method", "distant"), + [ + (None, "Q3", "617", 413, "nearest_locality_same_state", 0), + ("01000", "Q3", "617", 617, "explicit_postcode_association", 0), + ("01000", "Q2", "212", 413, "nearest_locality_same_state", 0), + ("01000", "Q4", "781", 413, "nearest_locality_same_state", 1), + ("01000-01099", "Q3", "617", 413, "nearest_locality_same_state", 0), + ("010*", "Q3", "617", 413, "nearest_locality_same_state", 0), + ("01000", "Q3", "0617", 413, "nearest_locality_same_state", 0), + ("01000", "Q99", "617", 413, "nearest_locality_same_state", 0), + ], +) +def test_assign_area_codes( + geography: tuple[pd.DataFrame, pd.DataFrame], + postcode: str | None, + qid: str, + code: str, + expected: int, + method: str, + distant: int, +) -> None: + data, localities = geography + postcodes = ( + [] + if postcode is None + else [ + { + "place": {"value": f"http://www.wikidata.org/entity/{qid}"}, + "postcode": {"value": postcode}, + "code": {"value": code}, + } + ] + ) + result, lineage, rejected = assign_area_codes(data, localities, postcodes) + assert result.area_code.tolist() == [expected, 212] + assert lineage.method.iloc[0] == method + assert rejected == distant + pd.testing.assert_frame_equal(result[["zipcode", "count"]], data[["zipcode", "count"]]) + + +def test_colocated_localities_are_order_independent(geography: tuple[pd.DataFrame, pd.DataFrame]) -> None: + data, localities = geography + localities.loc[2, ["latitude", "longitude"]] = localities.loc[0, ["latitude", "longitude"]].to_numpy() + forward, _, _ = assign_area_codes(data, localities, []) + reversed_result, _, _ = assign_area_codes(data, localities.iloc[::-1], []) + pd.testing.assert_frame_equal(forward, reversed_result) + assert forward.area_code.iloc[0] == 413 + assert forward.area_codes.iloc[0] == [413, 617] + + +@pytest.mark.parametrize( + "associations", [[("Q1", "413")], [("Q3", "617")], [("Q1", "413"), ("Q3", "617"), ("Q3", "617")]] +) +def test_explicit_postcode_retains_colocated_codes( + geography: tuple[pd.DataFrame, pd.DataFrame], associations: list[tuple[str, str]] +) -> None: + data, localities = geography + localities.loc[[1, 2], ["latitude", "longitude"]] = localities.loc[0, ["latitude", "longitude"]].to_numpy() + postcodes = [ + {"place": {"value": qid}, "postcode": {"value": "01000"}, "code": {"value": code}} for qid, code in associations + ] + result, lineage, _ = assign_area_codes(data, localities, postcodes) + assert result.area_codes.tolist() == [[413, 617], [212]] + assert lineage[lineage.zipcode == "01000"].area_code.tolist() == [413, 617] + + +def test_different_locality_points_are_not_combined(geography: tuple[pd.DataFrame, pd.DataFrame]) -> None: + data, localities = geography + postcodes = [ + {"place": {"value": qid}, "postcode": {"value": "01000"}, "code": {"value": code}} + for qid, code in [("Q1", "413"), ("Q3", "617")] + ] + result, _, _ = assign_area_codes(data, localities, postcodes) + assert result.area_codes.iloc[0] == [413] + + +@pytest.mark.parametrize("contents", [b"original", b"changed"]) +def test_cached_source_checksum(tmp_path: Path, contents: bytes) -> None: + (tmp_path / "source").write_bytes(contents) + source = {"sha256": hashlib.sha256(b"original").hexdigest()} + if contents == b"original": + assert read_source("source", source, tmp_path, offline=True) == contents + else: + with pytest.raises(ValueError, match="checksum mismatch"): + read_source("source", source, tmp_path, offline=True) + + +def test_offline_missing_source(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="Missing cached source"): + read_source("missing", {}, tmp_path, offline=True) + + +def test_area_code_reconciliation_tracks_missing_codes() -> None: + inputs = pd.Series([212, 212, 332, 340, 773], dtype="int64") + localities = pd.DataFrame([(212, "NY"), (332, "NY"), (340, "VI")], columns=["area_code", "state"]) + result = pd.DataFrame([("10001", [212], 100.0)], columns=["zipcode", "area_codes", "count"]) + coverage = reconcile_area_codes(inputs, localities, {"NY"}, result).set_index("area_code") + assert coverage.status.to_dict() == { + 212: "selected", + 332: "not_selected", + 340: "outside_zcta_states", + 773: "no_retained_locality", + } + assert coverage.loc[212, "input_records"] == 2 + assert coverage.loc[212, "output_population"] == 100.0 + + +def test_reconciliation_splits_weight_between_candidates() -> None: + inputs = pd.Series([212, 332]) + localities = pd.DataFrame([(212, "NY"), (332, "NY")], columns=["area_code", "state"]) + result = pd.DataFrame([("10001", [212, 332], 100.0)], columns=["zipcode", "area_codes", "count"]) + coverage = reconcile_area_codes(inputs, localities, {"NY"}, result) + assert coverage.output_zipcodes.tolist() == [1, 1] + assert coverage.output_population.tolist() == [50.0, 50.0] + assert coverage.output_population.sum() == result["count"].sum() + + +def test_area_code_reconciliation_rejects_new_output_codes(geography: tuple[pd.DataFrame, pd.DataFrame]) -> None: + data, localities = geography + result = data[["zipcode", "count"]].assign(area_codes=[[999], [999]]) + with pytest.raises(ValueError, match="not a subset"): + reconcile_area_codes(pd.Series([413, 212]), localities, {"MA", "NY"}, result) + + +@pytest.fixture +def census_geometry() -> bytes: + rings = [ + [(0, 0), (4, 0), (4, 4), (0, 4), (0, 0)], + [(1, 1), (3, 1), (3, 3), (1, 3), (1, 1)], + [(5, 5), (7, 5), (7, 7), (5, 7), (5, 5)], + ] + points = np.concatenate(rings).astype("i", shp, 0, 9994) + struct.pack_into(">i", shp, 24, (108 + len(content)) // 2) + struct.pack_into("ii", 1, len(content) // 2) + content + dbf = bytearray(65) + dbf[0] = 3 + struct.pack_into(" None: + states = load_state_polygons(census_geometry) + points = pd.DataFrame([(0, 0), (0.5, 0.5), (2, 2), (1, 2), (4, 4), (6, 6), (8, 8)], columns=["x", "y"]) + assert states_for(points, "x", "y", states).tolist() == ["MA", "MA", "", "MA", "MA", "MA", ""] + + +@pytest.mark.parametrize("damage", ["shape_type", "truncated_shape", "missing_field", "deleted_record", "open_ring"]) +def test_census_geometry_rejects_unsupported_input(census_geometry: bytes, damage: str) -> None: + with ZipFile(io.BytesIO(census_geometry)) as archive: + shp = bytearray(archive.read("tl_2023_us_state.shp")) + dbf = bytearray(archive.read("tl_2023_us_state.dbf")) + if damage == "shape_type": + struct.pack_into(" None: + states = load_state_polygons(census_geometry) + states.append(("ZZ", [np.array([(4, 0), (8, 0), (8, 4), (4, 4), (4, 0)])])) + points = pd.DataFrame([(0.5, 0.5), (2, 2), (4, 2), (6, 2)] * 20, columns=["x", "y"]) + expected = ["MA", "", "ZZ", "ZZ"] * 20 + assert states_for(points, "x", "y", states).tolist() == expected + assert states_for(points.iloc[::-1], "x", "y", states).tolist() == expected[::-1] diff --git a/scripts/person_data/wikidata-postcode-query.sparql b/scripts/person_data/wikidata-postcode-query.sparql new file mode 100644 index 000000000..5d83f6abe --- /dev/null +++ b/scripts/person_data/wikidata-postcode-query.sparql @@ -0,0 +1,4 @@ +SELECT DISTINCT ?place ?postcode ?code WHERE { + VALUES ?country { wd:Q30 wd:Q1183 } + ?place wdt:P17 ?country; wdt:P473 ?code; wdt:P281 ?postcode. +} diff --git a/scripts/person_data/wikidata-query.sparql b/scripts/person_data/wikidata-query.sparql new file mode 100644 index 000000000..0a1993261 --- /dev/null +++ b/scripts/person_data/wikidata-query.sparql @@ -0,0 +1,3 @@ +SELECT ?place ?code ?coord WHERE { + ?place wdt:P17 wd:Q30; wdt:P473 ?code; wdt:P625 ?coord. +}