diff --git a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE index aca1a9dc6..2bd7fef93 100644 --- a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE +++ b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.NOTICE @@ -16,18 +16,18 @@ without attribution or share-alike requirements. https://www.wikidata.org/wiki/Wikidata:Licensing https://creativecommons.org/publicdomain/zero/1.0/ -Transformation: normalize locality statements, remove curated exclusions, -prefer nearby same-state literal postal-code associations, and otherwise use +Transformation: normalize locality statements, prefer nearby same-state +literal postal-code associations, and otherwise use the nearest same-state locality point. Retain the candidate codes at that point. Sample ZIPs by Census population, then choose uniformly among their candidates. Candidates are locality-level approximations, not exact telephone boundaries or measured telephone market shares. Each ZIP's population weight is stored once. -NANPA's 2026-11-06 NPA report was consulted only as a screening reference. -No explicit open-data license was established for that report. It is not -redistributed and is not required to rebuild from the pinned snapshots. +Future data improvements include finding additional permissively licensed +numbering references to validate active status, geographic use, and state +assignments. -Source snapshots, queries, exclusions, and permissions (immutable input revision): -https://github.com/NVIDIA-NeMo/DataDesigner/tree/eae83186b32d7c62ee880c6f538bda46a6831c72/scripts/person_data +Wikidata source snapshots (immutable revision): +https://github.com/NVIDIA-NeMo/DataDesigner/tree/43b49ab5cb4353d8a16f0cb9813bc20fc0eea1b8/scripts/person_data/snapshots The parquet key-value metadata contains the source manifest, its SHA-256, the rebuild script SHA-256, and the sampling policy under data_designer.provenance. diff --git a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet index 1388ab213..ba5f53843 100644 Binary files a/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet and b/packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet differ diff --git a/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py b/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py index 1b8534720..e75028aab 100644 --- a/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py +++ b/packages/data-designer-engine/tests/engine/sampling_gen/entities/test_phone_number_data.py @@ -35,14 +35,11 @@ def test_phone_data_provenance() -> None: assert lineage["format_version"] == 2 assert lineage["candidate_selection"] == "population_weighted_zip_then_uniform_area_code" assert lineage["source_manifest"] == json.loads(manifest_bytes) + assert set(lineage["source_manifest"]) == {"sources"} assert lineage["source_manifest_sha256"] == hashlib.sha256(manifest_bytes).hexdigest() - assert ( - lineage["source_manifest"]["excluded_localities_sha256"] - == hashlib.sha256((PROVENANCE / "excluded_localities.csv").read_bytes()).hexdigest() - ) assert lineage["rebuild_script_sha256"] == hashlib.sha256((PROVENANCE / "rebuild.py").read_bytes()).hexdigest() assert re.fullmatch( - r"https://github.com/NVIDIA-NeMo/DataDesigner/tree/[0-9a-f]{40}/scripts/person_data", + r"https://github.com/NVIDIA-NeMo/DataDesigner/tree/[0-9a-f]{40}/scripts/person_data/snapshots", lineage["source_revision_url"], ) assert lineage["source_revision_url"] in ASSET.with_suffix(".NOTICE").read_text() diff --git a/scripts/person_data/README.md b/scripts/person_data/README.md index 91463460f..5ae702939 100644 --- a/scripts/person_data/README.md +++ b/scripts/person_data/README.md @@ -28,15 +28,7 @@ are not used. These data permissions impose no noncommercial or share-alike restriction on the generated lookup. The repository's Apache-2.0 license covers the rebuild code; it does not replace the source-data terms. -The [NANPA NPA database](https://www.nanpa.com/reports/npa-reports) was consulted -on 2026-11-06 to screen locality associations against country, in-service status, -and state. No explicit open-data license was established for that report, so -it is **not described as CC0 or public domain**. The report is not redistributed -and is not an input required by the rebuild. Its URL, date, and checksum are -recorded as a validation reference. `excluded_localities.csv` records the 92 -rejected Wikidata rows using Wikidata fields and Census-derived states, rather -than copying the NANPA database. Regeneration reproduces this pinned curation; -it does not recheck current numbering assignments. +The lookup is derived from the Census and Wikidata inputs listed above. No third-party GitHub ZIP/area-code compilation is used. @@ -46,7 +38,7 @@ No third-party GitHub ZIP/area-code compilation is used. 2. Assign states using the full Census state polygons. A reader for the pinned Polygon/DBF format and a NumPy point-in-polygon calculation handle multipart polygons, holes, and boundaries without additional geometry packages. Normalize - Wikidata three-digit dialing codes and coordinates, then remove the exclusions. + Wikidata three-digit dialing codes and coordinates. 3. Prefer literal five-digit Wikidata postal-code associations when the locality lies in the same state and within 50 km of the ZCTA point. Postal-code ranges and wildcards are not expanded. @@ -62,28 +54,35 @@ No third-party GitHub ZIP/area-code compilation is used. among its candidates. An exact-ZIP request skips the ZIP draw. Having more candidates does not increase a ZIP's probability of being sampled. -The checked-in build has 18,749 explicit postal-code associations and 15,023 -nearest-locality fallbacks. All ZIP identifiers, representative codes, and population -weights match the initial rebuild in this PR. Candidate lists contain 37,239 -ZIP/code associations, and 2,322 ZIPs have more than one candidate. Agreement with -the original repository asset is not an accuracy measure. +The checked-in build has 18,786 explicit postal-code associations and 14,986 +nearest-locality fallbacks. All ZIP identifiers and population weights match the +previous build. Candidate lists change for 213 ZIPs (0.63%) and representative +codes change for 152 ZIPs. Candidate lists contain 37,294 +ZIP/code associations, and 2,374 ZIPs have more than one candidate. Agreement with +the previous asset is not an accuracy measure. This is a regional approximation for synthetic data, not a telephone service-area or address lookup. A city's candidate list can contain both overlays and geographic splits. Candidates are plausible at the chosen locality point, not verified for each postal address or exact ZIP boundary. Uniform candidate selection does not -estimate actual telephone market shares. There are 348 fallback ZIPs +estimate actual telephone market shares. There are 335 fallback ZIPs more than 50 km from their donor locality; the largest distance is about 231 km. -State-level screening does not establish ZIP-level accuracy. Population estimates -and numbering statements reflect the pinned source dates. +Population estimates and numbering statements reflect the pinned source dates. The code-level reconciliation counts 386 distinct three-digit source values, -358 codes with retained localities, 355 with localities in the covered ZCTA -states/territories, and 333 represented in candidate lists. Retaining alternatives -recovers 56 codes previously discarded, including 332, 726, and Chicago's 773 and -872. There are still 22 eligible but unselected codes. They have no selected -locality point under this method. These omissions are reported, not treated as -evidence that the codes are invalid. +386 codes with retained localities, 377 with localities in the covered ZCTA +states/territories, and 352 represented in candidate lists. There are 25 source +codes with covered localities but no selected locality point under this method. +These counts account for source statements; they do not establish numbering +validity. Omissions are reported without treating the codes as invalid. + +## Future data improvements + +Find additional permissively licensed numbering references to validate dialing +codes for active status, geographic use, and state assignments. Further work +includes refining ZIP-level boundaries, improving code coverage, and estimating +candidate usage proportions. Each additional reference needs documented terms, +provenance, and reproducible input snapshots. ## Rebuild @@ -135,14 +134,14 @@ The recorded build used Python 3.12, NumPy 2.4.3, pandas 3.0.5, SciPy 1.17.1, and PyArrow 24.0.0. The parquet's `data_designer.provenance` key-value metadata embeds the complete -source manifest (URLs, hashes, permissions, and screening reference), the manifest -SHA-256, the rebuild script SHA-256, and the ZIP/candidate sampling policy. The packaged NOTICE links to the immutable -input revision, so an older wheel retains a stable source reference. Refresh that -link and `SOURCE_REVISION_URL` when adopting a new committed input revision. +source manifest (URLs, hashes, and permissions), the manifest SHA-256, the rebuild +script SHA-256, and the ZIP/candidate sampling policy. The packaged NOTICE links +to the immutable Wikidata snapshot revision; Census source URLs and hashes are in +the embedded manifest. Refresh that link and `SOURCE_REVISION_URL` when adopting +new committed Wikidata snapshots. To update sources, rerun the two `.sparql` queries, replace their compressed -snapshots and hashes, review the exclusions against a newly recorded numbering -reference, and update the Census URLs/hashes if changing vintage. Rebuild, +snapshots and hashes, and update the Census URLs/hashes if changing vintage. Rebuild, inspect lineage and coverage, and update the build report and data notice. ## Source acknowledgment diff --git a/scripts/person_data/area_code_reconciliation.csv b/scripts/person_data/area_code_reconciliation.csv index 436ba55af..92194718b 100644 --- a/scripts/person_data/area_code_reconciliation.csv +++ b/scripts/person_data/area_code_reconciliation.csv @@ -1,46 +1,46 @@ area_code,input_records,retained_locality_records,eligible_locality_records,output_zipcodes,output_population,status -201,30,29,29,98,1695304.0,selected +201,30,30,30,99,1696158.0,selected 202,30,30,30,57,672079.0,selected 203,68,68,68,87,1393767.0,selected -205,168,168,168,188,1534246.0,selected +205,168,168,168,186,1532089.0,selected 206,41,41,41,42,567041.5,selected -207,852,852,852,427,1377734.0,selected -208,249,247,247,280,1893304.0,selected +207,852,852,852,425,1354799.0,selected +208,249,249,249,284,1891552.5,selected 209,101,101,101,137,1880553.0,selected 210,9,9,9,69,1156174.0,selected -212,55,55,55,70,638189.25,selected +212,55,55,55,70,585866.75,selected 213,3,3,3,63,377714.5,selected 214,7,7,7,73,850862.6666666666,selected 215,18,18,18,171,2108491.0,selected -216,29,28,28,41,995011.0,selected +216,29,29,29,42,995276.0,selected 217,136,136,136,350,1048108.5,selected -218,232,232,232,292,797314.0,selected +218,232,232,232,291,797049.0,selected 219,72,72,72,68,840925.0,selected 220,1,1,1,4,6160.5,selected 223,4,4,4,61,222153.0,selected 224,4,4,4,13,161519.5,selected -225,44,43,43,89,864103.5,selected +225,44,44,44,91,895223.5,selected 228,29,29,29,27,403017.0,selected -229,109,109,109,128,714067.0,selected +229,109,109,109,127,712886.0,selected 231,48,48,48,136,687894.0,selected 234,2,2,2,25,150196.5,selected 235,1,1,1,2,1246.0,selected -236,1,0,0,0,0.0,no_retained_locality +236,1,1,0,0,0.0,outside_zcta_states 239,62,62,62,58,1206864.0,selected 240,91,91,91,172,1261527.8,selected 248,30,30,30,73,1374096.75,selected -250,1,0,0,0,0.0,no_retained_locality +250,1,1,0,0,0.0,outside_zcta_states 251,84,84,84,104,749718.0,selected 252,147,147,147,198,1070963.0,selected 253,62,62,62,58,1269771.0,selected 254,89,89,89,131,1003984.0,selected -256,137,137,137,179,1564460.0,selected -257,1,0,0,0,0.0,no_retained_locality +256,137,137,137,177,1554385.0,selected +257,1,1,0,0,0.0,outside_zcta_states 260,66,66,66,93,732688.0,selected 262,85,85,85,103,1216795.0,selected 267,3,3,3,55,830644.0,selected 269,31,31,31,98,857116.0,selected -270,70,70,70,221,941466.5,selected +270,70,70,70,219,923062.5,selected 272,2,2,2,13,61779.0,selected 274,1,1,1,0,0.0,not_selected 276,51,51,51,125,465373.0,selected @@ -48,33 +48,33 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 281,47,47,47,157,3072186.7,selected 283,1,1,1,0,0.0,not_selected 301,121,121,121,231,1811124.8,selected -302,78,77,77,68,1006247.0,selected -303,26,26,26,118,2077404.6666666667,selected +302,78,78,78,69,1007401.0,selected +303,26,26,26,118,1877619.6666666667,selected 304,217,217,217,738,1705155.0,selected 305,74,74,74,95,2029129.0,selected 307,203,203,203,176,564413.0,selected 308,71,71,71,193,360503.0,selected 309,80,80,80,205,975700.5,selected -310,16,16,16,107,1448462.1666666667,selected -312,4,3,3,56,897075.3333333334,selected +310,16,16,16,106,1416289.1666666667,selected +312,4,4,4,57,929248.3333333334,selected 313,18,18,18,49,1161381.0,selected 314,56,56,56,54,1115598.0,selected -315,204,204,204,314,1440713.0,selected +315,204,204,204,307,1409089.0,selected 316,26,26,26,58,612536.0,selected -317,53,53,53,82,1827586.0,selected -318,169,168,168,209,1125622.0,selected -319,64,64,64,175,793459.0,selected +317,53,53,53,82,1338174.5,selected +318,169,169,169,219,1165208.0,selected +319,64,64,64,168,779574.0,selected 320,162,162,162,160,581609.0,selected 321,29,29,29,63,1154504.0,selected 323,16,16,16,87,1252435.1666666667,selected -325,55,54,54,103,461764.0,selected +325,55,55,55,105,466286.0,selected 326,1,1,1,1,284.0,selected 327,1,1,1,0,0.0,not_selected 329,1,1,1,0,0.0,not_selected 330,25,25,25,205,1631919.5,selected 331,1,1,1,15,235152.0,selected 332,1,1,1,56,326286.25,selected -334,150,149,149,185,1205713.0,selected +334,150,150,150,187,1224117.0,selected 336,71,71,71,145,1847267.0,selected 337,70,70,70,112,949012.0,selected 339,4,4,4,19,172263.5,selected @@ -82,24 +82,24 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 341,1,1,1,0,0.0,not_selected 346,3,3,3,97,658946.7,selected 347,7,7,7,57,812461.3333333334,selected -349,1,0,0,0,0.0,no_retained_locality +349,1,1,1,1,853.0,selected 350,1,1,1,1,2253.0,selected 352,120,120,120,126,1677904.0,selected 357,1,1,1,0,0.0,not_selected -360,525,524,524,216,2249084.5,selected +360,525,525,525,216,2249084.5,selected 361,62,62,62,116,774012.0,selected -363,6,6,6,19,164028.5,selected +363,6,6,6,18,155177.0,selected 364,7,7,7,27,213848.5,selected 380,1,1,1,0,0.0,not_selected -384,1,0,0,0,0.0,no_retained_locality +384,1,1,1,1,12467.0,selected 385,19,19,19,53,682646.0,selected -386,58,58,58,70,950185.0,selected -393,2,0,0,0,0.0,no_retained_locality +386,58,58,58,69,937718.0,selected +393,2,2,2,4,19780.0,selected 401,43,43,43,81,1095391.0,selected -402,135,133,133,393,1454988.0,selected +402,135,135,135,392,1462920.0,selected 404,9,9,9,51,628465.3333333334,selected -405,116,116,116,183,1912177.0,selected -406,184,183,183,369,1105740.0,selected +405,116,116,116,181,1909732.0,selected +406,184,184,184,372,1119723.5,selected 407,65,65,65,66,1631275.0,selected 408,16,16,16,49,1149360.0,selected 409,45,45,45,67,642668.0,selected @@ -116,7 +116,7 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 430,4,4,4,6,33844.0,selected 432,33,33,33,60,479969.0,selected 434,46,46,46,140,721434.0,selected -435,189,189,189,215,850174.0,selected +435,189,189,189,214,849658.0,selected 436,1,1,1,3,12930.5,selected 440,26,26,26,74,1127433.5,selected 442,2,2,2,1,0.0,selected @@ -124,10 +124,10 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 445,2,2,2,5,39226.0,selected 447,2,2,2,11,72250.5,selected 448,36,36,36,14,22591.0,selected -453,1,0,0,0,0.0,no_retained_locality -456,1,0,0,0,0.0,no_retained_locality +453,1,1,1,1,516.0,selected +456,1,1,1,2,3085.0,selected 457,1,1,1,0,0.0,not_selected -458,550,548,548,119,426747.5,selected +458,550,550,550,121,426880.1666666667,selected 463,1,1,1,0,0.0,not_selected 464,1,1,1,0,0.0,not_selected 469,5,5,5,69,652393.6666666666,selected @@ -138,30 +138,30 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 480,22,22,22,97,2301780.25,selected 483,1,1,1,0,0.0,not_selected 484,2,2,2,21,157124.83333333334,selected -486,1,0,0,0,0.0,no_retained_locality -501,111,110,110,121,990765.0,selected +486,1,1,1,38,489411.5,selected +501,111,111,111,122,990929.0,selected 502,55,55,55,101,1260079.0,selected -503,418,416,416,155,1482247.0,selected +503,418,418,418,155,1482247.0,selected 504,13,13,13,52,689834.5,selected -505,41,41,41,157,1370689.0,selected -507,237,236,236,240,839753.0,selected +505,41,41,41,155,1367345.0,selected +507,237,237,237,243,841163.0,selected 508,11,11,11,164,1540428.0,selected -509,462,461,461,248,1862724.0,selected +509,462,462,462,247,1861448.0,selected 510,31,31,31,55,1686242.0,selected 512,33,33,33,91,2309709.0,selected -513,18,18,18,93,1606474.0,selected +513,18,18,18,91,1592918.0,selected 515,53,53,53,140,892496.0,selected -516,20,20,20,75,1398673.5,selected +516,20,20,20,74,1389822.0,selected 517,28,28,28,109,961533.75,selected -518,183,183,183,322,1379116.0,selected -519,1,0,0,0,0.0,no_retained_locality +518,183,183,183,319,1369327.0,selected +519,1,1,1,3,9789.0,selected 520,173,173,173,147,1909131.25,selected -530,134,133,133,238,1352133.5,selected +530,134,134,134,238,1352133.5,selected 531,2,2,2,25,151362.0,selected -534,1,1,1,33,76022.5,selected +534,1,1,1,21,41275.5,selected 539,11,11,11,76,375014.0,selected 540,76,76,76,274,2094068.0,selected -541,769,767,767,273,1301254.5,selected +541,769,769,769,275,1301387.1666666667,selected 551,2,2,2,7,52179.0,selected 557,1,1,1,0,0.0,not_selected 559,98,98,98,111,1777565.0,selected @@ -173,62 +173,62 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 570,96,96,96,366,1510942.0,selected 571,5,5,5,65,886610.5,selected 572,1,1,1,0,0.0,not_selected -573,168,168,168,315,1196797.0,selected +573,168,168,168,312,1193295.0,selected 574,64,64,64,86,722612.0,selected -575,45,44,44,216,746106.0,selected +575,45,45,45,216,746106.0,selected 580,270,270,270,353,1050387.0,selected 582,2,2,2,19,25907.0,selected 585,92,92,92,141,1062260.0,selected 586,14,14,14,40,864477.0,selected -590,1,0,0,0,0.0,no_retained_locality +590,1,1,1,1,1276.0,selected 601,113,113,113,165,1358123.0,selected -602,3,2,2,43,363748.25,selected -603,216,216,216,246,1387500.0,selected -605,378,371,371,375,897979.0,selected +602,3,3,3,43,363748.25,selected +603,216,216,216,245,1386360.0,selected +605,378,378,378,402,968556.0,selected 606,97,97,97,364,897133.0,selected -607,138,138,138,206,831766.0,selected -608,360,360,360,318,1749167.0,selected -609,44,43,43,112,1304381.5,selected -610,29,28,28,226,2581691.8333333335,selected -611,2,0,0,0,0.0,no_retained_locality -612,6,5,5,34,797400.0,selected +607,138,138,138,204,828299.0,selected +608,360,360,360,291,1674504.0,selected +609,44,44,44,118,1310794.5,selected +610,29,29,29,227,2588973.8333333335,selected +611,2,2,2,0,0.0,not_selected +612,6,6,6,34,797400.0,selected 614,25,25,25,69,1723175.0,selected 615,55,55,55,107,1850092.0,selected 616,17,17,17,52,1029901.0,selected 617,7,7,7,72,1458474.5,selected 618,127,127,127,318,1191996.5,selected 619,27,27,27,73,1578285.0,selected -620,221,220,220,265,609661.0,selected +620,221,221,221,264,591305.0,selected 621,1,1,1,94,616910.2,selected -622,1,0,0,0,0.0,no_retained_locality +622,1,1,1,2,1873.0,selected 623,18,18,18,80,1573589.25,selected 624,1,1,1,1,1046.5,selected 626,35,35,35,42,1403788.0,selected -627,1,0,0,0,0.0,no_retained_locality +627,1,1,1,1,17703.0,selected 628,2,2,2,29,418160.5,selected 629,1,1,1,6,113114.0,selected -630,9,8,8,58,1379779.0,selected +630,9,9,9,59,1391175.5,selected 631,23,23,23,110,1525226.0,selected -635,1,0,0,0,0.0,no_retained_locality +635,1,1,1,1,11396.5,selected 636,42,42,42,65,976996.0,selected -639,1,0,0,0,0.0,no_retained_locality +639,1,1,1,1,104.0,selected 640,1,1,1,9,31807.5,selected 641,72,72,72,250,470730.0,selected -642,1,0,0,0,0.0,no_retained_locality +642,1,1,1,1,446.0,selected 646,17,17,17,58,333885.25,selected 650,36,36,36,39,954535.0,selected 651,70,70,70,69,1181181.0,selected -655,1,0,0,0,0.0,no_retained_locality +655,1,1,1,2,2649.0,selected 657,1,1,1,4,87538.0,selected 659,1,1,1,0,0.0,not_selected 660,167,167,167,229,434415.0,selected 661,55,55,55,60,1512997.0,selected -662,181,181,181,235,1158482.0,selected +662,181,181,181,233,1156609.0,selected 667,4,4,4,54,421944.13333333336,selected 669,3,3,3,32,553094.0,selected -670,2,0,0,0,0.0,no_retained_locality +670,2,2,0,0,0.0,outside_zcta_states 671,1,1,0,0,0.0,outside_zcta_states -672,1,0,0,0,0.0,no_retained_locality +672,1,1,0,0,0.0,outside_zcta_states 678,4,4,4,45,420698.6666666667,selected 679,2,2,2,0,0.0,not_selected 680,3,3,3,2,8683.0,selected @@ -236,33 +236,33 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 682,2,2,2,46,573491.1666666666,selected 684,2,2,0,0,0.0,outside_zcta_states 689,1,1,1,0,0.0,not_selected -701,401,400,400,388,778556.0,selected -702,21,20,20,76,2141300.0,selected +701,401,401,401,396,791295.0,selected +702,21,21,21,90,2341085.0,selected 703,7,7,7,79,1293539.5,selected 704,71,71,71,122,2467616.0,selected -706,134,132,132,197,1998223.0,selected -707,118,118,118,151,1473463.0,selected +706,134,134,134,202,2018900.0,selected +707,118,118,118,151,1473359.0,selected 708,82,82,82,83,1601918.0,selected 712,87,87,87,270,556654.0,selected 713,15,15,15,110,1251121.7,selected 714,9,9,9,44,1712647.0,selected -715,7,7,7,109,644854.5,selected +715,7,7,7,94,602449.5,selected 716,64,64,64,149,1304519.5,selected 717,19,19,19,196,1508027.0,selected -718,13,13,13,121,4335015.333333333,selected +718,13,13,13,121,4209775.333333333,selected 719,119,119,119,168,1148368.0,selected -720,7,6,6,62,1175307.6666666667,selected -722,1,0,0,0,0.0,no_retained_locality +720,7,7,7,64,1177085.6666666667,selected +722,1,1,1,11,197327.0,selected 724,39,39,39,353,1515985.6666666667,selected -725,5,4,4,12,152483.0,selected +725,5,5,5,13,174564.0,selected 726,1,1,1,59,927090.0,selected 727,45,45,45,56,1197710.0,selected 730,1,1,1,4,1828.5,selected -731,85,85,85,112,518297.0,selected -732,30,29,29,109,1948107.5,selected +731,85,85,85,110,515212.0,selected +732,30,30,30,113,2073347.5,selected 734,26,26,26,55,1106111.0,selected 737,1,1,1,0,0.0,not_selected -740,38,38,38,291,1319218.5,selected +740,38,38,38,289,1314808.5,selected 747,5,5,5,88,843052.5,selected 748,1,1,1,1,512.0,selected 754,1,1,1,7,172985.0,selected @@ -270,20 +270,20 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 760,79,79,79,135,1989949.0,selected 762,3,3,3,9,105422.0,selected 763,55,55,55,37,674605.0,selected -765,201,201,201,213,1127447.0,selected +765,201,201,201,212,1126436.0,selected 769,3,3,3,4,31695.0,selected -770,86,85,85,203,4998632.666666667,selected -772,29,29,29,31,672502.0,selected +770,86,86,86,203,4998632.666666667,selected +772,29,29,29,20,475175.0,selected 773,3,3,3,56,897075.3333333334,selected -774,3,2,2,35,250527.0,selected -775,57,56,56,104,846876.0,selected -778,1,0,0,0,0.0,no_retained_locality +774,3,3,3,36,251708.0,selected +775,57,57,57,104,846610.6666666666,selected +778,1,1,0,0,0.0,outside_zcta_states 779,3,3,3,9,10350.5,selected 781,7,7,7,59,978996.5,selected -785,231,231,231,307,780902.0,selected -786,5,3,3,61,977257.0,selected +785,231,231,231,306,780456.0,selected +786,5,5,5,62,1015015.5,selected 787,9,9,9,132,3006915.5,selected -800,1,0,0,0,0.0,no_retained_locality +800,1,1,1,0,0.0,not_selected 801,52,52,52,85,1811644.0,selected 802,256,256,256,265,645254.0,selected 803,110,110,110,149,1756437.0,selected @@ -292,14 +292,14 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 806,121,121,121,180,881959.0,selected 808,148,148,148,97,1445635.0,selected 810,22,22,22,62,670089.75,selected -812,190,190,190,265,1345239.5,selected +812,190,190,190,264,1345149.5,selected 813,48,48,48,65,1780976.0,selected 814,96,96,96,436,1297365.0,selected -815,59,58,58,259,1663801.5,selected +815,59,59,59,271,1710331.5,selected 816,87,87,87,154,1389068.0,selected 817,44,44,44,93,1767122.1666666667,selected 818,14,14,14,126,1823383.5,selected -820,4,1,1,3,29686.5,selected +820,4,4,4,5,30787.5,selected 821,2,2,2,2,3900.5,selected 828,89,89,89,130,1186881.0,selected 830,82,82,82,122,866123.0,selected @@ -325,18 +325,18 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 862,2,2,2,10,163822.5,selected 863,72,72,72,65,1136410.0,selected 864,65,65,65,121,1543525.5,selected -865,32,32,32,72,1028755.0,selected -870,329,328,328,379,1106081.0,selected +865,32,32,32,69,1013248.0,selected +870,329,329,329,378,1105915.0,selected 872,2,2,2,56,897075.3333333334,selected -877,1,0,0,0,0.0,no_retained_locality +877,1,1,1,1,7282.0,selected 878,2,2,2,60,261775.66666666666,selected -885,1,0,0,0,0.0,no_retained_locality -888,1,0,0,0,0.0,no_retained_locality +885,1,1,1,3,15507.0,selected +888,1,1,1,0,0.0,not_selected 901,25,25,25,55,1026506.0,selected 903,153,153,153,222,1428434.0,selected 904,39,39,39,63,1650501.0,selected -906,26,26,26,135,310975.0,selected -907,98,97,97,245,733346.0,selected +906,26,26,26,129,304562.0,selected +907,98,98,98,245,733346.0,selected 908,23,23,23,79,1009658.0,selected 909,34,34,34,68,1904719.3333333333,selected 910,136,136,136,158,1682177.0,selected @@ -346,19 +346,19 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 915,25,25,25,55,882756.0,selected 916,17,17,17,63,1803153.0,selected 917,12,12,12,112,1103007.5833333333,selected -918,14,14,14,114,657650.0,selected +918,14,14,14,113,645553.0,selected 919,49,49,49,100,1949247.5,selected -920,218,218,218,216,1256190.0,selected +920,218,218,218,203,1241039.0,selected 924,1,1,1,0,0.0,not_selected 925,46,46,46,36,1127028.0,selected -927,1,0,0,0,0.0,no_retained_locality +927,1,1,1,10,44442.0,selected 928,393,393,393,178,1118343.0,selected 929,3,3,3,8,212049.0,selected 930,6,6,6,20,215254.5,selected 931,72,72,72,143,1030631.0,selected 934,1,1,1,4,52147.0,selected 936,58,58,58,101,890746.0,selected -937,30,30,30,179,1441585.0,selected +937,30,30,30,173,1415109.0,selected 938,1,1,1,0,0.0,not_selected 939,2,2,2,20,247969.5,selected 940,74,74,74,95,701461.0,selected @@ -369,14 +369,14 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 951,22,22,22,53,2064580.3333333333,selected 952,40,40,40,49,842300.0,selected 954,3,3,3,29,957846.0,selected -956,70,70,70,72,1681656.0,selected +956,70,70,70,68,1663292.0,selected 959,8,8,8,15,120711.5,selected -970,145,144,144,212,1334473.0,selected -971,352,350,350,104,1028588.0,selected +970,145,145,145,212,1334473.0,selected +971,352,352,352,104,1028588.0,selected 972,68,68,68,158,3500046.6666666665,selected -973,27,25,25,116,1703324.5,selected +973,27,27,27,116,1703324.5,selected 975,1,1,1,0,0.0,not_selected -977,1,0,0,0,0.0,no_retained_locality +977,1,1,1,7,13885.0,selected 978,9,9,9,91,1417452.0,selected 979,58,58,58,91,726456.0,selected 980,2,2,2,2,16174.0,selected @@ -384,4 +384,4 @@ area_code,input_records,retained_locality_records,eligible_locality_records,outp 984,3,3,3,21,364074.5,selected 985,41,41,41,101,992006.0,selected 986,1,1,1,1,18.0,selected -989,52,52,52,192,1015622.0,selected +989,52,52,52,191,992829.0,selected diff --git a/scripts/person_data/build_report.json b/scripts/person_data/build_report.json index f6f252c54..edca502ad 100644 --- a/scripts/person_data/build_report.json +++ b/scripts/person_data/build_report.json @@ -2,55 +2,31 @@ "rows": 33772, "population_total": 335555510.0, "methods": { - "explicit_postcode_association": 18749, - "nearest_locality_same_state": 15023 + "explicit_postcode_association": 18786, + "nearest_locality_same_state": 14986 }, - "candidate_assignments": 37239, - "zipcodes_with_multiple_area_codes": 2322, + "candidate_assignments": 37294, + "zipcodes_with_multiple_area_codes": 2374, "raw_locality_rows": 22913, "noncanonical_rows_excluded": 116, - "curated_locality_rows_excluded": 92, - "accepted_locality_records": 22705, - "accepted_locality_items": 20860, + "accepted_locality_records": 22797, + "accepted_locality_items": 20928, "area_code_coverage": { "input_codes": 386, - "retained_codes": 358, - "eligible_codes": 355, - "output_codes": 333, - "no_retained_locality": [ + "retained_codes": 386, + "eligible_codes": 377, + "output_codes": 352, + "no_retained_locality": [], + "outside_zcta_states": [ 236, 250, 257, - 349, - 384, - 393, - 453, - 456, - 486, - 519, - 590, - 611, - 622, - 627, - 635, - 639, - 642, - 655, - 670, - 672, - 722, - 778, - 800, - 877, - 885, - 888, - 927, - 977 - ], - "outside_zcta_states": [ 340, + 670, 671, - 684 + 672, + 684, + 778 ], "not_selected": [ 274, @@ -68,24 +44,27 @@ 557, 567, 572, + 611, 659, 679, 689, 737, + 800, + 888, 924, 938, 975 ], - "reconciliation_sha256": "6b18cc988b83ef3a2de4c4954a39dc52267b536ecda11aaf230e5299b220954a" + "reconciliation_sha256": "b4f281325c1a647783acae16a39c138f27cea8012ead2ee43c899e39ed9e6237" }, "distant_explicit_associations_excluded": 46, - "nearest_locality_fallbacks_over_50km": 348, + "nearest_locality_fallbacks_over_50km": 335, "locality_distance_km_quantiles": { - "0.5": 6.5035926030747735, - "0.9": 19.572534448154848, - "0.95": 25.88603157156294, - "0.99": 50.42410233449431, + "0.5": 6.4811363499852614, + "0.9": 19.531414536518493, + "0.95": 25.763850358261713, + "0.99": 49.67497193169035, "1.0": 230.84417705249874 }, - "output_sha256": "b6fc7c8c27b008b8f0da89dfa75a4e2e5693cce18ae0c8a0d3527e326e17e5dd" + "output_sha256": "2af85012fadbd44d907d9785f56051639c51ed7b37104d93c704b0cbf328256b" } diff --git a/scripts/person_data/excluded_localities.csv b/scripts/person_data/excluded_localities.csv deleted file mode 100644 index 652c45e99..000000000 --- a/scripts/person_data/excluded_localities.csv +++ /dev/null @@ -1,93 +0,0 @@ -qid,area_code,longitude,latitude,state -Q881120,201,-67.868611111,46.504722222,ME -Q4788559,250,-124.0,54.0, -Q1952785,216,-94.20638889,47.92527778,MN -Q3261921,360,-84.760555555,33.550555555,GA -Q4788610,236,-124.0,54.0, -Q436898,349,-89.58139,37.63111,MO -Q684406,393,-75.483333333,44.7,NY -Q2272332,393,-75.435555555,44.5525,NY -Q27988976,509,-121.77638889,45.69805556,OR -Q2227968,770,-92.39797,46.66355,MN -Q518466,725,-69.963055555,43.910833333,ME -Q7580837,970,-122.329722222,45.518888888,OR -Q2431419,501,-101.281944444,48.971388888,ND -Q342064,605,-107.623611111,35.065833333,NM -Q1783975,605,-89.931666666,44.121666666,WI -Q1887524,605,-89.3643323,45.2478878,WI -Q2304624,605,-71.558611111,44.654166666,NH -Q5817097,605,-89.638333333,42.601111111,WI -Q7589473,605,-89.052777777,44.454166666,WI -Q2582333,605,-91.2522,36.0067,AR -Q2381270,384,-82.340833333,30.021666666,FL -Q286148,627,-73.693333333,40.792777777,NY -Q136503722,639,-123.6,39.8,CA -Q2270773,870,-97.168611111,35.134722222,OK -Q584340,722,-80.335555555,27.438888888,FL -Q2218020,318,-91.796666666,44.4575,WI -Q370523,620,-96.870833333,35.894166666,OK -Q1592949,610,-73.99715,40.722263888,NY -Q2200776,630,-83.83051,42.95725,MI -Q951376,774,-84.5383,32.0608,GA -Q518097,519,-73.65,43.05,NY -Q2200776,635,-83.83051,42.95725,MI -Q6646528,530,-120.297222222,41.994166666,OR -Q1886236,402,-87.869444444,44.029722222,WI -Q3709190,402,-75.098055555,43.096111111,NY -Q7410310,973,-122.974166666,42.491666666,OR -Q7970633,973,-122.016,45.6126,OR -Q3708399,325,-76.341666666,43.123055555,NY -Q602784,701,-88.714166666,45.743611111,WI -Q600237,720,-95.1708,37.9161,KS -Q114222598,971,-122.3337357,47.4631546,WA -Q114358755,971,-122.6507302,48.5728826,WA -Q14682133,458,-121.51944444,41.99805556,CA -Q2572411,458,-117.718333333,41.9975,NV -Q979819,885,-83.700443,36.150854,TN -Q2535145,702,-104.917,39.8389,CO -Q4788610,257,-124.0,54.0, -Q735981,977,-91.861111111,41.994722222,IA -Q67077,208,-87.907178,32.993811,AL -Q67870,208,-86.943126,34.241338,AL -Q1898553,312,-118.253,33.9175,CA -Q51679,670,145.75,15.183333333,MP -Q93022,706,-99.015833333,26.405,TX -Q3150256,670,145.75,15.18333333,MP -Q9049890,706,-76.370277777,42.022222222,NY -Q1630163,732,-73.981086,40.702553,NY -Q2527718,786,-97.7597,38.9694,KS -Q4532801,786,-73.988211111,40.718497222,NY -Q3287686,456,-88.245,36.037222222,TN -Q647127,225,-100.018333333,37.759722222,KS -Q6346,486,-86.158055555,39.768611111,IN -Q2418283,602,-71.768888888,42.743611111,NH -Q1264177,453,-112.751388888,40.231111111,UT -Q14623578,800,-74.010694444,40.710972222,NY -Q1592949,877,-73.99715,40.722263888,NY -Q14682133,541,-121.51944444,41.99805556,CA -Q114222598,503,-122.3337357,47.4631546,WA -Q114358755,503,-122.6507302,48.5728826,WA -Q1976579,334,-85.0669,36.9847,KY -Q2572411,541,-117.718333333,41.9975,NV -Q1506783,590,-120.543,46.4042,WA -Q968951,655,-91.5006,40.0422,MO -Q1842438,507,-98.105,40.431944444,NE -Q483932,406,-116.6,48.29,ID -Q7231843,907,-107.701,38.081,CO -Q5127590,820,-85.3478,39.4333,IN -Q5969554,820,-85.3075,39.41,IN -Q5969554,820,-85.3078,39.4103,IN -Q2303201,622,-90.1364,34.1731,MS -Q1887361,815,-89.742935,45.3180179,WI -Q7546719,302,-75.570738,42.687292,NY -Q3128858,612,-122.428,37.7728,CA -Q549475,575,-123.0,45.5,OR -Q6160569,775,-120.048,39.5094,CA -Q14931631,888,-74.013888888,40.706111111,NY -Q1966409,611,-117.833611111,34.617777777,CA -Q3458813,611,-119.389166666,35.85,CA -Q2673377,642,-97.7664,39.2747,KS -Q949651,927,-83.8292,39.4458,OH -Q4788610,778,-124.0,54.0, -Q4788610,672,-124.0,54.0, -Q2738898,609,-88.333888888,46.098055555,MI diff --git a/scripts/person_data/rebuild.py b/scripts/person_data/rebuild.py index f1d83b6af..edb2efe93 100644 --- a/scripts/person_data/rebuild.py +++ b/scripts/person_data/rebuild.py @@ -29,7 +29,8 @@ / "packages/data-designer-engine/src/data_designer/engine/sampling_gen/entities/assets/zip_area_code_map.parquet" ) SOURCE_REVISION_URL = ( - "https://github.com/NVIDIA-NeMo/DataDesigner/tree/eae83186b32d7c62ee880c6f538bda46a6831c72/scripts/person_data" + "https://github.com/NVIDIA-NeMo/DataDesigner/tree/" + "43b49ab5cb4353d8a16f0cb9813bc20fc0eea1b8/scripts/person_data/snapshots" ) @@ -251,9 +252,6 @@ def reconcile_area_codes( def rebuild(cache: Path, output: Path, report_dir: Path, offline: bool = False) -> dict: """Rebuild the asset and write row-level lineage and a validation summary.""" manifest = json.loads((ROOT / "sources.json").read_text()) - excluded_bytes = (ROOT / "excluded_localities.csv").read_bytes() - if hashlib.sha256(excluded_bytes).hexdigest() != manifest["excluded_localities_sha256"]: - raise ValueError("Exclusion snapshot checksum mismatch") inputs = {name: read_source(name, source, cache, offline) for name, source in manifest["sources"].items()} states = load_state_polygons(inputs["census-state-boundaries-full-2023.zip"]) @@ -280,11 +278,6 @@ def rebuild(cache: Path, output: Path, report_dir: Path, offline: bool = False) if not localities.latitude.between(-90, 90).all() or not localities.longitude.between(-180, 180).all(): raise ValueError("Invalid locality coordinates") localities["state"] = states_for(localities, "longitude", "latitude", states) - excluded = pd.read_csv(io.BytesIO(excluded_bytes)) - excluded_pairs = pd.MultiIndex.from_frame(excluded[["qid", "area_code"]]) - accepted = ~pd.MultiIndex.from_frame(localities[["qid", "area_code"]]).isin(excluded_pairs) - rejected = localities[~accepted] - localities = localities[accepted].copy() population = pd.read_csv(io.BytesIO(inputs["census-acs5-2023-b01003.dat"]), sep="|", dtype={"GEO_ID": str}) population = population[population.GEO_ID.str.fullmatch(r"860Z200US\d{5}")].copy() @@ -345,7 +338,6 @@ def rebuild(cache: Path, output: Path, report_dir: Path, offline: bool = False) "zipcodes_with_multiple_area_codes": int((result.area_codes.str.len() > 1).sum()), "raw_locality_rows": len(raw), "noncanonical_rows_excluded": len(raw) - len(rows), - "curated_locality_rows_excluded": len(rejected), "accepted_locality_records": len(localities), "accepted_locality_items": int(localities.qid.nunique()), "area_code_coverage": { diff --git a/scripts/person_data/sources.json b/scripts/person_data/sources.json index 5a887f667..d65893cb1 100644 --- a/scripts/person_data/sources.json +++ b/scripts/person_data/sources.json @@ -37,14 +37,5 @@ "license": "CC0-1.0", "license_url": "https://www.wikidata.org/wiki/Wikidata:Licensing" } - }, - "excluded_localities_sha256": "dc1d1ae22bc8ee4ecd2641a8d19b760f7612a1e6e4950c1fb0649b21f8a01e2b", - "validation_reference": { - "name": "NANPA NPA database", - "url": "https://reports.nanpa.com/public/npa_report.csv", - "retrieved": "2026-11-06", - "sha256": "dcbbcd9143d91a19c8573d2ffbb1777cfff10f7f16e86cd52cf2b8930c26b5bb", - "role": "Screening reference for foreign, inactive, and wrong-state Wikidata locality associations; report not redistributed or required for rebuilding.", - "license": "No explicit open-data license established; not claimed to be CC0 or public domain." } }