Skip to content

Commit 6301337

Browse files
committed
fix(ingest): close the draft-run gaps found in the #132 review
Regressions from #132: - CPU: a table with both "Base / Min" and "Base / Max" columns gives no base clock; either one can belong to the nominal TDP (11980HK: Min). - GPU: a laptop-table row named without "Mobile" ("Quadro M5500") is also checked against curated as the -mobile part (new IngestCandidate.aliases). - GPU: an AMD-page board is ATI when every date its launch cell names is before the rebrand ("2004?", "2007 (2008)", two dates). CPU and GPU rows without a release date are never written, as for phones. - GPU backfill: _to_gb has no 1 MiB floor again; the backfill compares values and does not write them. Draft-run gaps that predate #132: - parse_tdp_w: two bare numbers before the unit ("92.6 68 W") give None. - CPU tables drop {{0}} hidden padding, which read "65 W" as "0 65 W". - GPU bus: ISA/EISA/VLB/MCA lists and "AGP 4x,8x" give None. - GPU memory type: a cell naming SGRAM, EDO, VRAM, WRAM, FPM or DRAM gives None. - Dedup: curated names are indexed too ("3D Rage II" is rage-ii.json), smartphone keys fold gen/generation, "XT PE" is XT Platinum and "Quadro4 200 NVS" is NVS 200. Also: GPU names no longer carry the brand, which the dataset leaves to manufacturer, and GT2xx, GF11x and GK2xx map to Tesla 2.0, Fermi 2.0 and Kepler 2.0 as the curated records name them.
1 parent 0c5012b commit 6301337

14 files changed

Lines changed: 276 additions & 34 deletions

‎app/ingest/gpu_identity.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,14 @@ def gpu_family_key(slug: str) -> str:
6868
tokens = _tokens(slug)
6969
if len(tokens) > 1 and tokens[0] == "quadro" and _QUADRO_RENAMED_RE.fullmatch(tokens[1]):
7070
tokens = tokens[1:]
71+
# "Quadro4 200 NVS" is "NVS 200".
72+
if tokens[:1] == ["quadro4"] and tokens[-1:] == ["nvs"]:
73+
tokens = ["nvs", *tokens[1:-1]]
74+
# "X800 XT PE" is the X800 XT Platinum Edition.
75+
tokens = [
76+
"platinum" if token == "pe" and tokens[i - 1 : i] == ["xt"] else token
77+
for i, token in enumerate(tokens)
78+
]
7179
# "Radeon Instinct MI100" became "Instinct MI100".
7280
if tokens[:2] == ["radeon", "instinct"]:
7381
tokens = tokens[1:]

‎app/ingest/normalize.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,16 @@ def parse_tdp_w(text: str) -> int | None:
8585
8686
``"45 W (35–54 W)"`` → ``45``: the default precedes its cTDP range. A cell
8787
that is only a range (``"12–25 W"``) states no default, so it gives ``None``;
88-
neither end of the range is the part's TDP.
88+
neither end of the range is the part's TDP. Two bare numbers before the
89+
unit (``"92.6 68 W"``) are two parts' values and give ``None`` too.
8990
"""
9091
if not text:
9192
return None
92-
match = _TDP_RE.search(_TDP_RANGE_RE.sub(" ", text))
93-
return math.floor(float(match.group(1)) + 0.5) if match else None
93+
cleaned = _TDP_RANGE_RE.sub(" ", text)
94+
match = _TDP_RE.search(cleaned)
95+
if match is None or re.search(r"\d\s+$", cleaned[: match.start()]):
96+
return None # "92.6 68 W": one value per OPN, so neither is the part's
97+
return math.floor(float(match.group(1)) + 0.5)
9498

9599

96100
def parse_frequency_range_ghz(text: str) -> tuple[float, float] | None:

‎app/ingest/pipeline.py‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
import json
1111
from collections import Counter
1212
from collections.abc import Iterable
13-
from dataclasses import dataclass, field
13+
from dataclasses import dataclass, field, replace
1414
from datetime import date
1515
from pathlib import Path
1616
from typing import Any
1717

18+
from app.coverage.normalize import slugify
19+
1820
from .cpu_identity import cpu_key, same_cpu
1921
from .gpu_identity import (
2022
datacenter_family,
@@ -139,6 +141,11 @@ def add(self, record: dict[str, Any], path: str, manufacturer: str) -> None:
139141
if not isinstance(slug, str) or not slug:
140142
return
141143
names: list[object] = [slug]
144+
# The curated name can be spelled unlike the slug: "3D Rage II" is
145+
# rage-ii.json, "iPhone SE (2nd generation)" iphone-se-2020.json.
146+
display = record.get("name")
147+
if isinstance(display, str) and display:
148+
names.append(slugify(display, manufacturer=manufacturer))
142149
if self.category == "smartphone":
143150
# A variant record (".../2022/galaxy-m33-5g/sm-m336b.json") names
144151
# its phone in base_model_slug and in its folder.
@@ -150,14 +157,30 @@ def add(self, record: dict[str, Any], path: str, manufacturer: str) -> None:
150157
if isinstance(name, str) and name:
151158
self.slugs.setdefault(name, path)
152159
self.keys.setdefault(self._key(name), path)
153-
if self.category == "gpu":
154-
self.families.setdefault(gpu_family_key(slug), path)
160+
if self.category == "gpu":
161+
self.families.setdefault(gpu_family_key(name), path)
155162

156163
def _key(self, slug: str) -> str:
157-
return gpu_key(slug) if self.category == "gpu" else slug.replace("-", "")
164+
if self.category == "gpu":
165+
return gpu_key(slug)
166+
tokens = slug.split("-")
167+
if self.category == "smartphone":
168+
# "iPhone SE (2nd)" and "iPhone SE (2nd generation)".
169+
tokens = [token for token in tokens if token not in {"gen", "generation"}]
170+
return "".join(tokens)
158171

159172
def match(self, candidate: IngestCandidate) -> tuple[str, str] | None:
160-
"""``(path, why)`` of the record the candidate duplicates, or ``None``."""
173+
"""``(path, why)`` of the record the candidate or one of its
174+
``aliases`` duplicates, or ``None``."""
175+
if (found := self._match(candidate)) is not None:
176+
return found
177+
for alias in candidate.aliases:
178+
if (found := self._match(replace(candidate, slug=alias))) is not None:
179+
path, why = found
180+
return path, f"as {alias}: {why}"
181+
return None
182+
183+
def _match(self, candidate: IngestCandidate) -> tuple[str, str] | None:
161184
target = candidate.output_path.as_posix()
162185
if target in self.paths:
163186
return target, f"path {target} already exists"

‎app/ingest/sources/base.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ class IngestCandidate:
2121
fill but that cannot be trusted; such a candidate is never written, not
2222
even as a draft. ``released_by`` is the last day of the release period the
2323
source states (``"Q4 2026"`` → Dec 31): before it, the part may not have
24-
shipped.
24+
shipped. ``aliases`` are other slugs the part may be curated under (a
25+
laptop-table row named without "Mobile"); dedup checks them too.
2526
"""
2627

2728
category: str
@@ -33,6 +34,7 @@ class IngestCandidate:
3334
missing_fields: tuple[str, ...] = field(default_factory=tuple)
3435
problems: tuple[str, ...] = field(default_factory=tuple)
3536
released_by: date | None = None
37+
aliases: tuple[str, ...] = field(default_factory=tuple)
3638

3739
@property
3840
def is_complete(self) -> bool:

‎app/ingest/sources/cpu_tables.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,10 @@ def parse_cpu_table(table: Tag) -> Iterator[dict[str, str]]:
197197
column += 1
198198
for sup in cell.find_all("sup"):
199199
sup.decompose()
200+
# {{0}} pads numbers with a hidden "0" ("065 W" shows as " 65 W").
201+
for pad in cell.select('[aria-hidden="true"]'):
202+
if "visibility:hidden" in str(pad.get("style", "")).replace(" ", ""):
203+
pad.decompose()
200204
text = re.sub(r"\[[^\]]*\]", "", cell.get_text(" ", strip=True)).strip()
201205
width = min(_span(cell, "colspan"), 64 - column)
202206
for offset in range(width):
@@ -228,6 +232,11 @@ def parse_cpu_table(table: Tag) -> Iterator[dict[str, str]]:
228232
parts = list(dict.fromkeys(row[column] for row in grid[start:end] if column < len(row)))
229233
labels.append(" / ".join(parts))
230234
fields = [_field(label) for label in labels]
235+
# "Base / Min" and "Base / Max" pair with "TDP / Min" and "TDP / Max", and
236+
# either one can be the nominal TDP's base (11980HK: Min, 11800H: Max).
237+
leaves = {label.lower().split(" / ")[-1].strip() for label in labels if "base" in label.lower()}
238+
if {"min", "max"} <= leaves:
239+
fields = [None if field == "base_clock" else field for field in fields]
231240
# A colspan Model heading describes two distinct body columns.
232241
for column in range(len(fields) - 1):
233242
if fields[column : column + 2] == ["model", "model"]:

‎app/ingest/sources/wikipedia_cpu.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,11 @@ def _build_candidate(
323323

324324
required = ("release_date", "architecture", "cores", "threads")
325325
missing = tuple(field for field in required if record.get(field) in (None, ""))
326-
problems = (row["cores_note"],) if row.get("cores_note") else ()
326+
problems: tuple[str, ...] = (row["cores_note"],) if row.get("cores_note") else ()
327+
if release_date is None:
328+
# The layout files a part under its release year; without one there is
329+
# no folder to put it in, even as a draft.
330+
problems += ("no release date to file it under",)
327331

328332
year = release_date.year if release_date else "unknown"
329333
bucket = "enterprise" if segment in ("server", "hedt") else "consumer"

‎app/ingest/sources/wikipedia_gpu.py‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,6 @@
5454
("intel", "List_of_Intel_graphics_processing_units", "Intel Graphics"),
5555
]
5656

57-
_BRAND_DISPLAY: dict[str, str] = {"nvidia": "NVIDIA", "amd": "AMD", "ati": "ATI", "intel": "Intel"}
58-
5957
# The dataset keeps ATI-branded boards under ``ati``; AMD retired the ATI brand
6058
# in August 2010 and the curated records switch to ``amd`` from October 2010.
6159
_ATI_BRAND_END = date(2010, 10, 1)
@@ -69,14 +67,17 @@
6967
"DDR5", "DDR4", "DDR3", "DDR2", "DDR", "SDRAM", "SDR",
7068
)
7169
_MEMORY_TYPE_DISPLAY = {"HBM2E": "HBM2e", "HBM3E": "HBM3e"}
70+
# Types the curated records spell their own way ("SGRAM", "EDO DRAM"): a cell
71+
# naming one of them states no type from MEMORY_TYPES.
72+
_OTHER_MEMORY_RE = re.compile(r"(?<![A-Z0-9])(?:SGRAM|EDO|VRAM|WRAM|FPM|DRAM)(?![A-Z0-9])")
7273

7374
_NUMBER_RE = re.compile(r"\d[\d,]*(?:\.\d+)?")
7475
_RANGE_RE = re.compile(r"(\d[\d,]*(?:\.\d+)?)\s*(?:–|—|-|to)\s*(\d[\d,]*(?:\.\d+)?)")
7576
_FOOTNOTE_RE = re.compile(r"\s*\[[^\]]{1,12}\]")
7677
_TRAILING_PAREN_RE = re.compile(r"\s*\([^)]*\)\s*$")
7778
_MULTI_GPU_RE = re.compile(r"\d\s*[×x]\s*\d")
7879
_BUS_RE = re.compile(r"\b(PCIe|PCI-E|PCI Express|AGP|PCI|MXM(?:-[A-Z]+)?)\b[^/,;]*", re.IGNORECASE)
79-
_BUS_NAME_RE = re.compile(r"\b(PCIe|AGP|PCI|MXM)\b", re.IGNORECASE)
80+
_BUS_NAME_RE = re.compile(r"\b(PCIe|AGP|PCI|MXM|ISA|EISA|VLB|VL-Bus|MCA)\b", re.IGNORECASE)
8081
# Integrated/chipset graphics are out of scope for the discrete-GPU table.
8182
_OUT_OF_SCOPE_RE = re.compile(r"\bIGP\b|integrated|on-?die|\bAPU\b|nForce", re.IGNORECASE)
8283
# Chipset IGPs: a memory cell naming only system memory ("Up to 256 from system
@@ -93,6 +94,7 @@
9394
r"\b(?:mobile|mobility|laptop|notebook|max-q|go)\b|\b[a-z]?\d{2,5}m[a-z]*\b|\bm\d{3,4}|\bmx\s?\d",
9495
re.IGNORECASE,
9596
)
97+
_LAPTOP_TIER_RE = re.compile(r"\b(?:mobile|laptop|notebook)\b")
9698
_BRAND_WORD_RE = re.compile(r"\b(?:geforce|radeon|quadro|firepro|firegl|tesla)\b", re.IGNORECASE)
9799
# "Clock Rate / Base (Boost) ( MHz )": "1291 (1392)" is base 1291, boost 1392.
98100
_BASE_BOOST_RE = re.compile(r"\b(?:base|core)\b[^/]*\(\s*boost|\bbase boost\b")
@@ -128,8 +130,13 @@
128130
(r"NV2\d", "Kelvin"),
129131
(r"NV3\d", "Rankine"),
130132
(r"NV4\d|G7\d", "Curie"),
131-
(r"G8\d|G9\d|GT2\d\d", "Tesla"),
133+
# Curated records name the second generations (GT200, GF11x, GK2xx) per
134+
# chip; the versioned entries come before the generic ones.
135+
(r"GT2\d\d", "Tesla 2.0"),
136+
(r"G8\d|G9\d", "Tesla"),
137+
(r"GF11\d", "Fermi 2.0"),
132138
(r"GF1\d\d", "Fermi"),
139+
(r"GK2\d\d", "Kepler 2.0"),
133140
(r"GK\d", "Kepler"),
134141
(r"GM\d", "Maxwell"),
135142
(r"GP\d", "Pascal"),
@@ -269,6 +276,8 @@ def _memory_gb(text: str, unit: str) -> float | None:
269276
def _memory_type(text: str) -> str | None:
270277
"""The memory type a cell names; ``None`` when it names several."""
271278
upper = text.upper()
279+
if _OTHER_MEMORY_RE.search(upper):
280+
return None
272281
kinds = [
273282
kind
274283
for kind in MEMORY_TYPES
@@ -279,9 +288,16 @@ def _memory_type(text: str) -> str | None:
279288

280289
def _release_period(text: str) -> tuple[date, date] | None:
281290
"""The release period a cell names; ``None`` for several dates or a ``?``."""
282-
cleaned = _FOOTNOTE_RE.sub("", text)
283-
if "?" in cleaned:
291+
if "?" in text:
284292
return None
293+
periods = _cell_periods(text)
294+
return periods.pop() if len(periods) == 1 else None
295+
296+
297+
def _cell_periods(text: str) -> set[tuple[date, date]]:
298+
"""Every date period a cell names: ``"May 1, 2007 (PCIe) October 1, 2007 (AGP)"``
299+
names two, ``"2004?"`` one."""
300+
cleaned = _FOOTNOTE_RE.sub("", text)
285301
taken: list[tuple[int, int]] = []
286302
periods: set[tuple[date, date]] = set()
287303
for pattern in _DATE_IN_CELL_RES:
@@ -291,7 +307,7 @@ def _release_period(text: str) -> tuple[date, date] | None:
291307
taken.append(match.span())
292308
if (period := parse_date_period(match.group(0))) is not None:
293309
periods.add(period)
294-
return periods.pop() if len(periods) == 1 else None
310+
return periods
295311

296312

297313
def normalize_bus_interface(text: str) -> str | None:
@@ -311,6 +327,9 @@ def normalize_bus_interface(text: str) -> str | None:
311327
if not match:
312328
return None
313329
if match.group(1).lower() != "pcie":
330+
# "AGP 4x,8x": board variants again.
331+
if len(set(re.findall(r"(\d+)\s*x\b", cleaned[match.start() :]))) > 1:
332+
return None
314333
value = match.group(0)
315334
value = re.sub(r"\s+x\s*(\d+)", r" x\1", value)
316335
value = re.sub(r"(\d)\s*x\b", r"\1x", value)
@@ -418,18 +437,29 @@ def unit(key: str) -> str:
418437
# "2× 128": per-GPU values on dual-GPU boards don't fit one record.
419438
return None
420439
problems: tuple[str, ...] = ()
421-
if headings and headings[-1].lower().startswith("mobile") and not _LAPTOP_MARKER_RE.search(
422-
raw_model
423-
):
440+
laptop_table = bool(headings) and headings[-1].lower().startswith("mobile")
441+
if laptop_table and not _LAPTOP_MARKER_RE.search(raw_model):
424442
problems = ("laptop table row without a laptop marker in its name",)
425443

426444
release_period = _release_period(text("release_date"))
427445
release_date = release_date_of(release_period)
428-
if manufacturer == "amd" and release_period is not None and release_period[1] < _ATI_BRAND_END:
446+
# Before the rebrand by every date the cell names ("2004?", "May 1, 2007
447+
# (PCIe) October 1, 2007 (AGP)"), even when no one date is the release.
448+
periods = _cell_periods(text("release_date"))
449+
if manufacturer == "amd" and periods and all(last < _ATI_BRAND_END for _, last in periods):
429450
manufacturer = "ati"
430451
slug = slugify(model, manufacturer=manufacturer)
431452
if len(slug) < 4 or not any(ch.isdigit() for ch in slug):
432453
return None
454+
if release_date is None:
455+
# The layout files a board under its release year; without one there
456+
# is no folder to put it in, even as a draft.
457+
problems += ("no release date to file it under",)
458+
# "Quadro M5500" in a laptop table is the laptop part, curated as "Quadro
459+
# M5500 Mobile"; dedup checks that name too.
460+
aliases: tuple[str, ...] = ()
461+
if laptop_table and not _LAPTOP_TIER_RE.search(slug):
462+
aliases = (f"{slug}-mobile",)
433463

434464
base_clock = _clock_mhz(text("base_clock"), unit("base_clock"))
435465
boost_clock = _clock_mhz(
@@ -467,8 +497,8 @@ def unit(key: str) -> str:
467497
architecture_from_codename(text("codename"), manufacturer)
468498
)
469499

470-
brand = _BRAND_DISPLAY.get(manufacturer, manufacturer.title())
471-
name = model if model.lower().startswith(manufacturer) else f"{brand} {model}"
500+
# Dataset GPU names leave the brand to ``manufacturer``: "GeForce GTX 1660 Ti".
501+
name = re.sub(r"^(?:NVIDIA|AMD|ATI|Intel)\s+", "", model, flags=re.IGNORECASE)
472502
record: dict[str, object | None] = {
473503
"slug": slug,
474504
"name": name,
@@ -503,4 +533,5 @@ def unit(key: str) -> str:
503533
missing_fields=missing,
504534
problems=problems,
505535
released_by=release_period[1] if release_period else None,
536+
aliases=aliases,
506537
)

‎app/verify/wikipedia_gpu_backfill.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@
5151
from bs4 import BeautifulSoup, Tag
5252

5353
from app.ingest.normalize import (
54-
mib_to_gb,
5554
parse_date,
5655
parse_frequency_mhz,
5756
parse_memory_bus_bit,
@@ -230,13 +229,15 @@ def crossref_pages() -> list[tuple[str, str, str]]:
230229
# --- name split -----------------------------------------------------------------
231230

232231

233-
def _to_gb(value: float, unit: str) -> float | None:
232+
def _to_gb(value: float, unit: str) -> float:
233+
"""Binary GB, unrounded. Unlike ``mib_to_gb`` there is no 1 MiB floor: that
234+
is a rule for writing a record, and the backfill only compares values."""
234235
unit_l = unit.lower()
235236
if unit_l == "gb":
236237
return value
237238
if unit_l == "mb":
238-
return mib_to_gb(value)
239-
return mib_to_gb(value / 1024)
239+
return value / 1024
240+
return value / (1024 * 1024)
240241

241242

242243
def memory_close(left: float, right: float) -> bool:
@@ -315,13 +316,13 @@ def parse_memory_options(text: str) -> tuple[float, ...]:
315316
if not piece:
316317
continue
317318
value = _to_gb(float(piece), unit)
318-
if value is not None and value not in found:
319+
if value not in found:
319320
found.append(value)
320321
for match in _MEMORY_TOKEN_RE.finditer(text):
321322
if any(start <= match.start() < end for start, end in consumed):
322323
continue
323324
value = _to_gb(float(match.group(1)), match.group(2))
324-
if value is not None and value not in found:
325+
if value not in found:
325326
found.append(value)
326327
return tuple(found)
327328

@@ -419,7 +420,7 @@ def _bare_memory(text: str, unit: str) -> tuple[float, ...]:
419420
found: list[float] = []
420421
for number in numbers:
421422
value = _to_gb(float(number), unit)
422-
if value is not None and value not in found:
423+
if value not in found:
423424
found.append(value)
424425
return tuple(found)
425426

‎docs/DATA_PIPELINE.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,9 @@ over. A field is filled only when its cell states one value: a range, an
8383
estimate, a cell listing variants, or a year or half-year alone leaves it empty
8484
(neither end of a range stands in), and a row that then misses a required
8585
field is skipped. Laptop parts are their own records (`-mobile`), integrated
86-
GPUs are out of scope, and a phone without a release date has no
87-
`<year>` folder, so it is not written. Both run on manual dispatch
86+
GPUs are out of scope, GPU names leave the brand to `manufacturer`, and a
87+
record without a release date has no `<year>` folder, so it is not written
88+
even as a draft. Both run on manual dispatch
8889
and their output goes to TechAPI as a PR for human review before anything
8990
lands (SPEC Section 9.3). The curated
9091
dataset is a **subset, not exhaustive.**

0 commit comments

Comments
 (0)