Skip to content

Commit 458ad31

Browse files
committed
fix(ingest): smartphone collector writes one phone per row, by year
Only wikipedia_smartphone._extract changes; wikitable is shared with the backfills. - Citation markers are stripped. A model cell with "/" or a repeated product word ("iPhone 5 iPhone 5c") names two phones and is skipped. - A row where none of soc, release_date, ram_gb, battery_mah or weight_g parses is not a phone (captions, legends, spec labels). - Under a two-row "Release" header the "Date" leaf is the release date. - Records go to smartphone/<brand>/<year>/<slug>.json; a row without a release date is never written. - The dedup index also knows a variant record's base_model_slug and its folder name. List_of_Xiaomi_smartphones returns 404 and is dropped.
1 parent d9bd93a commit 458ad31

4 files changed

Lines changed: 153 additions & 13 deletions

File tree

‎app/ingest/pipeline.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,18 @@ def add(self, record: dict[str, Any], path: str, manufacturer: str) -> None:
138138
return
139139
if not isinstance(slug, str) or not slug:
140140
return
141-
self.slugs.setdefault(slug, path)
142-
self.keys.setdefault(self._key(slug), path)
141+
names: list[object] = [slug]
142+
if self.category == "smartphone":
143+
# A variant record (".../2022/galaxy-m33-5g/sm-m336b.json") names
144+
# its phone in base_model_slug and in its folder.
145+
names.append(record.get("base_model_slug"))
146+
parts = path.split("/")
147+
if len(parts) == 5:
148+
names.append(parts[3])
149+
for name in names:
150+
if isinstance(name, str) and name:
151+
self.slugs.setdefault(name, path)
152+
self.keys.setdefault(self._key(name), path)
143153
if self.category == "gpu":
144154
self.families.setdefault(gpu_family_key(slug), path)
145155

‎app/ingest/sources/wikipedia_smartphone.py‎

Lines changed: 72 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@
55
SoC catalog, and emit candidates only for phones whose SoC is already
66
curated — the validator enforces ``smartphone.soc`` foreign-key integrity
77
so unknown SoCs would otherwise tank the whole dataset.
8+
9+
A row is a phone only when it names one model and at least one spec parses.
10+
Captions, year dividers, legends and the attribute rows of transposed
11+
comparison tables name no phone; "iPhone 13 Pro / 13 Pro Max" names two.
12+
Records go to ``smartphone/<brand>/<year>/<slug>.json``, so a row without a
13+
release date is never written.
814
"""
915

1016
from __future__ import annotations
@@ -13,7 +19,7 @@
1319
from collections.abc import Iterator
1420
from pathlib import Path
1521

16-
from bs4 import BeautifulSoup
22+
from bs4 import BeautifulSoup, Tag
1723

1824
from app.coverage.curated import curated_slugs
1925
from app.coverage.normalize import slugify
@@ -25,16 +31,16 @@
2531
parse_date_period,
2632
parse_ram_gb,
2733
parse_weight_g,
34+
release_date_of,
2835
)
2936
from .base import IngestCandidate
30-
from .wikitable import parse_table
37+
from .wikitable import _detect_headers, _table_to_grid
3138

3239
PAGES: list[tuple[str, str]] = [
3340
("samsung", "List_of_Samsung_Galaxy_smartphones"),
3441
("apple", "List_of_iPhone_models"),
3542
("google", "Pixel_(smartphone)"),
3643
("oneplus", "List_of_OnePlus_products"),
37-
("xiaomi", "List_of_Xiaomi_smartphones"),
3844
]
3945

4046
# Brand keys are stored lowercase; these are their display forms used to
@@ -59,6 +65,13 @@
5965
# product family (Snapdragon / Dimensity / Exynos / Tensor / Kirin) and drop
6066
# only the company name. Apple is the exception — "Apple A17 Pro" slugs as
6167
# ``a17-pro`` because "Apple" is the family identifier, not a vendor prefix.
68+
# Citation markers left in model cells ("Galaxy Ace 3 [ 35 ]").
69+
_FOOTNOTE_RE = re.compile(r"\s*\[\s*[\w\s]{1,6}\]")
70+
# A product word named twice means two phones in one cell ("iPhone 5 iPhone 5c").
71+
_PRODUCT_WORD_RE = re.compile(r"\b(?:iphone|galaxy|pixel|oneplus|redmi|poco)\b", re.IGNORECASE)
72+
# Without one of these the row is not a phone (a caption, a legend, a spec label).
73+
_SPEC_FIELDS = ("soc", "release_date", "ram_gb", "battery_mah", "weight_g")
74+
6275
_SOC_VENDOR_PREFIXES = (
6376
"qualcomm",
6477
"mediatek",
@@ -110,19 +123,64 @@ def _extract(
110123
soup = BeautifulSoup(html, "html.parser")
111124
source_url = f"https://en.wikipedia.org/wiki/{page}"
112125
for table in soup.select("table.wikitable"):
113-
for row in parse_table(table, HEADER_RULES):
114-
model = row.cells.get("model", "")
126+
for cells in _rows(table):
127+
model = _FOOTNOTE_RE.sub("", cells.get("model", "")).strip()
128+
if "/" in model or len(_PRODUCT_WORD_RE.findall(model)) > 1:
129+
continue # two phones in one cell; splitting would guess the names
115130
slug = slugify(model, manufacturer=brand)
116131
if len(slug) < 3:
117132
continue
118-
yield _build_candidate(
133+
candidate = _build_candidate(
119134
brand=brand,
120135
model=model,
121136
slug=slug,
122-
row=row.cells,
137+
row=cells,
123138
source_url=source_url,
124139
known_socs=known_socs,
125140
)
141+
if any(candidate.record.get(field) is not None for field in _SPEC_FIELDS):
142+
yield candidate
143+
144+
145+
def _rows(table: Tag) -> Iterator[dict[str, str]]:
146+
"""Body rows of ``table`` keyed by field, the way ``wikitable.parse_table``
147+
reads them, and also through a two-row header: under "Release(d)" spanning
148+
"With OS" and "Date", the release date is the "Date" column, and the
149+
second header row is not a phone.
150+
"""
151+
grid = _table_to_grid(table)
152+
detected = _detect_headers(grid, HEADER_RULES) if grid else None
153+
if detected is None:
154+
return
155+
header, mapping = detected
156+
model_column = min(column for column, field in mapping.items() if field == "model")
157+
start = header + 1
158+
if (
159+
start < len(grid)
160+
and model_column < len(grid[start])
161+
and grid[start][model_column] == grid[header][model_column]
162+
):
163+
leaves = grid[start]
164+
dates = {
165+
column
166+
for column, field in mapping.items()
167+
if field == "release_date" and column < len(leaves) and "date" in leaves[column].lower()
168+
}
169+
if dates:
170+
mapping = {
171+
column: field
172+
for column, field in mapping.items()
173+
if field != "release_date" or column in dates
174+
}
175+
start += 1
176+
for row in grid[start:]:
177+
cells: dict[str, str] = {}
178+
for column, text in enumerate(row):
179+
field = mapping.get(column)
180+
if field and text:
181+
cells.setdefault(field, text)
182+
if cells:
183+
yield cells
126184

127185

128186
def soc_text_to_slug(text: str) -> str:
@@ -157,7 +215,7 @@ def _build_candidate(
157215
known_socs: set[str],
158216
) -> IngestCandidate:
159217
release_period = parse_date_period(row.get("release_date", ""))
160-
release_date = release_period[0] if release_period else None
218+
release_date = release_date_of(release_period)
161219
soc_text = row.get("soc", "")
162220
soc_slug = soc_text_to_slug(soc_text)
163221
soc_value = soc_slug if soc_slug in known_socs else None
@@ -191,7 +249,11 @@ def _build_candidate(
191249
required = ("soc", "release_date", "ram_gb", "battery_mah", "weight_g", "os")
192250
missing = tuple(field for field in required if record.get(field) in (None, ""))
193251

194-
output_path = Path("smartphone") / brand / f"{slug}.json"
252+
year = release_date.year if release_date else "unknown"
253+
output_path = Path("smartphone") / brand / str(year) / f"{slug}.json"
254+
# The layout files a phone under its release year; without one there is no
255+
# folder to put it in, even as a draft.
256+
problems = () if release_date else ("no release date to file it under",)
195257

196258
return IngestCandidate(
197259
category="smartphone",
@@ -201,5 +263,6 @@ def _build_candidate(
201263
source_url=source_url,
202264
output_path=output_path,
203265
missing_fields=missing,
266+
problems=problems,
204267
released_by=release_period[1] if release_period else None,
205268
)

‎tests/unit/test_ingest_pipeline.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,3 +126,25 @@ def test_rows_sharing_a_slug_that_only_fill_different_fields_do_not_conflict(
126126
assert result.written == [first]
127127
assert result.skipped_same_run == [second]
128128

129+
130+
def test_smartphone_variant_folder_and_base_model_count_as_on_file(tmp_path: Path) -> None:
131+
variant = tmp_path / "smartphone" / "samsung" / "2022" / "galaxy-m33-5g" / "sm-m336b.json"
132+
variant.parent.mkdir(parents=True)
133+
variant.write_text(
134+
json.dumps({"slug": "sm-m336b", "base_model_slug": "galaxy-m33"}), encoding="utf-8"
135+
)
136+
137+
def phone(slug: str) -> IngestCandidate:
138+
return IngestCandidate(
139+
category="smartphone",
140+
manufacturer="samsung",
141+
slug=slug,
142+
record={"slug": slug, "name": slug},
143+
source_url="https://example.test/phones",
144+
output_path=Path("smartphone/samsung/2022") / f"{slug}.json",
145+
)
146+
147+
folder, base = phone("galaxy-m33-5g"), phone("galaxy-m33")
148+
result = run([folder, base], data_root=tmp_path)
149+
assert not result.written
150+
assert result.skipped_existing == [folder, base]

‎tests/unit/test_ingest_wikipedia_smartphone.py‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
import pytest
99

10-
from app.ingest.sources.wikipedia_smartphone import WikipediaSmartphoneIngest
10+
from app.ingest.sources.wikipedia_smartphone import PAGES, WikipediaSmartphoneIngest
1111

1212
_HTML = """
1313
<html><body>
@@ -76,7 +76,7 @@ def test_extracts_phone_with_known_soc(
7676
assert ultra.record["os"] == "Android 15"
7777
assert ultra.record["verified"] is False
7878
assert ultra.is_complete
79-
assert ultra.output_path.as_posix() == "smartphone/samsung/galaxy-s25-ultra.json"
79+
assert ultra.output_path.as_posix() == "smartphone/samsung/2025/galaxy-s25-ultra.json"
8080

8181

8282
def test_unknown_soc_renders_candidate_incomplete() -> None:
@@ -93,3 +93,48 @@ def test_unknown_soc_renders_candidate_incomplete() -> None:
9393
assert ultra.record["soc"] is None
9494
assert "soc" in ultra.missing_fields
9595
assert not ultra.is_complete
96+
97+
98+
def _phones(html: str) -> dict[str, object]:
99+
candidates = WikipediaSmartphoneIngest._extract(
100+
html, "apple", "List_of_iPhone_models", known_socs=set()
101+
)
102+
return {c.slug: c for c in candidates}
103+
104+
105+
_IPHONE_HTML = """
106+
<table class="wikitable">
107+
<tr><th rowspan="2">Model</th><th colspan="2">Release</th><th rowspan="2">RAM</th></tr>
108+
<tr><th>With OS</th><th>Date</th></tr>
109+
<tr><td>iPhone 5s [ 12 ]</td><td>iOS 7</td><td>September 20, 2013</td><td>1 GB</td></tr>
110+
<tr><td>iPhone 13 Pro / 13 Pro Max</td><td>iOS 15</td><td>September 24, 2021</td>
111+
<td>6 GB</td></tr>
112+
<tr><td>iPhone 5 iPhone 5c</td><td>iOS 6</td><td>September 21, 2012</td><td>1 GB</td></tr>
113+
<tr><td>iPhone 2027 lineup</td><td></td><td></td><td></td></tr>
114+
<tr><td>iPhone SE (2nd gen)</td><td>iOS 13</td><td>2020</td><td>3 GB</td></tr>
115+
</table>
116+
"""
117+
118+
119+
def test_two_row_release_header_reads_the_date_leaf() -> None:
120+
phones = _phones(_IPHONE_HTML)
121+
five_s = phones["iphone-5s"]
122+
assert five_s.record["release_date"] == "2013-09-20" # type: ignore[attr-defined]
123+
assert five_s.output_path.as_posix() == ( # type: ignore[attr-defined]
124+
"smartphone/apple/2013/iphone-5s.json"
125+
)
126+
127+
128+
def test_rows_naming_two_phones_or_no_spec_are_dropped() -> None:
129+
phones = _phones(_IPHONE_HTML)
130+
assert set(phones) == {"iphone-5s", "iphone-se-2nd-gen"}
131+
132+
133+
def test_phone_without_a_release_date_is_never_written() -> None:
134+
se = _phones(_IPHONE_HTML)["iphone-se-2nd-gen"]
135+
assert se.record["release_date"] is None # type: ignore[attr-defined]
136+
assert se.problems == ("no release date to file it under",) # type: ignore[attr-defined]
137+
138+
139+
def test_missing_list_pages_are_not_fetched() -> None:
140+
assert "List_of_Xiaomi_smartphones" not in {page for _, page in PAGES}

0 commit comments

Comments
 (0)