55SoC catalog, and emit candidates only for phones whose SoC is already
66curated — the validator enforces ``smartphone.soc`` foreign-key integrity
77so unknown SoCs would otherwise tank the whole dataset.
8+
9+ A row is a phone only when it names one model and at least one spec parses.
10+ Captions, year dividers, legends and the attribute rows of transposed
11+ comparison tables name no phone; "iPhone 13 Pro / 13 Pro Max" names two.
12+ Records go to ``smartphone/<brand>/<year>/<slug>.json``, so a row without a
13+ release date is never written.
814"""
915
1016from __future__ import annotations
1319from collections .abc import Iterator
1420from pathlib import Path
1521
16- from bs4 import BeautifulSoup
22+ from bs4 import BeautifulSoup , Tag
1723
1824from app .coverage .curated import curated_slugs
1925from app .coverage .normalize import slugify
2531 parse_date_period ,
2632 parse_ram_gb ,
2733 parse_weight_g ,
34+ release_date_of ,
2835)
2936from .base import IngestCandidate
30- from .wikitable import parse_table
37+ from .wikitable import _detect_headers , _table_to_grid
3138
3239PAGES : list [tuple [str , str ]] = [
3340 ("samsung" , "List_of_Samsung_Galaxy_smartphones" ),
3441 ("apple" , "List_of_iPhone_models" ),
3542 ("google" , "Pixel_(smartphone)" ),
3643 ("oneplus" , "List_of_OnePlus_products" ),
37- ("xiaomi" , "List_of_Xiaomi_smartphones" ),
3844]
3945
4046# Brand keys are stored lowercase; these are their display forms used to
5965# product family (Snapdragon / Dimensity / Exynos / Tensor / Kirin) and drop
6066# only the company name. Apple is the exception — "Apple A17 Pro" slugs as
6167# ``a17-pro`` because "Apple" is the family identifier, not a vendor prefix.
68+ # Citation markers left in model cells ("Galaxy Ace 3 [ 35 ]").
69+ _FOOTNOTE_RE = re .compile (r"\s*\[\s*[\w\s]{1,6}\]" )
70+ # A product word named twice means two phones in one cell ("iPhone 5 iPhone 5c").
71+ _PRODUCT_WORD_RE = re .compile (r"\b(?:iphone|galaxy|pixel|oneplus|redmi|poco)\b" , re .IGNORECASE )
72+ # Without one of these the row is not a phone (a caption, a legend, a spec label).
73+ _SPEC_FIELDS = ("soc" , "release_date" , "ram_gb" , "battery_mah" , "weight_g" )
74+
6275_SOC_VENDOR_PREFIXES = (
6376 "qualcomm" ,
6477 "mediatek" ,
@@ -110,19 +123,64 @@ def _extract(
110123 soup = BeautifulSoup (html , "html.parser" )
111124 source_url = f"https://en.wikipedia.org/wiki/{ page } "
112125 for table in soup .select ("table.wikitable" ):
113- for row in parse_table (table , HEADER_RULES ):
114- model = row .cells .get ("model" , "" )
126+ for cells in _rows (table ):
127+ model = _FOOTNOTE_RE .sub ("" , cells .get ("model" , "" )).strip ()
128+ if "/" in model or len (_PRODUCT_WORD_RE .findall (model )) > 1 :
129+ continue # two phones in one cell; splitting would guess the names
115130 slug = slugify (model , manufacturer = brand )
116131 if len (slug ) < 3 :
117132 continue
118- yield _build_candidate (
133+ candidate = _build_candidate (
119134 brand = brand ,
120135 model = model ,
121136 slug = slug ,
122- row = row . cells ,
137+ row = cells ,
123138 source_url = source_url ,
124139 known_socs = known_socs ,
125140 )
141+ if any (candidate .record .get (field ) is not None for field in _SPEC_FIELDS ):
142+ yield candidate
143+
144+
145+ def _rows (table : Tag ) -> Iterator [dict [str , str ]]:
146+ """Body rows of ``table`` keyed by field, the way ``wikitable.parse_table``
147+ reads them, and also through a two-row header: under "Release(d)" spanning
148+ "With OS" and "Date", the release date is the "Date" column, and the
149+ second header row is not a phone.
150+ """
151+ grid = _table_to_grid (table )
152+ detected = _detect_headers (grid , HEADER_RULES ) if grid else None
153+ if detected is None :
154+ return
155+ header , mapping = detected
156+ model_column = min (column for column , field in mapping .items () if field == "model" )
157+ start = header + 1
158+ if (
159+ start < len (grid )
160+ and model_column < len (grid [start ])
161+ and grid [start ][model_column ] == grid [header ][model_column ]
162+ ):
163+ leaves = grid [start ]
164+ dates = {
165+ column
166+ for column , field in mapping .items ()
167+ if field == "release_date" and column < len (leaves ) and "date" in leaves [column ].lower ()
168+ }
169+ if dates :
170+ mapping = {
171+ column : field
172+ for column , field in mapping .items ()
173+ if field != "release_date" or column in dates
174+ }
175+ start += 1
176+ for row in grid [start :]:
177+ cells : dict [str , str ] = {}
178+ for column , text in enumerate (row ):
179+ field = mapping .get (column )
180+ if field and text :
181+ cells .setdefault (field , text )
182+ if cells :
183+ yield cells
126184
127185
128186def soc_text_to_slug (text : str ) -> str :
@@ -157,7 +215,7 @@ def _build_candidate(
157215 known_socs : set [str ],
158216) -> IngestCandidate :
159217 release_period = parse_date_period (row .get ("release_date" , "" ))
160- release_date = release_period [ 0 ] if release_period else None
218+ release_date = release_date_of ( release_period )
161219 soc_text = row .get ("soc" , "" )
162220 soc_slug = soc_text_to_slug (soc_text )
163221 soc_value = soc_slug if soc_slug in known_socs else None
@@ -191,7 +249,11 @@ def _build_candidate(
191249 required = ("soc" , "release_date" , "ram_gb" , "battery_mah" , "weight_g" , "os" )
192250 missing = tuple (field for field in required if record .get (field ) in (None , "" ))
193251
194- output_path = Path ("smartphone" ) / brand / f"{ slug } .json"
252+ year = release_date .year if release_date else "unknown"
253+ output_path = Path ("smartphone" ) / brand / str (year ) / f"{ slug } .json"
254+ # The layout files a phone under its release year; without one there is no
255+ # folder to put it in, even as a draft.
256+ problems = () if release_date else ("no release date to file it under" ,)
195257
196258 return IngestCandidate (
197259 category = "smartphone" ,
@@ -201,5 +263,6 @@ def _build_candidate(
201263 source_url = source_url ,
202264 output_path = output_path ,
203265 missing_fields = missing ,
266+ problems = problems ,
204267 released_by = release_period [1 ] if release_period else None ,
205268 )
0 commit comments