Merge pull request #129 from GetTechAPI/fix/verify-head-fallback

fix(verify): let GET decide when HEAD fails with a non-transient status

Author: Seungpyo1007

app/verify/http_check.py

@@ -155,6 +155,12 @@ def _is_automation_challenge(exc: Exception, code: int) -> bool:
 def _attempt(url: str, opener: Any) -> tuple[int | None, str | None, float | None, bool]:
     """One HEAD-then-GET pass.
 
+    HEAD is only a shortcut: a success answers the question, and a rate-limit
+    answer (429/503) goes straight back to :func:`check_one`'s backoff. Any other
+    status of 400 or more, or a network error, falls through to GET and GET's
+    answer is the result. Some hosts answer HEAD with a status GET does not
+    give: Kaggle dataset pages answer HEAD with 404 and GET with 200.
+
     Returns ``(status, final_url, retry_after, automation_challenge)``.  The last
     value is intentionally based on an explicit gateway signal, never on a bare
     403, so unavailable pages cannot become promotable merely by returning 403.
@@ -164,28 +170,26 @@ def _attempt(url: str, opener: Any) -> tuple[int | None, str | None, float | Non
     retry_after: float | None = None
     automation_challenge = False
     for method in ("HEAD", "GET"):
+        retry_after, automation_challenge = None, False
         try:
             status, final = opener.open(url, method)
-            if method == "HEAD" and status in (400, 403, 405, 501):
-                continue  # server dislikes HEAD -> retry GET
-            break
         except Exception as exc:  # HTTPError carries a code; everything else is dead
             code = getattr(exc, "code", None)
-            if isinstance(code, int):
-                status, final = code, getattr(exc, "url", None) or url
-                retry_after = _retry_after_seconds(exc)
-                automation_challenge = _is_automation_challenge(exc, code)
-                if method == "HEAD" and code in (400, 403, 405, 501):
-                    continue
-                break
-            status, final, automation_challenge = None, None, False
+            if not isinstance(code, int):
+                status, final = None, None
+                continue
+            status, final = code, getattr(exc, "url", None) or url
+            retry_after = _retry_after_seconds(exc)
+            automation_challenge = _is_automation_challenge(exc, code)
+        if status < 400 or status in TRANSIENT_STATUSES:
+            break
     return status, final, retry_after, automation_challenge
 
 
 def check_one(
     url: str, opener: Any, *, on_rate_limit: Callable[[str], None] | None = None
 ) -> CheckResult:
-    """HEAD first; fall back to GET when HEAD is rejected (405/403) or errors.
+    """HEAD first; when HEAD fails with anything but a rate limit, GET decides.
 
     A rate-limit answer (429/503) is retried with backoff — the host is telling us
     to wait, not that the page is gone.

tests/verify/test_http_check.py

@@ -61,6 +61,57 @@ def open(self, url, method):
     assert res.alive and res.status == 200
 
 
+class ByMethod:
+    """Answers HEAD and GET differently; an Exception value is raised."""
+
+    def __init__(self, head, get):
+        self.answers = {"HEAD": head, "GET": get}
+        self.calls = []
+
+    def open(self, url, method):
+        self.calls.append(method)
+        val = self.answers[method]
+        if isinstance(val, Exception):
+            raise val
+        return val
+
+
+def _http_error(code):
+    return type("E", (Exception,), {"code": code, "url": None, "headers": {}})()
+
+
+def test_head_404_then_get_200_is_alive():
+    # Kaggle dataset pages answer HEAD with 404 and GET with 200.
+    url = "https://www.kaggle.com/datasets/msainani/gsmarena-mobile-devices"
+    op = ByMethod(head=_http_error(404), get=(200, url))
+    res = http_check.check_one(url, op)
+    assert res.alive and res.status == 200
+    assert op.calls == ["HEAD", "GET"]
+
+
+def test_head_404_returned_then_get_200_is_alive():
+    url = "https://www.kaggle.com/datasets/jakubkhalponiak/phones-2024"
+    res = http_check.check_one(url, ByMethod(head=(404, url), get=(200, url)))
+    assert res.alive and res.status == 200
+
+
+def test_head_404_then_get_404_is_dead():
+    url = "https://example.com/gone"
+    op = ByMethod(head=_http_error(404), get=_http_error(404))
+    res = http_check.check_one(url, op)
+    assert not res.alive and res.reason == "http-404"
+    assert op.calls == ["HEAD", "GET"]
+
+
+def test_head_429_does_not_get_in_the_same_attempt(monkeypatch):
+    monkeypatch.setattr(http_check.time, "sleep", lambda _s: None)
+    url = "https://www.gsmarena.com/x-1.php"
+    op = ByMethod(head=_Http429(url, retry_after="0"), get=(200, url))
+    res = http_check.check_one(url, op)
+    assert res.status == 429 and res.transient
+    assert op.calls == ["HEAD"] * http_check.RETRY_ATTEMPTS
+
+
 def test_connection_error_is_dead():
     table = {"https://nope.invalid/x": ConnectionError("no route")}
     [res] = http_check.check_urls(list(table), opener_factory=_factory(table), min_interval=0)