diff --git a/src/crawlee/_utils/urls.py b/src/crawlee/_utils/urls.py
index 4b72634a48..0f5409e74f 100644
--- a/src/crawlee/_utils/urls.py
+++ b/src/crawlee/_utils/urls.py
@@ -42,16 +42,23 @@ def convert_to_absolute_url(base_url: str, relative_url: str) -> str:
def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger: Logger | None = None) -> Iterator[str]:
"""Convert an iterator of relative URLs to absolute URLs using a base URL."""
for url in urls:
- if is_url_absolute(url):
- yield url
- else:
- converted_url = convert_to_absolute_url(base_url, url)
- # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
- if not is_url_absolute(converted_url):
- if logger:
- logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
- continue
- yield converted_url
+ try:
+ converted_url = url if is_url_absolute(url) else convert_to_absolute_url(base_url, url)
+ # Reading the host decodes an IDNA host such as `xn--a.com`, which raises for an invalid one, so the URL
+ # would fail later when it is filtered or enqueued.
+ _ = URL(converted_url).host
+ except ValueError as error:
+ # The URL cannot be parsed, for example because its port is out of range or its host is invalid.
+ if logger:
+ logger.debug(f'Could not parse URL "{url}": {error}. Skipping it.')
+ continue
+
+ # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
+ if not is_url_absolute(converted_url):
+ if logger:
+ logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
+ continue
+ yield converted_url
def validate_http_url(value: str | None) -> str | None:
diff --git a/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
index d67727e8d3..b0089887c4 100644
--- a/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
+++ b/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
@@ -32,8 +32,8 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
assert handler.called
- # The handler should find three links
- assert len(handler.call_args[0][0]) == 3
+ # The handler should find four links
+ assert len(handler.call_args[0][0]) == 4
async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None:
diff --git a/tests/integration/crawlers/_parsel/test_parsel_crawler.py b/tests/integration/crawlers/_parsel/test_parsel_crawler.py
index 90ae0fc5cc..381197c7e8 100644
--- a/tests/integration/crawlers/_parsel/test_parsel_crawler.py
+++ b/tests/integration/crawlers/_parsel/test_parsel_crawler.py
@@ -31,8 +31,8 @@ async def request_handler(context: ParselCrawlingContext) -> None:
assert handler.called
- # The handler should find three links
- assert len(handler.call_args[0][0]) == 3
+ # The handler should find four links
+ assert len(handler.call_args[0][0]) == 4
async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None:
diff --git a/tests/integration/server_endpoints.py b/tests/integration/server_endpoints.py
index e5d68ea67c..bc8bcd7fa4 100644
--- a/tests/integration/server_endpoints.py
+++ b/tests/integration/server_endpoints.py
@@ -15,6 +15,7 @@
Link 1
Link 2
test@test.com
+ Phone