diff --git a/src/crawlee/_utils/urls.py b/src/crawlee/_utils/urls.py index 4b72634a48..0f5409e74f 100644 --- a/src/crawlee/_utils/urls.py +++ b/src/crawlee/_utils/urls.py @@ -42,16 +42,23 @@ def convert_to_absolute_url(base_url: str, relative_url: str) -> str: def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger: Logger | None = None) -> Iterator[str]: """Convert an iterator of relative URLs to absolute URLs using a base URL.""" for url in urls: - if is_url_absolute(url): - yield url - else: - converted_url = convert_to_absolute_url(base_url, url) - # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'. - if not is_url_absolute(converted_url): - if logger: - logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.') - continue - yield converted_url + try: + converted_url = url if is_url_absolute(url) else convert_to_absolute_url(base_url, url) + # Reading the host decodes an IDNA host such as `xn--a.com`, which raises for an invalid one, so the URL + # would fail later when it is filtered or enqueued. + _ = URL(converted_url).host + except ValueError as error: + # The URL cannot be parsed, for example because its port is out of range or its host is invalid. + if logger: + logger.debug(f'Could not parse URL "{url}": {error}. Skipping it.') + continue + + # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'. + if not is_url_absolute(converted_url): + if logger: + logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.') + continue + yield converted_url def validate_http_url(value: str | None) -> str | None: diff --git a/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py index d67727e8d3..b0089887c4 100644 --- a/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py +++ b/tests/integration/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py @@ -32,8 +32,8 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: assert handler.called - # The handler should find three links - assert len(handler.call_args[0][0]) == 3 + # The handler should find four links + assert len(handler.call_args[0][0]) == 4 async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None: diff --git a/tests/integration/crawlers/_parsel/test_parsel_crawler.py b/tests/integration/crawlers/_parsel/test_parsel_crawler.py index 90ae0fc5cc..381197c7e8 100644 --- a/tests/integration/crawlers/_parsel/test_parsel_crawler.py +++ b/tests/integration/crawlers/_parsel/test_parsel_crawler.py @@ -31,8 +31,8 @@ async def request_handler(context: ParselCrawlingContext) -> None: assert handler.called - # The handler should find three links - assert len(handler.call_args[0][0]) == 3 + # The handler should find four links + assert len(handler.call_args[0][0]) == 4 async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None: diff --git a/tests/integration/server_endpoints.py b/tests/integration/server_endpoints.py index e5d68ea67c..bc8bcd7fa4 100644 --- a/tests/integration/server_endpoints.py +++ b/tests/integration/server_endpoints.py @@ -15,6 +15,7 @@ Link 1 Link 2 test@test.com + Phone """ START_ENQUEUE_NON_HREF = """\ diff --git a/tests/unit/_utils/test_urls.py b/tests/unit/_utils/test_urls.py index 97cf8da643..e4291c51e9 100644 --- a/tests/unit/_utils/test_urls.py +++ b/tests/unit/_utils/test_urls.py @@ -9,6 +9,7 @@ convert_to_absolute_url, filter_url, is_url_absolute, + to_absolute_url_iterator, validate_http_url, ) @@ -43,6 +44,22 @@ def test_convert_to_absolute_url() -> None: assert absolute_url == 'http://example.com/path/to/resource' +@pytest.mark.parametrize( + ('url', 'expected'), + [ + pytest.param('https://example.com/page', ['https://example.com/page'], id='absolute'), + pytest.param('/page', ['https://base.com/page'], id='relative'), + pytest.param('mailto:test@test.com', [], id='mailto'), + pytest.param('http://tel:+911234567890', [], id='port_out_of_range'), + pytest.param('http://[::1', [], id='unclosed_ipv6_bracket'), + pytest.param('http://xn--a.com/', [], id='invalid_idna_host'), + ], +) +def test_to_absolute_url_iterator(url: str, expected: list[str]) -> None: + """Relative URLs are resolved against the base URL, and URLs that cannot be used are skipped.""" + assert list(to_absolute_url_iterator('https://base.com/dir/', iter([url]))) == expected + + def test_validate_http_url() -> None: assert validate_http_url(None) is None