Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 17 additions & 10 deletions src/crawlee/_utils/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,23 @@ def convert_to_absolute_url(base_url: str, relative_url: str) -> str:
def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger: Logger | None = None) -> Iterator[str]:
"""Convert an iterator of relative URLs to absolute URLs using a base URL."""
for url in urls:
if is_url_absolute(url):
yield url
else:
converted_url = convert_to_absolute_url(base_url, url)
# Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
if not is_url_absolute(converted_url):
if logger:
logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
continue
yield converted_url
try:
converted_url = url if is_url_absolute(url) else convert_to_absolute_url(base_url, url)
# Reading the host decodes an IDNA host such as `xn--a.com`, which raises for an invalid one, so the URL
# would fail later when it is filtered or enqueued.
_ = URL(converted_url).host
except ValueError as error:
# The URL cannot be parsed, for example because its port is out of range or its host is invalid.
if logger:
logger.debug(f'Could not parse URL "{url}": {error}. Skipping it.')
continue

# Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
if not is_url_absolute(converted_url):
if logger:
logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
Comment thread
vdusek marked this conversation as resolved.
Resolved
continue
yield converted_url


def validate_http_url(value: str | None) -> str | None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:

assert handler.called

# The handler should find three links
assert len(handler.call_args[0][0]) == 3
# The handler should find four links
assert len(handler.call_args[0][0]) == 4


async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None:
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/crawlers/_parsel/test_parsel_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ async def request_handler(context: ParselCrawlingContext) -> None:

assert handler.called

# The handler should find three links
assert len(handler.call_args[0][0]) == 3
# The handler should find four links
assert len(handler.call_args[0][0]) == 4


async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None:
Expand Down
1 change: 1 addition & 0 deletions tests/integration/server_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<a href="/sub_index" class="foo">Link 1</a>
<a href="/page_1">Link 2</a>
<a href="mailto:test@test.com">test@test.com</a>
<a href="http://tel:+911234567890">Phone</a>
Comment thread
vdusek marked this conversation as resolved.
Resolved
</body></html>"""

START_ENQUEUE_NON_HREF = """\
Expand Down
17 changes: 17 additions & 0 deletions tests/unit/_utils/test_urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
convert_to_absolute_url,
filter_url,
is_url_absolute,
to_absolute_url_iterator,
validate_http_url,
)

Expand Down Expand Up @@ -43,6 +44,22 @@ def test_convert_to_absolute_url() -> None:
assert absolute_url == 'http://example.com/path/to/resource'


@pytest.mark.parametrize(
('url', 'expected'),
[
pytest.param('https://example.com/page', ['https://example.com/page'], id='absolute'),
pytest.param('/page', ['https://base.com/page'], id='relative'),
pytest.param('mailto:test@test.com', [], id='mailto'),
pytest.param('http://tel:+911234567890', [], id='port_out_of_range'),
pytest.param('http://[::1', [], id='unclosed_ipv6_bracket'),
pytest.param('http://xn--a.com/', [], id='invalid_idna_host'),
],
)
def test_to_absolute_url_iterator(url: str, expected: list[str]) -> None:
"""Relative URLs are resolved against the base URL, and URLs that cannot be used are skipped."""
assert list(to_absolute_url_iterator('https://base.com/dir/', iter([url]))) == expected


def test_validate_http_url() -> None:
assert validate_http_url(None) is None

Expand Down
Loading