From 9c0eb8433b04e5ead01e380a4ceb91a225fa99e9 Mon Sep 17 00:00:00 2001 From: peterolkhov Date: Mon, 16 Nov 2026 22:35:45 -0500 Subject: [PATCH 1/3] fix: preserve encoded slashes in request unique keys --- src/crawlee/_utils/requests.py | 2 +- tests/unit/_utils/test_requests.py | 47 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index f413437fdd..39d3fdcea5 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -42,7 +42,7 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: # Construct the final URL yarl_new_url = parsed_url.with_query(sorted_search_params) yarl_new_url = yarl_new_url.with_path( - yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment + yarl_new_url.raw_path.removesuffix('/'), encoded=True, keep_query=True, keep_fragment=keep_url_fragment ) return str(yarl_new_url) diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index d9eefe38ea..55f3b11d23 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -2,6 +2,7 @@ import pytest +from crawlee import Request from crawlee._types import HttpHeaders from crawlee._utils.requests import compute_unique_key, normalize_url @@ -150,3 +151,49 @@ def test_compute_unique_key_with_whitespace_in_headers() -> None: uk_2 = compute_unique_key(url, headers=headers_with_whitespaces, use_extended_unique_key=True) assert uk_2 == expected_output + + +@pytest.mark.parametrize( + ('path', 'other_path'), + [ + pytest.param('/items/a%2Fb', '/items/a/b', id='encoded-slash'), + pytest.param('/items/a%2fb', '/items/a/b', id='lowercase-encoded-slash'), + pytest.param('/items/a%2F', '/items/a', id='encoded-trailing-slash'), + pytest.param('/items/a%2f', '/items/a', id='lowercase-encoded-trailing-slash'), + pytest.param('/items/a%2F/', '/items/a', id='encoded-before-literal-trailing-slash'), + pytest.param('/items/a%252Fb', '/items/a%2Fb', id='double-encoded-slash'), + ], +) +def test_request_unique_key_preserves_encoded_slashes(path: str, other_path: str) -> None: + """Keep escaped resource identifiers distinct from path separators in request keys.""" + request = Request.from_url(f'https://example.com{path}') + other_request = Request.from_url(f'https://example.com{other_path}') + + assert request.unique_key != other_request.unique_key + + +@pytest.mark.parametrize( + ('url', 'expected_output', 'keep_url_fragment'), + [ + pytest.param('https://example.com/a%2fb', 'https://example.com/a%2fb', False, id='escape-spelling'), + pytest.param('https://example.com/a%252fb', 'https://example.com/a%252fb', False, id='no-double-quoting'), + pytest.param('https://example.com/%61%7E', 'https://example.com/a~', False, id='unreserved-escapes'), + pytest.param('https://example.com/a%3Fb', 'https://example.com/a%3fb', False, id='escaped-question-mark'), + pytest.param('https://example.com/a/', 'https://example.com/a', False, id='literal-trailing-slash'), + pytest.param( + 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', + 'https://example.com/a%2f?a=1&z=2', + False, + id='query-filter-sort-drop-fragment', + ), + pytest.param( + 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', + 'https://example.com/a%2f?a=1&z=2#fragment', + True, + id='query-filter-sort-keep-fragment', + ), + ], +) +def test_normalize_url_encoded_path(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: + """Preserve path escapes while retaining URL canonicalization and fragment controls.""" + assert normalize_url(url, keep_url_fragment=keep_url_fragment) == expected_output From 13a9697acdd1bb91a24ae92cf5fdbb807ebfa560 Mon Sep 17 00:00:00 2001 From: peterolkhov Date: Tue, 17 Nov 2026 05:32:58 -0500 Subject: [PATCH 2/3] fix: preserve encoded slashes in request unique keys --- tests/unit/_utils/test_requests.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index 55f3b11d23..abaf41b288 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -2,7 +2,6 @@ import pytest -from crawlee import Request from crawlee._types import HttpHeaders from crawlee._utils.requests import compute_unique_key, normalize_url @@ -154,22 +153,24 @@ def test_compute_unique_key_with_whitespace_in_headers() -> None: @pytest.mark.parametrize( - ('path', 'other_path'), + ('path', 'expected_output'), [ - pytest.param('/items/a%2Fb', '/items/a/b', id='encoded-slash'), - pytest.param('/items/a%2fb', '/items/a/b', id='lowercase-encoded-slash'), - pytest.param('/items/a%2F', '/items/a', id='encoded-trailing-slash'), - pytest.param('/items/a%2f', '/items/a', id='lowercase-encoded-trailing-slash'), - pytest.param('/items/a%2F/', '/items/a', id='encoded-before-literal-trailing-slash'), - pytest.param('/items/a%252Fb', '/items/a%2Fb', id='double-encoded-slash'), + pytest.param('/items/a%2Fb', 'https://example.com/items/a%2fb', id='encoded-slash'), + pytest.param('/items/a%2fb', 'https://example.com/items/a%2fb', id='lowercase-encoded-slash'), + pytest.param('/items/a%2F', 'https://example.com/items/a%2f', id='encoded-trailing-slash'), + pytest.param('/items/a%2f', 'https://example.com/items/a%2f', id='lowercase-encoded-trailing-slash'), + pytest.param('/items/a%2F/', 'https://example.com/items/a%2f', id='encoded-before-literal-trailing-slash'), + pytest.param('/items/a%252Fb', 'https://example.com/items/a%252fb', id='double-encoded-slash'), + pytest.param('/items/a%2Bb', 'https://example.com/items/a%2bb', id='encoded-plus'), + pytest.param('/items/a%2bb', 'https://example.com/items/a%2bb', id='lowercase-encoded-plus'), + pytest.param('/items/a+b', 'https://example.com/items/a+b', id='literal-plus'), + pytest.param('/items/a/b', 'https://example.com/items/a/b', id='literal-separator'), + pytest.param('/items/a', 'https://example.com/items/a', id='literal-path'), ], ) -def test_request_unique_key_preserves_encoded_slashes(path: str, other_path: str) -> None: - """Keep escaped resource identifiers distinct from path separators in request keys.""" - request = Request.from_url(f'https://example.com{path}') - other_request = Request.from_url(f'https://example.com{other_path}') - - assert request.unique_key != other_request.unique_key +def test_compute_unique_key_encoded_path(path: str, expected_output: str) -> None: + """Keep encoded slashes and plus signs distinct from their literal forms in request keys.""" + assert compute_unique_key(f'https://example.com{path}') == expected_output @pytest.mark.parametrize( From 1933db8524c2d4859aa28002da45573bd751d040 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 18 Nov 2026 08:26:06 +0100 Subject: [PATCH 3/3] test: merge encoded path cases into the `normalize_url` table --- tests/unit/_utils/test_requests.py | 85 ++++++++++++++---------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index abaf41b288..d93e10696d 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -41,6 +41,41 @@ pytest.param( 'https://example.com/#Fragment', 'https://example.com/#Fragment', True, id='preserve_fragment_casing' ), + pytest.param('https://example.com/a/', 'https://example.com/a', False, id='remove_trailing_slash'), + pytest.param('https://example.com/a/b', 'https://example.com/a/b', False, id='keep_literal_slash'), + pytest.param('https://example.com/a%2Fb', 'https://example.com/a%2Fb', False, id='keep_encoded_slash'), + pytest.param( + 'https://example.com/a%2fb', 'https://example.com/a%2Fb', False, id='uppercase_lowercase_encoded_slash' + ), + pytest.param('https://example.com/a%2F', 'https://example.com/a%2F', False, id='keep_encoded_trailing_slash'), + pytest.param( + 'https://example.com/a%2F/', + 'https://example.com/a%2F', + False, + id='remove_literal_slash_after_encoded_slash', + ), + pytest.param( + 'https://example.com/a%252Fb', 'https://example.com/a%252Fb', False, id='keep_double_encoded_slash' + ), + pytest.param('https://example.com/a+b', 'https://example.com/a+b', False, id='keep_literal_plus'), + pytest.param('https://example.com/a%2Bb', 'https://example.com/a%2Bb', False, id='keep_encoded_plus'), + pytest.param( + 'https://example.com/a%2bb', 'https://example.com/a%2Bb', False, id='uppercase_lowercase_encoded_plus' + ), + pytest.param('https://example.com/a%3Fb', 'https://example.com/a%3Fb', False, id='keep_encoded_question_mark'), + pytest.param('https://example.com/%61%7E', 'https://example.com/a~', False, id='decode_unreserved_escapes'), + pytest.param( + 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', + 'https://example.com/a%2F?a=1&z=2', + False, + id='encoded_slash_with_query_remove_fragment', + ), + pytest.param( + 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', + 'https://example.com/a%2F?a=1&z=2#fragment', + True, + id='encoded_slash_with_query_retain_fragment', + ), ], ) def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: @@ -152,49 +187,7 @@ def test_compute_unique_key_with_whitespace_in_headers() -> None: assert uk_2 == expected_output -@pytest.mark.parametrize( - ('path', 'expected_output'), - [ - pytest.param('/items/a%2Fb', 'https://example.com/items/a%2fb', id='encoded-slash'), - pytest.param('/items/a%2fb', 'https://example.com/items/a%2fb', id='lowercase-encoded-slash'), - pytest.param('/items/a%2F', 'https://example.com/items/a%2f', id='encoded-trailing-slash'), - pytest.param('/items/a%2f', 'https://example.com/items/a%2f', id='lowercase-encoded-trailing-slash'), - pytest.param('/items/a%2F/', 'https://example.com/items/a%2f', id='encoded-before-literal-trailing-slash'), - pytest.param('/items/a%252Fb', 'https://example.com/items/a%252fb', id='double-encoded-slash'), - pytest.param('/items/a%2Bb', 'https://example.com/items/a%2bb', id='encoded-plus'), - pytest.param('/items/a%2bb', 'https://example.com/items/a%2bb', id='lowercase-encoded-plus'), - pytest.param('/items/a+b', 'https://example.com/items/a+b', id='literal-plus'), - pytest.param('/items/a/b', 'https://example.com/items/a/b', id='literal-separator'), - pytest.param('/items/a', 'https://example.com/items/a', id='literal-path'), - ], -) -def test_compute_unique_key_encoded_path(path: str, expected_output: str) -> None: - """Keep encoded slashes and plus signs distinct from their literal forms in request keys.""" - assert compute_unique_key(f'https://example.com{path}') == expected_output - - -@pytest.mark.parametrize( - ('url', 'expected_output', 'keep_url_fragment'), - [ - pytest.param('https://example.com/a%2fb', 'https://example.com/a%2fb', False, id='escape-spelling'), - pytest.param('https://example.com/a%252fb', 'https://example.com/a%252fb', False, id='no-double-quoting'), - pytest.param('https://example.com/%61%7E', 'https://example.com/a~', False, id='unreserved-escapes'), - pytest.param('https://example.com/a%3Fb', 'https://example.com/a%3fb', False, id='escaped-question-mark'), - pytest.param('https://example.com/a/', 'https://example.com/a', False, id='literal-trailing-slash'), - pytest.param( - 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', - 'https://example.com/a%2f?a=1&z=2', - False, - id='query-filter-sort-drop-fragment', - ), - pytest.param( - 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', - 'https://example.com/a%2f?a=1&z=2#fragment', - True, - id='query-filter-sort-keep-fragment', - ), - ], -) -def test_normalize_url_encoded_path(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: - """Preserve path escapes while retaining URL canonicalization and fragment controls.""" - assert normalize_url(url, keep_url_fragment=keep_url_fragment) == expected_output +def test_compute_unique_key_keeps_encoded_slash() -> None: + """An encoded slash in the path gives a different unique key than a literal slash.""" + assert compute_unique_key('https://example.com/a%2Fb') == 'https://example.com/a%2Fb' + assert compute_unique_key('https://example.com/a/b') == 'https://example.com/a/b'