diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index f413437fdd..39d3fdcea5 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -42,7 +42,7 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: # Construct the final URL yarl_new_url = parsed_url.with_query(sorted_search_params) yarl_new_url = yarl_new_url.with_path( - yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment + yarl_new_url.raw_path.removesuffix('/'), encoded=True, keep_query=True, keep_fragment=keep_url_fragment ) return str(yarl_new_url) diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index d9eefe38ea..d93e10696d 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -41,6 +41,41 @@ pytest.param( 'https://example.com/#Fragment', 'https://example.com/#Fragment', True, id='preserve_fragment_casing' ), + pytest.param('https://example.com/a/', 'https://example.com/a', False, id='remove_trailing_slash'), + pytest.param('https://example.com/a/b', 'https://example.com/a/b', False, id='keep_literal_slash'), + pytest.param('https://example.com/a%2Fb', 'https://example.com/a%2Fb', False, id='keep_encoded_slash'), + pytest.param( + 'https://example.com/a%2fb', 'https://example.com/a%2Fb', False, id='uppercase_lowercase_encoded_slash' + ), + pytest.param('https://example.com/a%2F', 'https://example.com/a%2F', False, id='keep_encoded_trailing_slash'), + pytest.param( + 'https://example.com/a%2F/', + 'https://example.com/a%2F', + False, + id='remove_literal_slash_after_encoded_slash', + ), + pytest.param( + 'https://example.com/a%252Fb', 'https://example.com/a%252Fb', False, id='keep_double_encoded_slash' + ), + pytest.param('https://example.com/a+b', 'https://example.com/a+b', False, id='keep_literal_plus'), + pytest.param('https://example.com/a%2Bb', 'https://example.com/a%2Bb', False, id='keep_encoded_plus'), + pytest.param( + 'https://example.com/a%2bb', 'https://example.com/a%2Bb', False, id='uppercase_lowercase_encoded_plus' + ), + pytest.param('https://example.com/a%3Fb', 'https://example.com/a%3Fb', False, id='keep_encoded_question_mark'), + pytest.param('https://example.com/%61%7E', 'https://example.com/a~', False, id='decode_unreserved_escapes'), + pytest.param( + 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', + 'https://example.com/a%2F?a=1&z=2', + False, + id='encoded_slash_with_query_remove_fragment', + ), + pytest.param( + 'https://example.com/a%2F/?z=2&utm_source=test&a=1#fragment', + 'https://example.com/a%2F?a=1&z=2#fragment', + True, + id='encoded_slash_with_query_retain_fragment', + ), ], ) def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: @@ -150,3 +185,9 @@ def test_compute_unique_key_with_whitespace_in_headers() -> None: uk_2 = compute_unique_key(url, headers=headers_with_whitespaces, use_extended_unique_key=True) assert uk_2 == expected_output + + +def test_compute_unique_key_keeps_encoded_slash() -> None: + """An encoded slash in the path gives a different unique key than a literal slash.""" + assert compute_unique_key('https://example.com/a%2Fb') == 'https://example.com/a%2Fb' + assert compute_unique_key('https://example.com/a/b') == 'https://example.com/a/b'