From 7de320a207e531e6b8742cbc67006b1fd709cae4 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 12 Nov 2026 12:26:51 +0000 Subject: [PATCH 1/5] remove `use_session_pool` and always process requests with a session --- docs/03_concepts/02_requests.mdx | 2 +- docs/03_concepts/03_crawling_context.mdx | 2 +- docs/03_concepts/15_cookie_management.mdx | 6 -- .../cookie_management/playwright_cookies.py | 3 - .../cookie_management/read_write_cookies.py | 3 - .../cookie_management/retry_pinned_session.py | 3 - .../retry_restore_cookies.py | 5 +- .../cookie_management/retry_single_session.py | 3 - .../proxy_management/session_bs_example.py | 2 +- .../proxy_management/session_pw_example.py | 2 +- .../session_management/multi_sessions_http.py | 16 ++--- .../session_management/one_session_http.py | 5 +- .../session_management/sm_basic.py | 17 +++-- .../session_management/sm_beautifulsoup.py | 15 ++--- .../session_management/sm_http.py | 17 +++-- .../session_management/sm_parsel.py | 15 ++--- .../session_management/sm_playwright.py | 15 ++--- .../code_examples/login_crawler/http_login.py | 4 -- .../login_crawler/playwright_login.py | 4 -- .../scrapy_migration/crawlee_post.py | 4 -- docs/06_upgrading/upgrading_to_v2.md | 21 ++++++ src/crawlee/_types.py | 2 +- .../_abstract_http/_abstract_http_crawler.py | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 67 ++++++++----------- .../_playwright/_playwright_crawler.py | 7 +- .../test_adaptive_playwright_crawler.py | 5 +- .../crawlers/_basic/test_basic_crawler.py | 2 +- .../crawlers/_http/test_http_crawler.py | 25 +++---- .../_playwright/test_playwright_crawler.py | 14 ++-- .../crawlers/_basic/test_basic_crawler.py | 43 ++++++++---- .../unit/crawlers/_http/test_http_crawler.py | 1 - 31 files changed, 145 insertions(+), 187 deletions(-) diff --git a/docs/03_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx index b6efcc80da..85dd6c6210 100644 --- a/docs/03_concepts/02_requests.mdx +++ b/docs/03_concepts/02_requests.mdx @@ -72,7 +72,7 @@ The crawler's `max_crawl_depth` option limits how deep the crawl goes. Requests ## Binding to a session -Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. For details on sessions, see [Session management](./session-management). +Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. In the failed request handler, `context.session` is then a retired session with the bound id, not taken from the pool. For details on sessions, see [Session management](./session-management). ## Request lifecycle diff --git a/docs/03_concepts/03_crawling_context.mdx b/docs/03_concepts/03_crawling_context.mdx index b58f6dd3d0..b69162f112 100644 --- a/docs/03_concepts/03_crawling_context.mdx +++ b/docs/03_concepts/03_crawling_context.mdx @@ -30,7 +30,7 @@ For details on the crawler-specific attributes, see [HTTP crawlers](./http-crawl The `request` attribute is the `Request` being processed. It carries the URL, the label used for routing, user data and the retry count. For details, see [Requests](./requests). -The `session` attribute is the `Session` assigned to the request, or `None` when the crawler runs without a session pool. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management). +The `session` attribute is the `Session` assigned to the request. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management). The `proxy_info` attribute is a `ProxyInfo` describing the proxy used for the request, or `None` when no proxy is configured. For details, see [Proxy management](./proxy-management). diff --git a/docs/03_concepts/15_cookie_management.mdx b/docs/03_concepts/15_cookie_management.mdx index 973fc1a600..6e127da64c 100644 --- a/docs/03_concepts/15_cookie_management.mdx +++ b/docs/03_concepts/15_cookie_management.mdx @@ -31,12 +31,6 @@ Access the jar through `context.session.cook {ReadWriteCookies} -:::note - -`context.session` is `None` only when the crawler runs without a session pool (`use_session_pool` set to `False`). With the default pool it's always set. The examples guard with `if context.session is None: return` to cover that case. - -::: - A cookie has a `name`, a `value`, and optional parameters such as `domain`. If a cookie has no `domain`, it applies to any domain, so a login cookie set that way is sent to every host you crawl. Set `domain` on anything sensitive. For the full set of parameters, see `CookieParam`. To dump the whole jar, use `get_cookies_as_dicts()`, and to load several cookies at once, use `set_cookies()`. ## Seeding cookies on new sessions diff --git a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py index 26a7444937..7e87c74e18 100644 --- a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py +++ b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py @@ -13,9 +13,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: PlaywrightCrawlingContext) -> None: - if context.session is None: - return - # The browser receives the `visited` cookie during the first navigation. # It's synced back onto the session after the handler returns, so on the # second request it's already on `context.session`. diff --git a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py index 899ef0c3c0..57d85d4095 100644 --- a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py +++ b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py @@ -8,9 +8,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - # Set a cookie on the session. It's sent with every following # request that runs on this session. context.session.cookies.set( diff --git a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py index 919e7e5c20..0f4dc2fe0c 100644 --- a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py +++ b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py @@ -9,9 +9,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - if context.request.user_data.get('prepared'): # Cookies from the setup step are on this pinned session. context.log.info( diff --git a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py index 44cd965c46..dc565c4154 100644 --- a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py +++ b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py @@ -12,9 +12,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - state = await context.use_state(default_value={}) if not state.get('cookies'): # First pass: establish cookies once and store them for every session, @@ -30,7 +27,7 @@ async def apply_cookies(context: BasicCrawlingContext) -> None: # Runs before navigation. Apply the shared cookies to whichever session # handles the request, so every session ends up with them. state = await context.use_state(default_value={}) - if context.session and (cookies := state.get('cookies')): + if cookies := state.get('cookies'): context.session.cookies.set_cookies(cast('list[CookieParam]', cookies)) await crawler.run(['https://httpbingo.org/cookies']) diff --git a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py index 1eb8f650ed..a5390a321b 100644 --- a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py +++ b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py @@ -23,9 +23,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - # First attempt: establish cookies, then raise to trigger a retry. if not context.session.cookies: await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') diff --git a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py index 1243b0e488..4c9e615e06 100644 --- a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py +++ b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py @@ -14,7 +14,7 @@ async def main() -> None: ) crawler = BeautifulSoupCrawler( proxy_configuration=proxy_configuration, - use_session_pool=True, + save_response_cookies=True, ) # ... diff --git a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py index 68309bda59..02205a1ad6 100644 --- a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py +++ b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py @@ -14,7 +14,7 @@ async def main() -> None: ) crawler = PlaywrightCrawler( proxy_configuration=proxy_configuration, - use_session_pool=True, + save_response_cookies=True, ) # ... diff --git a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py index 98b2028597..e6275d0a61 100644 --- a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py +++ b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py @@ -45,21 +45,17 @@ async def basic_handler(context: HttpCrawlingContext) -> None: # Initialize the session and bind the next request to this session if needed @crawler.router.handler(label='session_init') async def session_init(context: HttpCrawlingContext) -> None: - next_requests = [] - if context.session: - context.log.info('Init session %s', context.session.id) - next_request = Request.from_url( - 'https://a.placeholder.com', session_id=context.session.id - ) - next_requests.append(next_request) - - await context.add_requests(next_requests) + context.log.info('Init session %s', context.session.id) + next_request = Request.from_url( + 'https://a.placeholder.com', session_id=context.session.id + ) + await context.add_requests([next_request]) # Handle errors when a session is blocked and no longer available in the pool # when attempting to execute requests bound to it @crawler.failed_request_handler async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: - if isinstance(error, RequestCollisionError) and context.session: + if isinstance(error, RequestCollisionError): context.log.error( 'Request %s failed, because the bound session is unavailable', context.request.url, diff --git a/docs/03_concepts/code_examples/session_management/one_session_http.py b/docs/03_concepts/code_examples/session_management/one_session_http.py index 6bb67a240d..b5c6bac0b1 100644 --- a/docs/03_concepts/code_examples/session_management/one_session_http.py +++ b/docs/03_concepts/code_examples/session_management/one_session_http.py @@ -39,13 +39,12 @@ async def basic_handler(context: HttpCrawlingContext) -> None: # Handler for session initialization (authentication, initial cookies, etc.) @crawler.router.handler(label='session_init') async def session_init(context: HttpCrawlingContext) -> None: - if context.session: - context.log.info('Init session %s', context.session.id) + context.log.info('Init session %s', context.session.id) # Monitor if our session gets blocked and explicitly stop the crawler @crawler.error_handler async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: - if isinstance(error, SessionError) and context.session: + if isinstance(error, SessionError): context.log.info('Session %s blocked', context.session.id) crawler.stop() diff --git a/docs/03_concepts/code_examples/session_management/sm_basic.py b/docs/03_concepts/code_examples/session_management/sm_basic.py index 30e1d7ae92..3371c6a0c4 100644 --- a/docs/03_concepts/code_examples/session_management/sm_basic.py +++ b/docs/03_concepts/code_examples/session_management/sm_basic.py @@ -16,8 +16,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = BasicCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -33,13 +31,14 @@ async def default_handler(context: BasicCrawlingContext) -> None: page_content = (await response.read()).decode() title_match = re.search(r'(.*?)', page_content) - if context.session and (title := title_match.group(1) if title_match else None): - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + title = title_match.group(1) if title_match else None + + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py index a54fd8425f..db8028213f 100644 --- a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py +++ b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py @@ -15,8 +15,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = BeautifulSoupCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -27,13 +25,12 @@ async def main() -> None: async def default_handler(context: BeautifulSoupCrawlingContext) -> None: title = context.soup.title.get_text() if context.soup.title else None - if context.session: - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_http.py b/docs/03_concepts/code_examples/session_management/sm_http.py index 9497594d3b..18c8585152 100644 --- a/docs/03_concepts/code_examples/session_management/sm_http.py +++ b/docs/03_concepts/code_examples/session_management/sm_http.py @@ -16,8 +16,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = HttpCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -29,13 +27,14 @@ async def default_handler(context: HttpCrawlingContext) -> None: page_content = (await context.http_response.read()).decode() title_match = re.search(r'(.*?)', page_content) - if context.session and (title := title_match.group(1) if title_match else None): - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + title = title_match.group(1) if title_match else None + + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_parsel.py b/docs/03_concepts/code_examples/session_management/sm_parsel.py index 66752a63c3..508a88a4c6 100644 --- a/docs/03_concepts/code_examples/session_management/sm_parsel.py +++ b/docs/03_concepts/code_examples/session_management/sm_parsel.py @@ -15,8 +15,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = ParselCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -27,13 +25,12 @@ async def main() -> None: async def default_handler(context: ParselCrawlingContext) -> None: title = context.selector.css('title::text').get() - if context.session: - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_playwright.py b/docs/03_concepts/code_examples/session_management/sm_playwright.py index 46a4c4f096..7e848fbcad 100644 --- a/docs/03_concepts/code_examples/session_management/sm_playwright.py +++ b/docs/03_concepts/code_examples/session_management/sm_playwright.py @@ -15,8 +15,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = PlaywrightCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -27,13 +25,12 @@ async def main() -> None: async def default_handler(context: PlaywrightCrawlingContext) -> None: title = await context.page.title() - if context.session: - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/04_guides/code_examples/login_crawler/http_login.py b/docs/04_guides/code_examples/login_crawler/http_login.py index 82848ce834..c5fe5a4cea 100644 --- a/docs/04_guides/code_examples/login_crawler/http_login.py +++ b/docs/04_guides/code_examples/login_crawler/http_login.py @@ -41,10 +41,6 @@ async def request_handler(context: HttpCrawlingContext) -> None: async def login_handler(context: HttpCrawlingContext) -> None: context.log.info('Processing login at %s ...', context.request.url) - # Verify that a session is available before proceeding - if not context.session: - raise RuntimeError('Session not found') - # Parse the API response containing authentication tokens and user data data = json.loads(await context.http_response.read()) diff --git a/docs/04_guides/code_examples/login_crawler/playwright_login.py b/docs/04_guides/code_examples/login_crawler/playwright_login.py index 650bdd917a..ad4a3cfe72 100644 --- a/docs/04_guides/code_examples/login_crawler/playwright_login.py +++ b/docs/04_guides/code_examples/login_crawler/playwright_login.py @@ -44,10 +44,6 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def login_handler(context: PlaywrightCrawlingContext) -> None: context.log.info('Processing login %s ...', context.request.url) - # Check if the session is available - if not context.session: - raise RuntimeError('Session not found') - # Entering data into the form, `delay` to simulate human typing # Without this, the data will be entered instantly await context.page.type('#userName', 'crawlee_test', delay=100) diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py index b97a77552e..a830c552ac 100644 --- a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py +++ b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py @@ -8,10 +8,6 @@ async def main() -> None: @crawler.router.default_handler async def login_page(context: ParselCrawlingContext) -> None: - # The CSRF token is tied to the session cookie issued for this GET. - if not context.session: - raise RuntimeError('Session not found') - # highlight-start # Like Scrapy's `FormRequest.from_response`, the helper keeps the hidden # `csrf_token` field, encodes the data and sets the `Content-Type` header. diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index da0b617265..8e588f40bd 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -561,6 +561,27 @@ The `ThrottlingRequestManager` warning about capping a delay now calls it a "req Log records hold the message template and its arguments separately. `record.msg` is the template, such as `Retrying request to %s due to: %s. %s`, and `record.args` holds the values. To read the final text, use `record.getMessage()`, which formatters and handlers already do. +## `use_session_pool` is removed + +The `use_session_pool` option is removed from the crawlers. Every crawler processes its requests with sessions from its `SessionPool`, the same as Crawlee JS v4. To configure the pool, pass your own `SessionPool` as `session_pool`. + +`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer available still fails with a `RequestCollisionError`, and its failed request handler gets a retired session with the bound id instead of `None`. + +Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request. + +To keep the response cookies out of the sessions, pass `save_response_cookies=False`. To retire every session after its first request, set `max_usage_count` to 1: + +```python +# v1 +crawler = ParselCrawler(use_session_pool=False) + +# v2 +crawler = ParselCrawler( + save_response_cookies=False, + session_pool=SessionPool(create_session_settings={'max_usage_count': 1}), +) +``` + ## Session retirement is permanent In v1, `Session.retire()` only added `max_error_score` to the error score, so a later `mark_good()` could lower it again and make the retired session usable. In v2, a separate `retired` flag tracks retirement, exposed as the `Session.retired` property, and a retired session is never usable again. Create a new session to replace it. diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index f9e7de1022..3ff3f950c1 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -654,7 +654,7 @@ class BasicCrawlingContext: request: Request """Request object for the current page being processed.""" - session: Session | None + session: Session """Session object for the current page being processed.""" proxy_info: ProxyInfo | None diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index 2559eff3a2..f13a8b6b48 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -309,7 +309,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera """ # Without saving, the session cookies are still sent, but the response cookies go to a copy. cookies = None - if context.session is not None and not self._save_response_cookies: + if not self._save_response_cookies: cookies = SessionCookies(context.session.cookies) async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout: diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index defd8d43d4..a4816bc230 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -70,7 +70,7 @@ from crawlee.http_clients import ImpitHttpClient from crawlee.request_loaders import MinIntervalSignal, RateLimitedSignal from crawlee.router import Router -from crawlee.sessions import SessionPool +from crawlee.sessions import Session, SessionPool from crawlee.statistics import Statistics, StatisticsState from crawlee.storages import Dataset, KeyValueStore, RequestQueue @@ -101,7 +101,7 @@ from crawlee.http_clients import HttpClient, HttpResponse from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo from crawlee.request_loaders import RequestManager - from crawlee.sessions import Session, SessionCookies + from crawlee.sessions import SessionCookies from crawlee.statistics import FinalStatistics from crawlee.storage_backends import StorageBackend from crawlee.storage_backends.models import DatasetItemsListPage @@ -167,9 +167,6 @@ class _BasicCrawlerOptions(TypedDict): If not set, crawling continues without depth restrictions. """ - use_session_pool: NotRequired[bool] - """Enable the use of a session pool for managing sessions during crawling.""" - retry_on_blocked: NotRequired[bool] """If True, the crawler attempts to bypass bot protections automatically.""" @@ -302,7 +299,6 @@ def __init__( max_requests_per_crawl: int | None = None, max_session_rotations: int = 10, max_crawl_depth: int | None = None, - use_session_pool: bool = True, retry_on_blocked: bool = True, additional_http_error_status_codes: Iterable[int] | None = None, ignore_http_error_status_codes: Iterable[int] | None = None, @@ -350,7 +346,6 @@ def __init__( this depth. The crawl depth starts at 0 for initial requests and increases with each subsequent level of links. Requests at the maximum depth will still be processed, but no new links will be enqueued from those requests. If not set, crawling continues without depth restrictions. - use_session_pool: Enable the use of a session pool for managing sessions during crawling. retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically. additional_http_error_status_codes: Additional HTTP status codes to treat as errors, triggering automatic retries when encountered. @@ -465,7 +460,6 @@ def __init__( ) # Retry and session settings - self._use_session_pool = use_session_pool self._retry_on_blocked = retry_on_blocked # Logging setup @@ -662,11 +656,8 @@ def _stop_if_max_requests_count_exceeded(self) -> None: reason=f'The crawler has reached its limit of {self._max_requests_per_crawl} requests per crawl. ' ) - async def _get_session(self) -> Session | None: - """If session pool is being used, try to take a session from it.""" - if not self._use_session_pool: - return None - + async def _get_session(self) -> Session: + """Take a session from the session pool.""" return await wait_for( self._session_pool.get_session, timeout=self._internal_timeout, @@ -676,11 +667,8 @@ async def _get_session(self) -> Session | None: logger=self._logger, ) - async def _get_session_by_id(self, session_id: str | None) -> Session | None: - """If session pool is being used, try to take a session by id from it.""" - if not self._use_session_pool or not session_id: - return None - + async def _get_session_by_id(self, session_id: str) -> Session | None: + """Take the session with the given id from the session pool.""" return await wait_for( partial(self._session_pool.get_session_by_id, session_id), timeout=self._internal_timeout, @@ -690,13 +678,13 @@ async def _get_session_by_id(self, session_id: str | None) -> Session | None: logger=self._logger, ) - async def _get_proxy_info(self, request: Request, session: Session | None) -> ProxyInfo | None: + async def _get_proxy_info(self, request: Request, session: Session) -> ProxyInfo | None: """Retrieve a new ProxyInfo object based on crawler configuration and the current request and session.""" if not self._proxy_configuration: return None return await self._proxy_configuration.new_proxy_info( - session_id=session.id if session else None, + session_id=session.id, request=request, ) @@ -798,9 +786,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta self._statistics.reset() await self._statistics.reset_store() - if self._use_session_pool: - self._session_pool.reset() - await self._session_pool.reset_store() + self._session_pool.reset() + await self._session_pool.reset_store() if requests is not None: await self.add_requests(requests) @@ -945,7 +932,7 @@ async def _run_crawler(self) -> None: for cm in ( owned_concurrency_system, self._statistics, - self._session_pool if self._use_session_pool else None, + self._session_pool, self._http_client, self._crawler_state_rec_task, *self._additional_context_managers, @@ -1322,7 +1309,7 @@ async def _handle_error_handler_replacement( if context.request.session_rotation_count is not None: new_request.session_rotation_count = context.request.session_rotation_count - if retire_session and context.session: + if retire_session: context.session.retire() await request_manager.add_request(new_request) @@ -1386,8 +1373,7 @@ async def _handle_request_error(self, context: TCrawlingContext | BasicCrawlingC context.request.state = RequestState.ERROR raise - if context.session: - context.session.mark_bad() + context.session.mark_bad() async def _handle_failed_request(self, context: TCrawlingContext | BasicCrawlingContext, error: Exception) -> None: self._logger.error( @@ -1445,7 +1431,7 @@ def _get_only_inner_most_exception(self, error: BaseException) -> BaseException: def _prepare_send_request_function( self, - session: Session | None, + session: Session, proxy_info: ProxyInfo | None, register_deferred_cleanup: Callable[[DeferredCleanupCallback], None], ) -> SendRequestFunction: @@ -1628,8 +1614,14 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 self._logger.debug('Processing request', extra=request_log_extra(request)) + bound_session_missing = False if request.session_id: session = await self._get_session_by_id(request.session_id) + if session is None: + bound_session_missing = True + # The bound session isn't available, so `_check_request_collision` fails the request. A retired + # stand-in with the bound id gives the failed request handler a session. + session = Session(id=request.session_id, retired=True) else: session = await self._get_session() proxy_info = await self._get_proxy_info(request, session) @@ -1657,7 +1649,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 request.state = RequestState.REQUEST_HANDLER try: - self._check_request_collision(request, session) + self._check_request_collision(request, bound_session_missing=bound_session_missing) await self._run_request_handler(context=context) except TimeoutError as e: raise RequestHandlerError(e, context) from e @@ -1668,7 +1660,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 await self._mark_request_as_handled(request) - if session and session.is_usable: + if session.is_usable: session.mark_good() self._statistics.record_request_processing_finish(request.unique_key) @@ -1696,9 +1688,6 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception) except SessionError as session_error: - if not session: - raise RuntimeError('SessionError raised in a crawling context without a session') from session_error - request.state = RequestState.ERROR_HANDLER if self._should_retry_request(context, session_error): @@ -1900,33 +1889,33 @@ def _record_rate_limit_status_code( ) return False - def _raise_for_session_blocked_status_code(self, session: Session | None, status_code: int) -> None: + def _raise_for_session_blocked_status_code(self, session: Session, status_code: int) -> None: """Raise an exception if the given status code indicates the session is blocked. Args: - session: The session used for the request. If `None`, no check is performed. + session: The session used for the request. status_code: The HTTP status code to check. Raises: SessionError: If the status code indicates the session is blocked. """ - if session is not None and session.is_blocked_status_code( + if session.is_blocked_status_code( status_code=status_code, ignore_http_error_status_codes=self._ignore_http_error_status_codes, ): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') - def _check_request_collision(self, request: Request, session: Session | None) -> None: + def _check_request_collision(self, request: Request, *, bound_session_missing: bool) -> None: """Raise an exception if a request cannot access required resources. Args: request: The `Request` that might require specific resources (like a session). - session: The `Session` that was retrieved for the request, or `None` if not available. + bound_session_missing: Whether the `Session` bound to the `Request` isn't available in the session pool. Raises: RequestCollisionError: If the `Session` referenced by the `Request` is not available. """ - if self._use_session_pool and request.session_id and not session: + if bound_session_missing: raise RequestCollisionError( f'The Session (id: {request.session_id}) bound to the Request is no longer available in SessionPool' ) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 2803ee3c18..72c0e5aafe 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -372,9 +372,8 @@ async def _navigate( context.register_deferred_cleanup(lambda: context.page.__aexit__(None, None, None)) - if context.session: - session_cookies = context.session.cookies.get_cookies_as_playwright_format() - await self._update_cookies(context.page, session_cookies) + session_cookies = context.session.cookies.get_cookies_as_playwright_format() + await self._update_cookies(context.page, session_cookies) if context.request.method != 'GET': # Call the notification only once @@ -592,7 +591,7 @@ async def _execute_post_navigation_hooks(self, context: TPostNavContext) -> Asyn async def _create_crawling_context(self, context: TPostNavContext) -> AsyncGenerator[TCrawlingContext, None]: yield self._build_crawling_context(context) - if context.session and self._save_response_cookies: + if self._save_response_cookies: pw_cookies = await self._get_cookies(context.page) context.session.cookies.set_cookies_from_playwright_format(pw_cookies) diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 813fe6e50e..59e0ff36ac 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -897,9 +897,8 @@ async def test_change_context_state_after_handling( async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: nonlocal used_session_id - if context.session is not None: - used_session_id = context.session.id - context.session.user_data['session_state'] = True + used_session_id = context.session.id + context.session.user_data['session_state'] = True try: # `page` is only available in the browser sub crawler, static crawling raises here. diff --git a/tests/integration/crawlers/_basic/test_basic_crawler.py b/tests/integration/crawlers/_basic/test_basic_crawler.py index eb08a4643d..285c2e0a28 100644 --- a/tests/integration/crawlers/_basic/test_basic_crawler.py +++ b/tests/integration/crawlers/_basic/test_basic_crawler.py @@ -83,7 +83,7 @@ async def test_session_rotation(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - session_ids.append(context.session.id if context.session else None) + session_ids.append(context.session.id) raise SessionError('Test error') await crawler.run([str(server_url)]) diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py index 79b310b6f4..673c7aa427 100644 --- a/tests/integration/crawlers/_http/test_http_crawler.py +++ b/tests/integration/crawlers/_http/test_http_crawler.py @@ -121,22 +121,17 @@ async def test_handles_client_errors( @pytest.mark.parametrize( - ('ignore_http_error_status_codes', 'use_session_pool', 'expected_session_rotate', 'expected_number_error'), + ('ignore_http_error_status_codes', 'expected_session_rotate', 'expected_number_error'), [ # change session and retry for no block 4xx statuses - pytest.param([], True, 4, 1, id='default_behavior'), - # error without retry for all 4xx statuses - pytest.param([], False, 0, 1, id='default_behavior_without_session_pool'), - # take as successful status codes from the `ignore_http_error_status_codes` list with Session Pool - pytest.param([403], True, 0, 0, id='ignore_error_status_codes'), - # take as successful status codes from the `ignore_http_error_status_codes` list without Session Pool - pytest.param([403], False, 0, 0, id='ignore_error_status_codes_without_session_pool'), + pytest.param([], 4, 1, id='default_behavior'), + # take as successful status codes from the `ignore_http_error_status_codes` list + pytest.param([403], 0, 0, id='ignore_error_status_codes'), ], ) async def test_handles_session_block_errors( *, ignore_http_error_status_codes: list[int], - use_session_pool: bool, expected_session_rotate: int, expected_number_error: int, mock_request_handler: AsyncMock, @@ -147,7 +142,6 @@ async def test_handles_session_block_errors( ignore_http_error_status_codes=ignore_http_error_status_codes, max_request_retries=3, max_session_rotations=5, - use_session_pool=use_session_pool, ) await crawler.add_requests([str(server_url / 'status/403')]) @@ -185,7 +179,7 @@ async def test_stores_cookies(http_client: HttpClient, server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: visit(context.request.url) - track_session_usage(context.session.id if context.session else None) + track_session_usage(context.session.id) await crawler.run( [ @@ -441,9 +435,6 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if not context.session: - return - sessions_ids.append(context.session.id) if context.request.unique_key not in {'1', '2'}: @@ -496,7 +487,7 @@ async def test_store_complex_cookies(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: visit(context.request.url) - track_session_usage(context.session.id if context.session else None) + track_session_usage(context.session.id) await crawler.run([str(server_url / 'set_complex_cookies')]) @@ -597,7 +588,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else [] + session_cookies = context.session.cookies.get_cookies_as_dicts() handler_call( sent_cookies=json.loads(await context.http_response.read())['cookies'], session_cookies={cookie['name'] for cookie in session_cookies}, @@ -617,7 +608,7 @@ async def test_context_send_request_saves_cookies(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: await context.send_request(str(server_url.with_path('set_cookies').extend_query(b=2))) - session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else [] + session_cookies = context.session.cookies.get_cookies_as_dicts() handler_call(session_cookies={cookie['name'] for cookie in session_cookies}) await crawler.run([str(server_url / 'cookies')]) diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py index 25750d60f8..b4372d501c 100644 --- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py @@ -446,9 +446,6 @@ async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL) @crawler.router.default_handler async def handler(context: PlaywrightCrawlingContext) -> None: - if not context.session: - return - sessions_ids.append(context.session.id) sessions[context.session.id] = context.session response_data = json.loads(await context.response.text()) @@ -530,8 +527,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Simulate cookies installed from an external source in the browser await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}]) - if context.session: - session_ids.append(context.session.id) + session_ids.append(context.session.id) await crawler.run([str(server_url)]) @@ -555,7 +551,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None: async def request_handler(context: PlaywrightCrawlingContext) -> None: await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}]) handler_call( - session_id=context.session.id if context.session else None, + session_id=context.session.id, page_cookies={cookie['name'] for cookie in await context.page.context.cookies()}, ) @@ -658,9 +654,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: cookies = await context.page.context.cookies() playwright_cookies.extend(cookies) - if context.session: - context.session.cookies.set_cookies_from_playwright_format(cookies) - session_cookies.extend(context.session.cookies.get_cookies_as_dicts()) + context.session.cookies.set_cookies_from_playwright_format(cookies) + session_cookies.extend(context.session.cookies.get_cookies_as_dicts()) await crawler.run([str(server_url / 'set_complex_cookies')]) @@ -1521,5 +1516,4 @@ async def handler(context: PlaywrightCrawlingContext) -> None: assert handled[0].request.retry_count == 0 assert (handled[0].request.session_rotation_count or 0) == 0 session = handled[0].session - assert session is not None assert session.error_score == 0 diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 4ce9024388..f17c813d0c 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -625,14 +625,16 @@ async def test_send_request_forwards_arguments_to_http_client() -> None: http_client = AsyncMock(spec=HttpClient) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response - crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1) + crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) timeout = timedelta(seconds=12) cookies = SessionCookies({'name': 'value'}) result = None + session = None @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - nonlocal result + nonlocal result, session + session = context.session result = await context.send_request( 'https://example.com', method='POST', @@ -645,12 +647,13 @@ async def handler(context: BasicCrawlingContext) -> None: await crawler.run(['https://a.placeholder.com']) assert result is response + assert session is not None http_client.send_request.assert_awaited_once_with( url='https://example.com', method='POST', payload=b'Hello, world!', headers={'Content-Type': 'text/plain'}, - session=None, + session=session, cookies=cookies, proxy_info=None, timeout=timeout, @@ -661,7 +664,7 @@ async def test_send_request_default_timeout() -> None: """Test that `context.send_request` without a `timeout` leaves the timeout to the default of the HTTP client.""" http_client = AsyncMock(spec=HttpClient) http_client.send_request.return_value = Mock(spec=HttpResponse) - crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1) + crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: @@ -2632,10 +2635,10 @@ async def test_session_retire_in_user_handler(*, retire: bool) -> None: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - if context.session: - sessions.append(context.session.id) + sessions.append(context.session.id) - context.session.retire() if retire else None + if retire: + context.session.retire() await context.add_requests(['https://b.placeholder.com']) @@ -2656,8 +2659,7 @@ async def test_bound_session_to_request() -> None: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - if context.session: - used_sessions.append(context.session.id) + used_sessions.append(context.session.id) requests = [ Request.from_url('https://a.placeholder.com', session_id=check_session.id, always_enqueue=True) @@ -2688,8 +2690,7 @@ def create_session() -> Session: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - if context.session: - used_sessions.append(context.session.id) + used_sessions.append(context.session.id) requests = [ Request.from_url('https://a.placeholder.com', session_id=str(session_id), use_extended_unique_key=True) @@ -2730,6 +2731,21 @@ async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> Non assert error_handler_mock.call_count == 1 +async def test_missing_bound_session_gives_retired_stand_in() -> None: + """Test that a request bound to a missing session reaches the failed request handler with a retired session.""" + failed_request_handler = AsyncMock() + crawler = BasicCrawler(request_handler=AsyncMock()) + crawler.failed_request_handler(failed_request_handler) + + await crawler.run([Request.from_url('https://a.placeholder.com', session_id='missing')]) + + failed_request_handler.assert_awaited_once() + context, error = failed_request_handler.call_args.args + assert isinstance(error, RequestCollisionError) + assert context.session.id == 'missing' + assert context.session.retired + + async def test_handles_session_error_in_failed_request_handler() -> None: crawler = BasicCrawler(max_session_rotations=1) handler_requests = set() @@ -3165,7 +3181,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ if isinstance(context.request.user_data['request_state'], list): context.request.user_data['request_state'].append('error') - if context.session and isinstance(context.session.user_data['session_state'], list): + if isinstance(context.session.user_data['session_state'], list): context.session.user_data['session_state'].append('error') context.session.cookies['error'] = 'yes' @@ -3173,7 +3189,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ async def handler(context: BasicCrawlingContext) -> None: if isinstance(context.request.user_data['request_state'], list): context.request.user_data['request_state'].append(f'modified_{context.request.retry_count}') - if context.session and isinstance(context.session.user_data['session_state'], list): + if isinstance(context.session.user_data['session_state'], list): context.session.user_data['session_state'].append(f'modified_{context.request.retry_count}') context.session.cookies[f'modified_{context.request.retry_count}'] = 'yes' @@ -3543,7 +3559,6 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo assert context.request.retry_count == 0 assert stats.requests_finished == 1 assert stats.requests_failed == 0 - assert context.session is not None assert context.session.error_score == 0 assert throttler._domain_states['throttled.placeholder.com'].consecutive_429_count == 1 diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py index b9076a0b88..0ef3c94208 100644 --- a/tests/unit/crawlers/_http/test_http_crawler.py +++ b/tests/unit/crawlers/_http/test_http_crawler.py @@ -127,7 +127,6 @@ async def handler(context: HttpCrawlingContext) -> None: assert request.retry_count == 0 assert (request.session_rotation_count or 0) == 0 session = handled[0].session - assert session is not None assert session.error_score == 0 assert session.is_usable From 71fefae3caf666049d46c6e1f673be13067626be Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 16 Nov 2026 16:42:29 +0000 Subject: [PATCH 2/5] address review --- docs/03_concepts/02_requests.mdx | 2 +- .../proxy_management/session_bs_example.py | 5 +-- .../proxy_management/session_pw_example.py | 5 +-- docs/06_upgrading/upgrading_to_v2.md | 9 ++++- src/crawlee/crawlers/_basic/_basic_crawler.py | 38 ++++++++----------- .../crawlers/_basic/test_basic_crawler.py | 29 +++++++++++++- 6 files changed, 53 insertions(+), 35 deletions(-) diff --git a/docs/03_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx index 85dd6c6210..e03bb760e8 100644 --- a/docs/03_concepts/02_requests.mdx +++ b/docs/03_concepts/02_requests.mdx @@ -72,7 +72,7 @@ The crawler's `max_crawl_depth` option limits how deep the crawl goes. Requests ## Binding to a session -Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. In the failed request handler, `context.session` is then a retired session with the bound id, not taken from the pool. For details on sessions, see [Session management](./session-management). +Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't usable anymore, the request fails with a `RequestCollisionError` and isn't retried. In the failed request handler, `context.session` is then that session while the pool still holds it, otherwise a retired session with the bound id. For details on sessions, see [Session management](./session-management). ## Request lifecycle diff --git a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py index 4c9e615e06..9798f2f31a 100644 --- a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py +++ b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py @@ -12,10 +12,7 @@ async def main() -> None: 'http://proxy-2.com/', ] ) - crawler = BeautifulSoupCrawler( - proxy_configuration=proxy_configuration, - save_response_cookies=True, - ) + crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration) # ... diff --git a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py index 02205a1ad6..a7018e31a1 100644 --- a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py +++ b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py @@ -12,10 +12,7 @@ async def main() -> None: 'http://proxy-2.com/', ] ) - crawler = PlaywrightCrawler( - proxy_configuration=proxy_configuration, - save_response_cookies=True, - ) + crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration) # ... diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 8e588f40bd..5f21a2c142 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -565,21 +565,26 @@ Log records hold the message template and its arguments separately. `record.msg` The `use_session_pool` option is removed from the crawlers. Every crawler processes its requests with sessions from its `SessionPool`, the same as Crawlee JS v4. To configure the pool, pass your own `SessionPool` as `session_pool`. -`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer available still fails with a `RequestCollisionError`, and its failed request handler gets a retired session with the bound id instead of `None`. +`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer usable still fails with a `RequestCollisionError`. Its failed request handler gets that session while the pool still holds it, or a retired session with the bound id, instead of `None`. Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request. -To keep the response cookies out of the sessions, pass `save_response_cookies=False`. To retire every session after its first request, set `max_usage_count` to 1: +To keep the response cookies out of the sessions, pass `save_response_cookies=False`. With `BasicCrawler`, which doesn't take this option, and with the HTTP crawlers, `context.send_request` still saves its response cookies to the session. To keep them out, pass `cookies=SessionCookies()` to `context.send_request`. To retire every session after its first request, set `max_usage_count` to 1: ```python # v1 crawler = ParselCrawler(use_session_pool=False) # v2 +from crawlee.sessions import SessionCookies, SessionPool + crawler = ParselCrawler( save_response_cookies=False, session_pool=SessionPool(create_session_settings={'max_usage_count': 1}), ) + +# In a request handler +response = await context.send_request(url, cookies=SessionCookies()) ``` ## Session retirement is permanent diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index a4816bc230..1180a8c1e4 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1614,17 +1614,22 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 self._logger.debug('Processing request', extra=request_log_extra(request)) - bound_session_missing = False + # TODO: Revisit the session binding with the error handling sync and the session reuse strategies. + # https://github.com/apify/crawlee-python/issues/1739 + # https://github.com/apify/crawlee-python/issues/2135 + bound_session_unusable = False if request.session_id: session = await self._get_session_by_id(request.session_id) if session is None: - bound_session_missing = True - # The bound session isn't available, so `_check_request_collision` fails the request. A retired - # stand-in with the bound id gives the failed request handler a session. - session = Session(id=request.session_id, retired=True) + bound_session_unusable = True + # The failed request handler gets the pool's unusable session, or a retired stand-in once it's dropped. + session = self._session_pool._state.current_value.sessions.get(request.session_id) # noqa: SLF001 + if session is None: + session = Session(id=request.session_id, retired=True) else: session = await self._get_session() - proxy_info = await self._get_proxy_info(request, session) + # A request bound to an unusable session fails before it's sent, so it gets no proxy. + proxy_info = None if bound_session_unusable else await self._get_proxy_info(request, session) result = RequestHandlerRunResult(key_value_store_getter=self.get_key_value_store) deferred_cleanup: list[Callable[[], Awaitable[None]]] = [] @@ -1649,7 +1654,11 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 request.state = RequestState.REQUEST_HANDLER try: - self._check_request_collision(request, bound_session_missing=bound_session_missing) + if bound_session_unusable: + raise RequestCollisionError( + f'The Session (id: {request.session_id}) bound to the Request is no longer available in ' + 'SessionPool' + ) await self._run_request_handler(context=context) except TimeoutError as e: raise RequestHandlerError(e, context) from e @@ -1905,21 +1914,6 @@ def _raise_for_session_blocked_status_code(self, session: Session, status_code: ): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') - def _check_request_collision(self, request: Request, *, bound_session_missing: bool) -> None: - """Raise an exception if a request cannot access required resources. - - Args: - request: The `Request` that might require specific resources (like a session). - bound_session_missing: Whether the `Session` bound to the `Request` isn't available in the session pool. - - Raises: - RequestCollisionError: If the `Session` referenced by the `Request` is not available. - """ - if bound_session_missing: - raise RequestCollisionError( - f'The Session (id: {request.session_id}) bound to the Request is no longer available in SessionPool' - ) - async def _is_allowed_based_on_robots_txt_file(self, url: str) -> bool: """Check if the URL is allowed based on the robots.txt file. diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index f17c813d0c..a2c2fce42f 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -45,6 +45,7 @@ LocalEventManager, ) from crawlee.http_clients import HttpClient, HttpResponse +from crawlee.proxy_configuration import ProxyConfiguration from crawlee.request_loaders import ( FinishedStatus, MinIntervalSignal, @@ -2732,9 +2733,13 @@ async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> Non async def test_missing_bound_session_gives_retired_stand_in() -> None: - """Test that a request bound to a missing session reaches the failed request handler with a retired session.""" + """Test that the failed request handler gets a retired session and no proxy for a missing bound session.""" failed_request_handler = AsyncMock() - crawler = BasicCrawler(request_handler=AsyncMock()) + new_url_function = Mock(side_effect=RuntimeError('Unknown session')) + crawler = BasicCrawler( + request_handler=AsyncMock(), + proxy_configuration=ProxyConfiguration(new_url_function=new_url_function), + ) crawler.failed_request_handler(failed_request_handler) await crawler.run([Request.from_url('https://a.placeholder.com', session_id='missing')]) @@ -2744,6 +2749,26 @@ async def test_missing_bound_session_gives_retired_stand_in() -> None: assert isinstance(error, RequestCollisionError) assert context.session.id == 'missing' assert context.session.retired + assert context.proxy_info is None + new_url_function.assert_not_called() + + +async def test_unusable_bound_session_reaches_failed_request_handler() -> None: + """Test that a request bound to an unusable session in the pool reaches the failed request handler with it.""" + failed_request_handler = AsyncMock() + + async with SessionPool() as session_pool: + session = await session_pool.get_session() + session.retire() + crawler = BasicCrawler(session_pool=session_pool, request_handler=AsyncMock()) + crawler.failed_request_handler(failed_request_handler) + + await crawler.run([Request.from_url('https://a.placeholder.com', session_id=session.id)]) + + failed_request_handler.assert_awaited_once() + context, error = failed_request_handler.call_args.args + assert isinstance(error, RequestCollisionError) + assert context.session is session async def test_handles_session_error_in_failed_request_handler() -> None: From 6facbf0d4732b766a64baa6e04560321ac403fc5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 17 Nov 2026 10:10:18 +0100 Subject: [PATCH 3/5] docs: Show `retry_on_blocked=False` for v1 fail-fast behavior in the upgrading guide --- docs/06_upgrading/upgrading_to_v2.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 5f21a2c142..04c631c8f6 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -567,7 +567,7 @@ The `use_session_pool` option is removed from the crawlers. Every crawler proces `context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer usable still fails with a `RequestCollisionError`. Its failed request handler gets that session while the pool still holds it, or a retired session with the bound id, instead of `None`. -Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request. +Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. To fail the request without a retry, as in v1, pass `retry_on_blocked=False`. This option also turns off the check of the response content for blocking. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request. To keep the response cookies out of the sessions, pass `save_response_cookies=False`. With `BasicCrawler`, which doesn't take this option, and with the HTTP crawlers, `context.send_request` still saves its response cookies to the session. To keep them out, pass `cookies=SessionCookies()` to `context.send_request`. To retire every session after its first request, set `max_usage_count` to 1: @@ -579,6 +579,7 @@ crawler = ParselCrawler(use_session_pool=False) from crawlee.sessions import SessionCookies, SessionPool crawler = ParselCrawler( + retry_on_blocked=False, save_response_cookies=False, session_pool=SessionPool(create_session_settings={'max_usage_count': 1}), ) From c85e73fb5074a1a2ed695daaaf2e8c1f9710e17c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 17 Nov 2026 10:41:40 +0100 Subject: [PATCH 4/5] fix: Keep a proxy for requests whose bound session is no longer usable --- src/crawlee/crawlers/_basic/_basic_crawler.py | 9 +++++---- tests/unit/crawlers/_basic/test_basic_crawler.py | 11 +++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 1180a8c1e4..3bb88aeaa6 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -678,13 +678,13 @@ async def _get_session_by_id(self, session_id: str) -> Session | None: logger=self._logger, ) - async def _get_proxy_info(self, request: Request, session: Session) -> ProxyInfo | None: + async def _get_proxy_info(self, request: Request, session: Session | None) -> ProxyInfo | None: """Retrieve a new ProxyInfo object based on crawler configuration and the current request and session.""" if not self._proxy_configuration: return None return await self._proxy_configuration.new_proxy_info( - session_id=session.id, + session_id=session.id if session is not None else None, request=request, ) @@ -1628,8 +1628,9 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 session = Session(id=request.session_id, retired=True) else: session = await self._get_session() - # A request bound to an unusable session fails before it's sent, so it gets no proxy. - proxy_info = None if bound_session_unusable else await self._get_proxy_info(request, session) + # A request bound to an unusable session fails before it's sent. Its proxy isn't tied to that session, so a + # `send_request` call in the failed request handler still goes through the proxy configuration. + proxy_info = await self._get_proxy_info(request, None if bound_session_unusable else session) result = RequestHandlerRunResult(key_value_store_getter=self.get_key_value_store) deferred_cleanup: list[Callable[[], Awaitable[None]]] = [] diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index a2c2fce42f..0ba2671621 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -2733,9 +2733,9 @@ async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> Non async def test_missing_bound_session_gives_retired_stand_in() -> None: - """Test that the failed request handler gets a retired session and no proxy for a missing bound session.""" + """Test that the failed request handler gets a retired session and a proxy not tied to a missing bound session.""" failed_request_handler = AsyncMock() - new_url_function = Mock(side_effect=RuntimeError('Unknown session')) + new_url_function = Mock(return_value='http://proxy.placeholder.com:8000') crawler = BasicCrawler( request_handler=AsyncMock(), proxy_configuration=ProxyConfiguration(new_url_function=new_url_function), @@ -2749,8 +2749,11 @@ async def test_missing_bound_session_gives_retired_stand_in() -> None: assert isinstance(error, RequestCollisionError) assert context.session.id == 'missing' assert context.session.retired - assert context.proxy_info is None - new_url_function.assert_not_called() + assert context.proxy_info is not None + assert context.proxy_info.url == 'http://proxy.placeholder.com:8000' + assert context.proxy_info.session_id is None + new_url_function.assert_called_once() + assert new_url_function.call_args.args[0] is None async def test_unusable_bound_session_reaches_failed_request_handler() -> None: From 4faf29b23d99473d2f9f70c3f51871c64c852c94 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 17 Nov 2026 10:42:14 +0100 Subject: [PATCH 5/5] fix!: Say the bound session is no longer usable in `RequestCollisionError` BREAKING CHANGE: The `RequestCollisionError` message for a request whose bound session is no longer usable changed from "... is no longer available in SessionPool" to "... is no longer usable". --- .../code_examples/session_management/multi_sessions_http.py | 3 +-- docs/06_upgrading/upgrading_to_v2.md | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py index e6275d0a61..caa163eddf 100644 --- a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py +++ b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py @@ -51,8 +51,7 @@ async def session_init(context: HttpCrawlingContext) -> None: ) await context.add_requests([next_request]) - # Handle errors when a session is blocked and no longer available in the pool - # when attempting to execute requests bound to it + # Handle errors when the session bound to a request is no longer usable @crawler.failed_request_handler async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: if isinstance(error, RequestCollisionError): diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 04c631c8f6..d42347c048 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -565,7 +565,7 @@ Log records hold the message template and its arguments separately. `record.msg` The `use_session_pool` option is removed from the crawlers. Every crawler processes its requests with sessions from its `SessionPool`, the same as Crawlee JS v4. To configure the pool, pass your own `SessionPool` as `session_pool`. -`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer usable still fails with a `RequestCollisionError`. Its failed request handler gets that session while the pool still holds it, or a retired session with the bound id, instead of `None`. +`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer usable still fails with a `RequestCollisionError`, now with the message `The Session (id: ) bound to the Request is no longer usable`. Its failed request handler gets that session while the pool still holds it, or a retired session with the bound id, instead of `None`. Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. To fail the request without a retry, as in v1, pass `retry_on_blocked=False`. This option also turns off the check of the response content for blocking. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request. diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 3bb88aeaa6..58ba465544 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1657,8 +1657,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 try: if bound_session_unusable: raise RequestCollisionError( - f'The Session (id: {request.session_id}) bound to the Request is no longer available in ' - 'SessionPool' + f'The Session (id: {request.session_id}) bound to the Request is no longer usable' ) await self._run_request_handler(context=context) except TimeoutError as e: