diff --git a/docs/03_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx
index b6efcc80da..e03bb760e8 100644
--- a/docs/03_concepts/02_requests.mdx
+++ b/docs/03_concepts/02_requests.mdx
@@ -72,7 +72,7 @@ The crawler's `max_crawl_depth` option limits how deep the crawl goes. Requests
## Binding to a session
-Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. For details on sessions, see [Session management](./session-management).
+Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't usable anymore, the request fails with a `RequestCollisionError` and isn't retried. In the failed request handler, `context.session` is then that session while the pool still holds it, otherwise a retired session with the bound id. For details on sessions, see [Session management](./session-management).
## Request lifecycle
diff --git a/docs/03_concepts/03_crawling_context.mdx b/docs/03_concepts/03_crawling_context.mdx
index b33e1b6037..831fa732a7 100644
--- a/docs/03_concepts/03_crawling_context.mdx
+++ b/docs/03_concepts/03_crawling_context.mdx
@@ -30,7 +30,7 @@ For details on the crawler-specific attributes, see [HTTP crawlers](./http-crawl
The `request` attribute is the `Request` being processed. It carries the URL, the label used for routing, user data and the retry count. For details, see [Requests](./requests).
-The `session` attribute is the `Session` assigned to the request, or `None` when the crawler runs without a session pool. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management).
+The `session` attribute is the `Session` assigned to the request. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management).
The `proxy_info` attribute is a `ProxyInfo` describing the proxy used for the request, or `None` when no proxy is configured. For details, see [Proxy management](./proxy-management).
diff --git a/docs/03_concepts/15_cookie_management.mdx b/docs/03_concepts/15_cookie_management.mdx
index 973fc1a600..6e127da64c 100644
--- a/docs/03_concepts/15_cookie_management.mdx
+++ b/docs/03_concepts/15_cookie_management.mdx
@@ -31,12 +31,6 @@ Access the jar through `context.session.cook
{ReadWriteCookies}
-:::note
-
-`context.session` is `None` only when the crawler runs without a session pool (`use_session_pool` set to `False`). With the default pool it's always set. The examples guard with `if context.session is None: return` to cover that case.
-
-:::
-
A cookie has a `name`, a `value`, and optional parameters such as `domain`. If a cookie has no `domain`, it applies to any domain, so a login cookie set that way is sent to every host you crawl. Set `domain` on anything sensitive. For the full set of parameters, see `CookieParam`. To dump the whole jar, use `get_cookies_as_dicts()`, and to load several cookies at once, use `set_cookies()`.
## Seeding cookies on new sessions
diff --git a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py
index 26a7444937..7e87c74e18 100644
--- a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py
+++ b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py
@@ -13,9 +13,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: PlaywrightCrawlingContext) -> None:
- if context.session is None:
- return
-
# The browser receives the `visited` cookie during the first navigation.
# It's synced back onto the session after the handler returns, so on the
# second request it's already on `context.session`.
diff --git a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py
index 899ef0c3c0..57d85d4095 100644
--- a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py
+++ b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py
@@ -8,9 +8,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
# Set a cookie on the session. It's sent with every following
# request that runs on this session.
context.session.cookies.set(
diff --git a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py
index 919e7e5c20..0f4dc2fe0c 100644
--- a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py
+++ b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py
@@ -9,9 +9,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
if context.request.user_data.get('prepared'):
# Cookies from the setup step are on this pinned session.
context.log.info(
diff --git a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py
index 44cd965c46..dc565c4154 100644
--- a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py
+++ b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py
@@ -12,9 +12,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
state = await context.use_state(default_value={})
if not state.get('cookies'):
# First pass: establish cookies once and store them for every session,
@@ -30,7 +27,7 @@ async def apply_cookies(context: BasicCrawlingContext) -> None:
# Runs before navigation. Apply the shared cookies to whichever session
# handles the request, so every session ends up with them.
state = await context.use_state(default_value={})
- if context.session and (cookies := state.get('cookies')):
+ if cookies := state.get('cookies'):
context.session.cookies.set_cookies(cast('list[CookieParam]', cookies))
await crawler.run(['https://httpbingo.org/cookies'])
diff --git a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py
index 1eb8f650ed..a5390a321b 100644
--- a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py
+++ b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py
@@ -23,9 +23,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
# First attempt: establish cookies, then raise to trigger a retry.
if not context.session.cookies:
await context.send_request('https://httpbingo.org/cookies/set?logged_in=1')
diff --git a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py
index 1243b0e488..9798f2f31a 100644
--- a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py
+++ b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py
@@ -12,10 +12,7 @@ async def main() -> None:
'http://proxy-2.com/',
]
)
- crawler = BeautifulSoupCrawler(
- proxy_configuration=proxy_configuration,
- use_session_pool=True,
- )
+ crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# ...
diff --git a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py
index 68309bda59..a7018e31a1 100644
--- a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py
+++ b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py
@@ -12,10 +12,7 @@ async def main() -> None:
'http://proxy-2.com/',
]
)
- crawler = PlaywrightCrawler(
- proxy_configuration=proxy_configuration,
- use_session_pool=True,
- )
+ crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# ...
diff --git a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py
index 98b2028597..caa163eddf 100644
--- a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py
+++ b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py
@@ -45,21 +45,16 @@ async def basic_handler(context: HttpCrawlingContext) -> None:
# Initialize the session and bind the next request to this session if needed
@crawler.router.handler(label='session_init')
async def session_init(context: HttpCrawlingContext) -> None:
- next_requests = []
- if context.session:
- context.log.info('Init session %s', context.session.id)
- next_request = Request.from_url(
- 'https://a.placeholder.com', session_id=context.session.id
- )
- next_requests.append(next_request)
-
- await context.add_requests(next_requests)
+ context.log.info('Init session %s', context.session.id)
+ next_request = Request.from_url(
+ 'https://a.placeholder.com', session_id=context.session.id
+ )
+ await context.add_requests([next_request])
- # Handle errors when a session is blocked and no longer available in the pool
- # when attempting to execute requests bound to it
+ # Handle errors when the session bound to a request is no longer usable
@crawler.failed_request_handler
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
- if isinstance(error, RequestCollisionError) and context.session:
+ if isinstance(error, RequestCollisionError):
context.log.error(
'Request %s failed, because the bound session is unavailable',
context.request.url,
diff --git a/docs/03_concepts/code_examples/session_management/one_session_http.py b/docs/03_concepts/code_examples/session_management/one_session_http.py
index 6bb67a240d..b5c6bac0b1 100644
--- a/docs/03_concepts/code_examples/session_management/one_session_http.py
+++ b/docs/03_concepts/code_examples/session_management/one_session_http.py
@@ -39,13 +39,12 @@ async def basic_handler(context: HttpCrawlingContext) -> None:
# Handler for session initialization (authentication, initial cookies, etc.)
@crawler.router.handler(label='session_init')
async def session_init(context: HttpCrawlingContext) -> None:
- if context.session:
- context.log.info('Init session %s', context.session.id)
+ context.log.info('Init session %s', context.session.id)
# Monitor if our session gets blocked and explicitly stop the crawler
@crawler.error_handler
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
- if isinstance(error, SessionError) and context.session:
+ if isinstance(error, SessionError):
context.log.info('Session %s blocked', context.session.id)
crawler.stop()
diff --git a/docs/03_concepts/code_examples/session_management/sm_basic.py b/docs/03_concepts/code_examples/session_management/sm_basic.py
index 30e1d7ae92..3371c6a0c4 100644
--- a/docs/03_concepts/code_examples/session_management/sm_basic.py
+++ b/docs/03_concepts/code_examples/session_management/sm_basic.py
@@ -16,8 +16,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = BasicCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -33,13 +31,14 @@ async def default_handler(context: BasicCrawlingContext) -> None:
page_content = (await response.read()).decode()
title_match = re.search(r'(.*?)', page_content)
- if context.session and (title := title_match.group(1) if title_match else None):
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ title = title_match.group(1) if title_match else None
+
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py
index a54fd8425f..db8028213f 100644
--- a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py
+++ b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py
@@ -15,8 +15,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = BeautifulSoupCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -27,13 +25,12 @@ async def main() -> None:
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
title = context.soup.title.get_text() if context.soup.title else None
- if context.session:
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_http.py b/docs/03_concepts/code_examples/session_management/sm_http.py
index 9497594d3b..18c8585152 100644
--- a/docs/03_concepts/code_examples/session_management/sm_http.py
+++ b/docs/03_concepts/code_examples/session_management/sm_http.py
@@ -16,8 +16,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = HttpCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -29,13 +27,14 @@ async def default_handler(context: HttpCrawlingContext) -> None:
page_content = (await context.http_response.read()).decode()
title_match = re.search(r'(.*?)', page_content)
- if context.session and (title := title_match.group(1) if title_match else None):
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ title = title_match.group(1) if title_match else None
+
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_parsel.py b/docs/03_concepts/code_examples/session_management/sm_parsel.py
index 66752a63c3..508a88a4c6 100644
--- a/docs/03_concepts/code_examples/session_management/sm_parsel.py
+++ b/docs/03_concepts/code_examples/session_management/sm_parsel.py
@@ -15,8 +15,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = ParselCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -27,13 +25,12 @@ async def main() -> None:
async def default_handler(context: ParselCrawlingContext) -> None:
title = context.selector.css('title::text').get()
- if context.session:
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_playwright.py b/docs/03_concepts/code_examples/session_management/sm_playwright.py
index 46a4c4f096..7e848fbcad 100644
--- a/docs/03_concepts/code_examples/session_management/sm_playwright.py
+++ b/docs/03_concepts/code_examples/session_management/sm_playwright.py
@@ -15,8 +15,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = PlaywrightCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -27,13 +25,12 @@ async def main() -> None:
async def default_handler(context: PlaywrightCrawlingContext) -> None:
title = await context.page.title()
- if context.session:
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/04_guides/code_examples/login_crawler/http_login.py b/docs/04_guides/code_examples/login_crawler/http_login.py
index 82848ce834..c5fe5a4cea 100644
--- a/docs/04_guides/code_examples/login_crawler/http_login.py
+++ b/docs/04_guides/code_examples/login_crawler/http_login.py
@@ -41,10 +41,6 @@ async def request_handler(context: HttpCrawlingContext) -> None:
async def login_handler(context: HttpCrawlingContext) -> None:
context.log.info('Processing login at %s ...', context.request.url)
- # Verify that a session is available before proceeding
- if not context.session:
- raise RuntimeError('Session not found')
-
# Parse the API response containing authentication tokens and user data
data = json.loads(await context.http_response.read())
diff --git a/docs/04_guides/code_examples/login_crawler/playwright_login.py b/docs/04_guides/code_examples/login_crawler/playwright_login.py
index 650bdd917a..ad4a3cfe72 100644
--- a/docs/04_guides/code_examples/login_crawler/playwright_login.py
+++ b/docs/04_guides/code_examples/login_crawler/playwright_login.py
@@ -44,10 +44,6 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
async def login_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info('Processing login %s ...', context.request.url)
- # Check if the session is available
- if not context.session:
- raise RuntimeError('Session not found')
-
# Entering data into the form, `delay` to simulate human typing
# Without this, the data will be entered instantly
await context.page.type('#userName', 'crawlee_test', delay=100)
diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py
index b97a77552e..a830c552ac 100644
--- a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py
+++ b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py
@@ -8,10 +8,6 @@ async def main() -> None:
@crawler.router.default_handler
async def login_page(context: ParselCrawlingContext) -> None:
- # The CSRF token is tied to the session cookie issued for this GET.
- if not context.session:
- raise RuntimeError('Session not found')
-
# highlight-start
# Like Scrapy's `FormRequest.from_response`, the helper keeps the hidden
# `csrf_token` field, encodes the data and sets the `Content-Type` header.
diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md
index ecbb8d0e93..1174379fe4 100644
--- a/docs/06_upgrading/upgrading_to_v2.md
+++ b/docs/06_upgrading/upgrading_to_v2.md
@@ -586,6 +586,33 @@ The error of an `AdaptivePlaywrightCrawler` sub crawler that timed out now reads
Log records hold the message template and its arguments separately. `record.msg` is the template, such as `Retrying request to %s due to: %s. %s`, and `record.args` holds the values. To read the final text, use `record.getMessage()`, which formatters and handlers already do.
+## `use_session_pool` is removed
+
+The `use_session_pool` option is removed from the crawlers. Every crawler processes its requests with sessions from its `SessionPool`, the same as Crawlee JS v4. To configure the pool, pass your own `SessionPool` as `session_pool`.
+
+`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer usable still fails with a `RequestCollisionError`, now with the message `The Session (id: ) bound to the Request is no longer usable`. Its failed request handler gets that session while the pool still holds it, or a retired session with the bound id, instead of `None`.
+
+Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. To fail the request without a retry, as in v1, pass `retry_on_blocked=False`. This option also turns off the check of the response content for blocking. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request.
+
+To keep the response cookies out of the sessions, pass `save_response_cookies=False`. With `BasicCrawler`, which doesn't take this option, and with the HTTP crawlers, `context.send_request` still saves its response cookies to the session. To keep them out, pass `cookies=SessionCookies()` to `context.send_request`. To retire every session after its first request, set `max_usage_count` to 1:
+
+```python
+# v1
+crawler = ParselCrawler(use_session_pool=False)
+
+# v2
+from crawlee.sessions import SessionCookies, SessionPool
+
+crawler = ParselCrawler(
+ retry_on_blocked=False,
+ save_response_cookies=False,
+ session_pool=SessionPool(create_session_settings={'max_usage_count': 1}),
+)
+
+# In a request handler
+response = await context.send_request(url, cookies=SessionCookies())
+```
+
## Session retirement is permanent
In v1, `Session.retire()` only added `max_error_score` to the error score, so a later `mark_good()` could lower it again and make the retired session usable. In v2, a separate `retired` flag tracks retirement, exposed as the `Session.retired` property, and a retired session is never usable again. Create a new session to replace it.
diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py
index 95e2ca546a..0009c43570 100644
--- a/src/crawlee/_types.py
+++ b/src/crawlee/_types.py
@@ -679,7 +679,7 @@ class BasicCrawlingContext:
request: Request
"""Request object for the current page being processed."""
- session: Session | None
+ session: Session
"""Session object for the current page being processed."""
proxy_info: ProxyInfo | None
diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
index 9b5ad9fdfd..6daf9729a8 100644
--- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
+++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
@@ -314,7 +314,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
"""
# Without saving, the session cookies are still sent, but the response cookies go to a copy.
cookies = None
- if context.session is not None and not self._save_response_cookies:
+ if not self._save_response_cookies:
cookies = SessionCookies(context.session.cookies)
async with self._shared_navigation_timeouts[id(context.request)]:
diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py
index 2bba136b26..11a6a7581c 100644
--- a/src/crawlee/crawlers/_basic/_basic_crawler.py
+++ b/src/crawlee/crawlers/_basic/_basic_crawler.py
@@ -71,7 +71,7 @@
from crawlee.http_clients import ImpitHttpClient
from crawlee.request_loaders import MinIntervalSignal, RateLimitedSignal
from crawlee.router import Router
-from crawlee.sessions import SessionPool
+from crawlee.sessions import Session, SessionPool
from crawlee.statistics import Statistics, StatisticsState
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
@@ -103,7 +103,7 @@
from crawlee.http_clients import HttpClient, HttpResponse
from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo
from crawlee.request_loaders import RequestManager
- from crawlee.sessions import Session, SessionCookies
+ from crawlee.sessions import SessionCookies
from crawlee.statistics import FinalStatistics
from crawlee.storage_backends import StorageBackend
from crawlee.storage_backends.models import DatasetItemsListPage
@@ -169,9 +169,6 @@ class _BasicCrawlerOptions(TypedDict):
If not set, crawling continues without depth restrictions.
"""
- use_session_pool: NotRequired[bool]
- """Enable the use of a session pool for managing sessions during crawling."""
-
retry_on_blocked: NotRequired[bool]
"""If True, the crawler attempts to bypass bot protections automatically."""
@@ -310,7 +307,6 @@ def __init__(
max_requests_per_crawl: int | None = None,
max_session_rotations: int = 10,
max_crawl_depth: int | None = None,
- use_session_pool: bool = True,
retry_on_blocked: bool = True,
additional_http_error_status_codes: Iterable[int] | None = None,
ignore_http_error_status_codes: Iterable[int] | None = None,
@@ -358,7 +354,6 @@ def __init__(
this depth. The crawl depth starts at 0 for initial requests and increases with each subsequent level
of links. Requests at the maximum depth will still be processed, but no new links will be enqueued
from those requests. If not set, crawling continues without depth restrictions.
- use_session_pool: Enable the use of a session pool for managing sessions during crawling.
retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically.
additional_http_error_status_codes: Additional HTTP status codes to treat as errors,
triggering automatic retries when encountered.
@@ -474,7 +469,6 @@ def __init__(
)
# Retry and session settings
- self._use_session_pool = use_session_pool
self._retry_on_blocked = retry_on_blocked
# Logging setup
@@ -671,11 +665,8 @@ def _stop_if_max_requests_count_exceeded(self) -> None:
reason=f'The crawler has reached its limit of {self._max_requests_per_crawl} requests per crawl. '
)
- async def _get_session(self) -> Session | None:
- """If session pool is being used, try to take a session from it."""
- if not self._use_session_pool:
- return None
-
+ async def _get_session(self) -> Session:
+ """Take a session from the session pool."""
return await wait_for(
self._session_pool.get_session,
timeout=self._internal_timeout,
@@ -685,11 +676,8 @@ async def _get_session(self) -> Session | None:
logger=self._logger,
)
- async def _get_session_by_id(self, session_id: str | None) -> Session | None:
- """If session pool is being used, try to take a session by id from it."""
- if not self._use_session_pool or not session_id:
- return None
-
+ async def _get_session_by_id(self, session_id: str) -> Session | None:
+ """Take the session with the given id from the session pool."""
return await wait_for(
partial(self._session_pool.get_session_by_id, session_id),
timeout=self._internal_timeout,
@@ -705,7 +693,7 @@ async def _get_proxy_info(self, request: Request, session: Session | None) -> Pr
return None
return await self._proxy_configuration.new_proxy_info(
- session_id=session.id if session else None,
+ session_id=session.id if session is not None else None,
request=request,
)
@@ -843,9 +831,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta
self._statistics.reset()
await self._statistics.reset_store()
- if self._use_session_pool:
- self._session_pool.reset()
- await self._session_pool.reset_store()
+ self._session_pool.reset()
+ await self._session_pool.reset_store()
if requests is not None:
await self.add_requests(requests)
@@ -996,7 +983,7 @@ async def _run_crawler(self) -> None:
for cm in (
owned_concurrency_system,
self._statistics,
- self._session_pool if self._use_session_pool else None,
+ self._session_pool,
self._http_client,
self._crawler_state_rec_task,
*self._additional_context_managers,
@@ -1373,7 +1360,7 @@ async def _handle_error_handler_replacement(
if context.request.session_rotation_count is not None:
new_request.session_rotation_count = context.request.session_rotation_count
- if retire_session and context.session:
+ if retire_session:
context.session.retire()
await request_manager.add_request(new_request)
@@ -1437,8 +1424,7 @@ async def _handle_request_error(self, context: TCrawlingContext | BasicCrawlingC
context.request.state = RequestState.ERROR
raise
- if context.session:
- context.session.mark_bad()
+ context.session.mark_bad()
async def _handle_failed_request(self, context: TCrawlingContext | BasicCrawlingContext, error: Exception) -> None:
self._logger.error(
@@ -1496,7 +1482,7 @@ def _get_only_inner_most_exception(self, error: BaseException) -> BaseException:
def _prepare_send_request_function(
self,
- session: Session | None,
+ session: Session,
proxy_info: ProxyInfo | None,
register_deferred_cleanup: Callable[[DeferredCleanupCallback], None],
) -> SendRequestFunction:
@@ -1706,11 +1692,23 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
self._logger.debug('Processing request', extra=request_log_extra(request))
+ # TODO: Revisit the session binding with the error handling sync and the session reuse strategies.
+ # https://github.com/apify/crawlee-python/issues/1739
+ # https://github.com/apify/crawlee-python/issues/2135
+ bound_session_unusable = False
if request.session_id:
session = await self._get_session_by_id(request.session_id)
+ if session is None:
+ bound_session_unusable = True
+ # The failed request handler gets the pool's unusable session, or a retired stand-in once it's dropped.
+ session = self._session_pool._state.current_value.sessions.get(request.session_id) # noqa: SLF001
+ if session is None:
+ session = Session(id=request.session_id, retired=True)
else:
session = await self._get_session()
- proxy_info = await self._get_proxy_info(request, session)
+ # A request bound to an unusable session fails before it's sent. Its proxy isn't tied to that session, so a
+ # `send_request` call in the failed request handler still goes through the proxy configuration.
+ proxy_info = await self._get_proxy_info(request, None if bound_session_unusable else session)
result = RequestHandlerRunResult(key_value_store_getter=self.get_key_value_store)
deferred_cleanup: list[Callable[[], Awaitable[None]]] = []
@@ -1736,7 +1734,10 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
request.state = RequestState.REQUEST_HANDLER
try:
- self._check_request_collision(request, session)
+ if bound_session_unusable:
+ raise RequestCollisionError(
+ f'The Session (id: {request.session_id}) bound to the Request is no longer usable'
+ )
await self._run_request_handler(context=context)
except TimeoutError as e:
raise RequestHandlerError(e, context) from e
@@ -1747,7 +1748,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
await self._mark_request_as_handled(request)
- if session and session.is_usable:
+ if session.is_usable:
session.mark_good()
self._statistics.record_request_processing_finish(request.unique_key)
@@ -1775,9 +1776,6 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception)
except SessionError as session_error:
- if not session:
- raise RuntimeError('SessionError raised in a crawling context without a session') from session_error
-
request.state = RequestState.ERROR_HANDLER
if self._should_retry_request(context, session_error):
@@ -1983,37 +1981,22 @@ def _record_rate_limit_status_code(
)
return False
- def _raise_for_session_blocked_status_code(self, session: Session | None, status_code: int) -> None:
+ def _raise_for_session_blocked_status_code(self, session: Session, status_code: int) -> None:
"""Raise an exception if the given status code indicates the session is blocked.
Args:
- session: The session used for the request. If `None`, no check is performed.
+ session: The session used for the request.
status_code: The HTTP status code to check.
Raises:
SessionError: If the status code indicates the session is blocked.
"""
- if session is not None and session.is_blocked_status_code(
+ if session.is_blocked_status_code(
status_code=status_code,
ignore_http_error_status_codes=self._ignore_http_error_status_codes,
):
raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}')
- def _check_request_collision(self, request: Request, session: Session | None) -> None:
- """Raise an exception if a request cannot access required resources.
-
- Args:
- request: The `Request` that might require specific resources (like a session).
- session: The `Session` that was retrieved for the request, or `None` if not available.
-
- Raises:
- RequestCollisionError: If the `Session` referenced by the `Request` is not available.
- """
- if self._use_session_pool and request.session_id and not session:
- raise RequestCollisionError(
- f'The Session (id: {request.session_id}) bound to the Request is no longer available in SessionPool'
- )
-
async def _is_allowed_based_on_robots_txt_file(self, url: str) -> bool:
"""Check if the URL is allowed based on the robots.txt file.
diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
index e14e8f8844..39888d9e95 100644
--- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py
+++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
@@ -380,9 +380,8 @@ async def _navigate(
context.register_deferred_cleanup(lambda: context.page.__aexit__(None, None, None))
- if context.session:
- session_cookies = context.session.cookies.get_cookies_as_playwright_format()
- await self._update_cookies(context.page, session_cookies)
+ session_cookies = context.session.cookies.get_cookies_as_playwright_format()
+ await self._update_cookies(context.page, session_cookies)
if context.request.method != 'GET':
# Call the notification only once
@@ -600,7 +599,7 @@ async def _execute_post_navigation_hooks(self, context: TPostNavContext) -> Asyn
async def _create_crawling_context(self, context: TPostNavContext) -> AsyncGenerator[TCrawlingContext, None]:
yield self._build_crawling_context(context)
- if context.session and self._save_response_cookies:
+ if self._save_response_cookies:
pw_cookies = await self._get_cookies(context.page)
context.session.cookies.set_cookies_from_playwright_format(pw_cookies)
diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
index dfef93118a..c6931a9571 100644
--- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
+++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
@@ -949,9 +949,8 @@ async def test_change_context_state_after_handling(
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
nonlocal used_session_id
- if context.session is not None:
- used_session_id = context.session.id
- context.session.user_data['session_state'] = True
+ used_session_id = context.session.id
+ context.session.user_data['session_state'] = True
try:
# `page` is only available in the browser sub crawler, static crawling raises here.
diff --git a/tests/integration/crawlers/_basic/test_basic_crawler.py b/tests/integration/crawlers/_basic/test_basic_crawler.py
index eb08a4643d..285c2e0a28 100644
--- a/tests/integration/crawlers/_basic/test_basic_crawler.py
+++ b/tests/integration/crawlers/_basic/test_basic_crawler.py
@@ -83,7 +83,7 @@ async def test_session_rotation(server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- session_ids.append(context.session.id if context.session else None)
+ session_ids.append(context.session.id)
raise SessionError('Test error')
await crawler.run([str(server_url)])
diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py
index 7713bdc6fa..ed4d5108bf 100644
--- a/tests/integration/crawlers/_http/test_http_crawler.py
+++ b/tests/integration/crawlers/_http/test_http_crawler.py
@@ -122,22 +122,17 @@ async def test_handles_client_errors(
@pytest.mark.parametrize(
- ('ignore_http_error_status_codes', 'use_session_pool', 'expected_session_rotate', 'expected_number_error'),
+ ('ignore_http_error_status_codes', 'expected_session_rotate', 'expected_number_error'),
[
# change session and retry for no block 4xx statuses
- pytest.param([], True, 4, 1, id='default_behavior'),
- # error without retry for all 4xx statuses
- pytest.param([], False, 0, 1, id='default_behavior_without_session_pool'),
- # take as successful status codes from the `ignore_http_error_status_codes` list with Session Pool
- pytest.param([403], True, 0, 0, id='ignore_error_status_codes'),
- # take as successful status codes from the `ignore_http_error_status_codes` list without Session Pool
- pytest.param([403], False, 0, 0, id='ignore_error_status_codes_without_session_pool'),
+ pytest.param([], 4, 1, id='default_behavior'),
+ # take as successful status codes from the `ignore_http_error_status_codes` list
+ pytest.param([403], 0, 0, id='ignore_error_status_codes'),
],
)
async def test_handles_session_block_errors(
*,
ignore_http_error_status_codes: list[int],
- use_session_pool: bool,
expected_session_rotate: int,
expected_number_error: int,
mock_request_handler: AsyncMock,
@@ -148,7 +143,6 @@ async def test_handles_session_block_errors(
ignore_http_error_status_codes=ignore_http_error_status_codes,
max_request_retries=3,
max_session_rotations=5,
- use_session_pool=use_session_pool,
)
await crawler.add_requests([str(server_url / 'status/403')])
@@ -186,7 +180,7 @@ async def test_stores_cookies(http_client: HttpClient, server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
visit(context.request.url)
- track_session_usage(context.session.id if context.session else None)
+ track_session_usage(context.session.id)
await crawler.run(
[
@@ -505,9 +499,6 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if not context.session:
- return
-
sessions_ids.append(context.session.id)
if context.request.unique_key not in {'1', '2'}:
@@ -560,7 +551,7 @@ async def test_store_complex_cookies(server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
visit(context.request.url)
- track_session_usage(context.session.id if context.session else None)
+ track_session_usage(context.session.id)
await crawler.run([str(server_url / 'set_complex_cookies')])
@@ -661,7 +652,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else []
+ session_cookies = context.session.cookies.get_cookies_as_dicts()
handler_call(
sent_cookies=json.loads(await context.http_response.read())['cookies'],
session_cookies={cookie['name'] for cookie in session_cookies},
@@ -681,7 +672,7 @@ async def test_context_send_request_saves_cookies(server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
await context.send_request(str(server_url.with_path('set_cookies').extend_query(b=2)))
- session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else []
+ session_cookies = context.session.cookies.get_cookies_as_dicts()
handler_call(session_cookies={cookie['name'] for cookie in session_cookies})
await crawler.run([str(server_url / 'cookies')])
diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py
index 58ff078752..888ffa0535 100644
--- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py
+++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py
@@ -446,9 +446,6 @@ async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL)
@crawler.router.default_handler
async def handler(context: PlaywrightCrawlingContext) -> None:
- if not context.session:
- return
-
sessions_ids.append(context.session.id)
sessions[context.session.id] = context.session
response_data = json.loads(await context.response.text())
@@ -530,8 +527,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
# Simulate cookies installed from an external source in the browser
await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}])
- if context.session:
- session_ids.append(context.session.id)
+ session_ids.append(context.session.id)
await crawler.run([str(server_url)])
@@ -555,7 +551,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None:
async def request_handler(context: PlaywrightCrawlingContext) -> None:
await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}])
handler_call(
- session_id=context.session.id if context.session else None,
+ session_id=context.session.id,
page_cookies={cookie['name'] for cookie in await context.page.context.cookies()},
)
@@ -658,9 +654,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
cookies = await context.page.context.cookies()
playwright_cookies.extend(cookies)
- if context.session:
- context.session.cookies.set_cookies_from_playwright_format(cookies)
- session_cookies.extend(context.session.cookies.get_cookies_as_dicts())
+ context.session.cookies.set_cookies_from_playwright_format(cookies)
+ session_cookies.extend(context.session.cookies.get_cookies_as_dicts())
await crawler.run([str(server_url / 'set_complex_cookies')])
@@ -1550,5 +1545,4 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
assert handled[0].request.retry_count == 0
assert (handled[0].request.session_rotation_count or 0) == 0
session = handled[0].session
- assert session is not None
assert session.error_score == 0
diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py
index 4bd8a4341d..1577561704 100644
--- a/tests/unit/crawlers/_basic/test_basic_crawler.py
+++ b/tests/unit/crawlers/_basic/test_basic_crawler.py
@@ -46,6 +46,7 @@
LocalEventManager,
)
from crawlee.http_clients import HttpClient, HttpResponse
+from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.request_loaders import (
FinishedStatus,
MinIntervalSignal,
@@ -627,14 +628,16 @@ async def test_send_request_forwards_arguments_to_http_client() -> None:
http_client = AsyncMock(spec=HttpClient)
response = Mock(spec=HttpResponse)
http_client.send_request.return_value = response
- crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
+ crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1)
timeout = timedelta(seconds=12)
cookies = SessionCookies({'name': 'value'})
result = None
+ session = None
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- nonlocal result
+ nonlocal result, session
+ session = context.session
result = await context.send_request(
'https://example.com',
method='POST',
@@ -648,12 +651,13 @@ async def handler(context: BasicCrawlingContext) -> None:
await crawler.run(['https://a.placeholder.com'])
assert result is response
+ assert session is not None
http_client.send_request.assert_awaited_once_with(
url='https://example.com',
method='POST',
payload=b'Hello, world!',
headers={'Content-Type': 'text/plain'},
- session=None,
+ session=session,
cookies=cookies,
proxy_info=None,
timeout=timeout,
@@ -665,7 +669,7 @@ async def test_send_request_default_timeout() -> None:
"""Test that `context.send_request` without a `timeout` leaves the timeout to the default of the HTTP client."""
http_client = AsyncMock(spec=HttpClient)
http_client.send_request.return_value = Mock(spec=HttpResponse)
- crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
+ crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1)
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
@@ -2969,10 +2973,10 @@ async def test_session_retire_in_user_handler(*, retire: bool) -> None:
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- if context.session:
- sessions.append(context.session.id)
+ sessions.append(context.session.id)
- context.session.retire() if retire else None
+ if retire:
+ context.session.retire()
await context.add_requests(['https://b.placeholder.com'])
@@ -2993,8 +2997,7 @@ async def test_bound_session_to_request() -> None:
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- if context.session:
- used_sessions.append(context.session.id)
+ used_sessions.append(context.session.id)
requests = [
Request.from_url('https://a.placeholder.com', session_id=check_session.id, always_enqueue=True)
@@ -3025,8 +3028,7 @@ def create_session() -> Session:
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- if context.session:
- used_sessions.append(context.session.id)
+ used_sessions.append(context.session.id)
requests = [
Request.from_url('https://a.placeholder.com', session_id=str(session_id), use_extended_unique_key=True)
@@ -3067,6 +3069,48 @@ async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> Non
assert error_handler_mock.call_count == 1
+async def test_missing_bound_session_gives_retired_stand_in() -> None:
+ """Test that the failed request handler gets a retired session and a proxy not tied to a missing bound session."""
+ failed_request_handler = AsyncMock()
+ new_url_function = Mock(return_value='http://proxy.placeholder.com:8000')
+ crawler = BasicCrawler(
+ request_handler=AsyncMock(),
+ proxy_configuration=ProxyConfiguration(new_url_function=new_url_function),
+ )
+ crawler.failed_request_handler(failed_request_handler)
+
+ await crawler.run([Request.from_url('https://a.placeholder.com', session_id='missing')])
+
+ failed_request_handler.assert_awaited_once()
+ context, error = failed_request_handler.call_args.args
+ assert isinstance(error, RequestCollisionError)
+ assert context.session.id == 'missing'
+ assert context.session.retired
+ assert context.proxy_info is not None
+ assert context.proxy_info.url == 'http://proxy.placeholder.com:8000'
+ assert context.proxy_info.session_id is None
+ new_url_function.assert_called_once()
+ assert new_url_function.call_args.args[0] is None
+
+
+async def test_unusable_bound_session_reaches_failed_request_handler() -> None:
+ """Test that a request bound to an unusable session in the pool reaches the failed request handler with it."""
+ failed_request_handler = AsyncMock()
+
+ async with SessionPool() as session_pool:
+ session = await session_pool.get_session()
+ session.retire()
+ crawler = BasicCrawler(session_pool=session_pool, request_handler=AsyncMock())
+ crawler.failed_request_handler(failed_request_handler)
+
+ await crawler.run([Request.from_url('https://a.placeholder.com', session_id=session.id)])
+
+ failed_request_handler.assert_awaited_once()
+ context, error = failed_request_handler.call_args.args
+ assert isinstance(error, RequestCollisionError)
+ assert context.session is session
+
+
async def test_handles_session_error_in_failed_request_handler() -> None:
crawler = BasicCrawler(max_session_rotations=1)
handler_requests = set()
@@ -3524,7 +3568,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ
if isinstance(context.request.user_data['request_state'], list):
context.request.user_data['request_state'].append('error')
- if context.session and isinstance(context.session.user_data['session_state'], list):
+ if isinstance(context.session.user_data['session_state'], list):
context.session.user_data['session_state'].append('error')
context.session.cookies['error'] = 'yes'
@@ -3532,7 +3576,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ
async def handler(context: BasicCrawlingContext) -> None:
if isinstance(context.request.user_data['request_state'], list):
context.request.user_data['request_state'].append(f'modified_{context.request.retry_count}')
- if context.session and isinstance(context.session.user_data['session_state'], list):
+ if isinstance(context.session.user_data['session_state'], list):
context.session.user_data['session_state'].append(f'modified_{context.request.retry_count}')
context.session.cookies[f'modified_{context.request.retry_count}'] = 'yes'
@@ -3902,7 +3946,6 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo
assert context.request.retry_count == 0
assert stats.requests_finished == 1
assert stats.requests_failed == 0
- assert context.session is not None
assert context.session.error_score == 0
assert throttler._domain_states['throttled.placeholder.com'].consecutive_429_count == 1
diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py
index f6e310daae..d80425ad8c 100644
--- a/tests/unit/crawlers/_http/test_http_crawler.py
+++ b/tests/unit/crawlers/_http/test_http_crawler.py
@@ -127,7 +127,6 @@ async def handler(context: HttpCrawlingContext) -> None:
assert request.retry_count == 0
assert (request.session_rotation_count or 0) == 0
session = handled[0].session
- assert session is not None
assert session.error_score == 0
assert session.is_usable