Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion docs/resiliency.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ so retries always give up rather than looping indefinitely.
| Failure | Retried | Budget | On retry |
| --- | --- | --- | --- |
| Connection errors — `TimeoutError`, `ClientConnectorError`, `ServerDisconnectedError` | yes | 3 tries / ~30s | reopen the connection |
| `NotAuthenticatedError` (session expired) | yes | 2 tries / ~60s | call `login()` |
| `NotAuthenticatedError` (session expired) | yes | 2 tries / ~60s | authenticate without registering a listener |
| `TooManyConcurrentRequestsError` | yes | 5 tries / ~120s | — |
| `TooManyExecutionsError` | yes | 5 tries / ~300s | — |
| `ExecutionQueueFullError` | yes | 5 tries / ~120s | — |
Expand All @@ -23,6 +23,22 @@ so retries always give up rather than looping indefinitely.
Everything else — `BadCredentialsError`, `TooManyRequestsError`, `MaintenanceError`,
`UnsupportedOperationError`, and so on — is **not** retried and is raised directly.

### Authentication and listener recovery

Authentication requests use their own connection-retry budget. Listener registration
uses the existing HTTP request retries, so a registration outage does not restart
an already completed login or multiply the connection attempts.

Automatic reauthentication invalidates the old listener without registering a new
one. The next event fetch registers a replacement before fetching events. A failed
registration also clears the listener ID because the server may have replaced the
old listener even if its response was lost.

Recovery callbacks propagate failures after their retry budgets are exhausted.
For example, a login timeout remains a `TimeoutError`; the client does not suppress
it and retry an unauthenticated request. A failed listener registration similarly
stops the current fetch, and a later poll can try registration again.

### Backoff timing

Delays grow exponentially — the ceilings are 1s, 2s, 4s, 8s, … — but **full jitter**
Expand Down
19 changes: 16 additions & 3 deletions pyoverkiz/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def _get_client_from_invocation(invocation: Details) -> OverkizClient:

async def relogin(invocation: Details) -> None:
"""Re-authenticate using the main `OverkizClient` instance."""
await _get_client_from_invocation(invocation).login()
await _get_client_from_invocation(invocation).login(register_event_listener=False)


async def refresh_listener(invocation: Details) -> None:
Expand Down Expand Up @@ -293,6 +293,12 @@ async def close(self) -> None:
await self._auth.close()
await self.session.close()

@retry_on_connection_failure
async def _authenticate(self) -> None:
"""Retry authentication without repeating listener registration."""
await self._auth.login()
self._event_listener_id = None

async def login(
self,
register_event_listener: bool = True,
Expand All @@ -306,14 +312,15 @@ async def login(
TooManyAttemptsBannedError: When too many failed login attempts have been made.
TooManyRequestsError: When the API rate limit has been exceeded.
"""
await self._auth.login()
await self._authenticate()

if self.server_config.api_type == APIType.LOCAL:
if register_event_listener:
await self.register_event_listener()
else:
# Validate local API token by calling a simple endpoint
await self.get_gateways()
# Auth recovery must not recurse through get_gateways' auth decorator.
await self._get("setup/gateways")

return

Expand Down Expand Up @@ -461,6 +468,7 @@ async def refresh_device_states(self, device_url: str) -> None:
)

@retry_on_concurrent_requests
@retry_on_auth_error
async def register_event_listener(self) -> str:
"""Register a new setup event listener on the current session and return a new.

Expand All @@ -471,6 +479,8 @@ async def register_event_listener(self) -> str:
timeout : listening sessions are expected to call the /events/{listenerId}/fetch
API on a regular basis.
"""
# Registration may invalidate the old listener even if its response is lost.
self._event_listener_id = None
response = await self._post("events/register")
listener_id = cast(str, response.get("id"))
self._event_listener_id = listener_id
Expand All @@ -487,6 +497,9 @@ async def fetch_events(self) -> list[Event]:
Per-session rate-limit : 1 calls per 1 SECONDS period for this particular
operation (polling).
"""
if self.event_listener_id is None:
await self.register_event_listener()
Comment on lines +500 to +501

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, but this only happens if the server rejects a new login straight away. Even then, it stops after a few tries. Not worth the extra code, so I'll leave it as is.


response = await self._post(f"events/{self.event_listener_id}/fetch")
return converter.structure(response, list[Event])

Expand Down
3 changes: 3 additions & 0 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ async def test_fetch_events_basic(
self, client: OverkizClient, fixture_name: str, event_length: int
):
"""Parameterised test that fetches events fixture and checks the expected count."""
client._event_listener_id = "listener-1"
with (CURRENT_DIR / "fixtures" / "event" / fixture_name).open(
encoding="utf-8",
) as raw_events:
Expand All @@ -207,6 +208,7 @@ async def test_fetch_events_basic(
@pytest.mark.asyncio
async def test_fetch_events_simple_cast(self, client: OverkizClient):
"""Check that event state values from the cloud (strings) are cast to appropriate types."""
client._event_listener_id = "listener-1"
with (CURRENT_DIR / "fixtures" / "event" / "events.json").open(
encoding="utf-8",
) as raw_events:
Expand Down Expand Up @@ -308,6 +310,7 @@ async def test_backoff_retries_on_concurrent_requests(
@pytest.mark.asyncio
async def test_fetch_events_casting(self, client: OverkizClient, fixture_name: str):
"""Validate that fetched event states are cast to the expected Python types for each data type."""
client._event_listener_id = "listener-1"
with (CURRENT_DIR / "fixtures" / "event" / fixture_name).open(
encoding="utf-8",
) as raw_events:
Expand Down
Loading
Loading