diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..234877da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Kernel HTTPS traffic negotiates HTTP/2 with HTTP/1.1 fallback. Set `enable_http2=False` to force HTTP/1.1. Configured proxies also force HTTP/1.1. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index f0b3d2ee7..88ee4edd8 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -114,6 +114,7 @@ to change without notice. | Option | Type | Thrift | Kernel | Default Value | Note | | ------------------------------------ | ----------- | :----: | :----: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `enable_http2` | `bool` | ❌ | ✅ | `None` (kernel default `True`) | Negotiate HTTP/2 for HTTPS with HTTP/1.1 fallback. `False` or configured proxies force HTTP/1.1. An explicit value requires a supporting kernel wheel; otherwise raises `NotSupportedError`. | | `_socket_timeout` | `float` (s) | ✅ | ✅ | `900` (Thrift); `120` (kernel) | Thrift: socket send/recv/connect timeout. Kernel: total HTTP request deadline from connect through response-body completion. A positive value is forwarded; unset or `0` selects the kernel's 120s default. On the kernel path, `0` is neither unlimited nor an immediate timeout. | | `_pool_connections` | `int` | ⚠️ | ❌ | `10` | Number of pools in the shared Python HTTP client. On Thrift this affects auxiliary auth/feature-flag/telemetry traffic, not the separate Thrift query transport. Kernel-owned network traffic does not use this client. | | `_pool_maxsize` | `int` | ⚠️ | ⚠️ | `20` (Python); `100` (kernel when unset) | Max idle connections retained per host. It configures the shared Python client but not the Thrift query transport. With kernel ≥ 1.1.0, a positive value also configures the kernel's Rust HTTP pool. Unset or `0` keeps each client's default. | diff --git a/README.md b/README.md index 36162e69b..b890bd992 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,9 @@ connection = sql.connect( ``` Notes: +- HTTPS uses HTTP/2 with HTTP/1.1 fallback. `enable_http2=False` or configured + proxy environment variables force HTTP/1.1 for session requests, including + bypassed destinations. - Requires **Python >= 3.10** (the kernel wheel is published as `cp310-abi3`). On older interpreters the `[kernel]` extra installs nothing and `use_kernel=True` raises an `ImportError`. diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index eab9847ca..1ecc2a7f9 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -218,6 +218,19 @@ def _kernel_session_accepts_kwarg(name: str) -> bool: return support.accepts_kwargs or name in support.parameters +def _kernel_http2_kwargs(value: Optional[bool]) -> Dict[str, bool]: + if value is None: + return {} + if not isinstance(value, bool): + raise ValueError("enable_http2 must be a bool") + if not _kernel_session_accepts_kwarg("enable_http2"): + raise NotSupportedError( + "enable_http2 requires a newer databricks-sql-kernel wheel " + "that supports HTTP/2 configuration." + ) + return {"enable_http2": value} + + def _kernel_geospatial_kwargs(value: bool) -> Dict[str, bool]: """Build the explicit native geospatial-support kwarg. @@ -354,6 +367,7 @@ def __init__( or self._max_download_threads <= 0 ): raise ValueError("max_download_threads must be a positive integer") + self._http2_kwargs = _kernel_http2_kwargs(kwargs.get("enable_http2")) # Opt-in native GEOMETRY / GEOGRAPHY support. True requests # the canonical Arrow struct and surfaces as ``{"srid": int, "wkb": # bytes}`` through pyarrow; False requests WKT / EWKT strings. This is @@ -538,6 +552,7 @@ def open_session( **geospatial_kwargs, **max_connections_kwargs, **http_headers_kwargs, + **self._http2_kwargs, ) except Exception as exc: raise _wrap_kernel_exception("open_session", exc) from exc diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..a932219d2 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,10 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + :param enable_http2: `bool`, optional (kernel default is True) + Kernel backend only. Negotiate HTTP/2 for session HTTPS + traffic, with HTTP/1.1 fallback. ``False`` or configured proxy + environment variables force HTTP/1.1, including bypassed hosts. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..23693141c 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -337,6 +337,7 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, + enable_http2=kwargs.get("enable_http2"), use_cloud_fetch=kwargs.get("use_cloud_fetch", True), max_download_threads=kwargs.get("max_download_threads", 16), enable_geospatial_support=kwargs.get( diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5bad2040b..2cfc769bd 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -407,6 +407,51 @@ def fake_session(**options): assert configuration == original +@pytest.mark.parametrize("enabled", [None, True, False]) +def test_open_session_preserves_http2_option(monkeypatch, enabled): + captured = {} + + def fake_session(**kwargs): + captured.update(kwargs) + session = MagicMock() + session.session_id = "sess-id" + return session + + monkeypatch.setattr(kernel_client._kernel, "Session", fake_session) + client = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + **({"enable_http2": enabled} if enabled is not None else {}), + ) + client.open_session(session_configuration=None, catalog=None, schema=None) + if enabled is None: + assert "enable_http2" not in captured + else: + assert captured["enable_http2"] is enabled + + +def test_http2_option_rejects_non_boolean(): + with pytest.raises(ValueError, match="enable_http2 must be a bool"): + kernel_client._kernel_http2_kwargs("false") + + +def test_http2_option_requires_kernel_support(monkeypatch): + def old_session(): + pytest.fail("unsupported HTTP/2 option reached Session") + + monkeypatch.setattr(kernel_client._kernel, "Session", old_session) + with pytest.raises(NotSupportedError, match="enable_http2 requires a newer"): + kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + enable_http2=False, + ) + + @pytest.mark.parametrize("timeout", [None, 0, 12.5]) def test_open_session_passes_request_timeout_to_kernel(monkeypatch, timeout): captured = {} diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..08b6668a7 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -540,6 +540,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _pool_maxsize=41, use_cloud_fetch=False, max_download_threads=4, + enable_http2=False, enable_geospatial_support=False, ) try: @@ -553,6 +554,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): assert kwargs["max_connections"] == 41 assert kwargs["use_cloud_fetch"] is False assert kwargs["max_download_threads"] == 4 + assert kwargs["enable_http2"] is False assert kwargs["enable_geospatial_support"] is False finally: conn.close() @@ -569,6 +571,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _, kwargs = mock_kernel_client.call_args assert kwargs["max_connections"] is None assert kwargs["max_download_threads"] == 16 + assert kwargs["enable_http2"] is None finally: conn.close()