From 449e23efff82e3bdbad1e732b210ea2ca95345c0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 03:07:51 +0000 Subject: [PATCH 1/9] feat(kernel): expose HTTPS protocol negotiation Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 1 + README.md | 6 ++++ src/databricks/sql/backend/kernel/client.py | 15 ++++++++ src/databricks/sql/client.py | 6 ++++ src/databricks/sql/session.py | 1 + tests/unit/test_kernel_client.py | 39 +++++++++++++++++++++ tests/unit/test_session.py | 3 ++ 7 files changed, 71 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..f0d86877f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. Explicit settings require a kernel wheel with HTTP/2 configuration support. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/README.md b/README.md index 36162e69b..d33fc41fc 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,12 @@ Install using `pip install databricks-sql-connector` Install using `pip install databricks-sql-connector[pyarrow]` ### Installing with the Rust kernel backend (`use_kernel=True`) + +The kernel negotiates HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the +peer does not support HTTP/2. Pass `enable_http2=False` with `use_kernel=True` to +force HTTP/1.1 for authentication, SQL requests, result downloads, and telemetry. +This setting requires a kernel wheel with HTTP/2 configuration support. + Install using `pip install databricks-sql-connector[kernel]` This adds the optional [`databricks-sql-kernel`](https://pypi.org/project/databricks-sql-kernel/) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index eab9847ca..ebb658fc2 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -218,6 +218,18 @@ def _kernel_session_accepts_kwarg(name: str) -> bool: return support.accepts_kwargs or name in support.parameters +def _kernel_http2_kwargs(value: Optional[bool]) -> Dict[str, bool]: + if value is None: + return {} + if not isinstance(value, bool): + raise ValueError("enable_http2 must be a bool") + if not _kernel_session_accepts_kwarg("enable_http2"): + raise NotSupportedError( + "enable_http2 requires a databricks-sql-kernel wheel with HTTP/2 configuration support." + ) + return {"enable_http2": value} + + def _kernel_geospatial_kwargs(value: bool) -> Dict[str, bool]: """Build the explicit native geospatial-support kwarg. @@ -346,6 +358,7 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") + self._enable_http2 = kwargs.get("enable_http2") self._use_cloud_fetch = kwargs.get("use_cloud_fetch", True) self._max_download_threads = kwargs.get("max_download_threads", 16) if ( @@ -485,6 +498,7 @@ def open_session( # kernel's ``retry_*`` kwargs. Empty when at defaults. retry_kwargs = _kernel_retry_kwargs(self._retry_options) telemetry_kwargs = _kernel_telemetry_kwargs(self._telemetry_options) + http2_kwargs = _kernel_http2_kwargs(self._enable_http2) geospatial_kwargs = _kernel_geospatial_kwargs( self._enable_geospatial_support ) @@ -538,6 +552,7 @@ def open_session( **geospatial_kwargs, **max_connections_kwargs, **http_headers_kwargs, + **http2_kwargs, ) except Exception as exc: raise _wrap_kernel_exception("open_session", exc) from exc diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..fb8af39a0 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,12 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + :param enable_http2: `bool`, optional (kernel default is True) + Kernel backend only. Negotiate HTTP/2 for HTTPS requests, with + HTTP/1.1 fallback when the peer does not support HTTP/2. Set + ``False`` to force HTTP/1.1 for authentication, SQL requests, + result downloads, and telemetry. Requires a kernel wheel with + HTTP/2 configuration support. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..23693141c 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -337,6 +337,7 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, + enable_http2=kwargs.get("enable_http2"), use_cloud_fetch=kwargs.get("use_cloud_fetch", True), max_download_threads=kwargs.get("max_download_threads", 16), enable_geospatial_support=kwargs.get( diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5bad2040b..31d0ef7be 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -407,6 +407,45 @@ def fake_session(**options): assert configuration == original +@pytest.mark.parametrize("enabled", [None, True, False]) +def test_open_session_preserves_http2_option(monkeypatch, enabled): + captured = {} + + def fake_session(**kwargs): + captured.update(kwargs) + session = MagicMock() + session.session_id = "sess-id" + return session + + monkeypatch.setattr(kernel_client._kernel, "Session", fake_session) + client = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + **({"enable_http2": enabled} if enabled is not None else {}), + ) + client.open_session(session_configuration=None, catalog=None, schema=None) + if enabled is None: + assert "enable_http2" not in captured + else: + assert captured["enable_http2"] is enabled + + +@pytest.mark.parametrize("value", ["false", 0, 1]) +def test_http2_option_rejects_non_boolean(value): + with pytest.raises(ValueError, match="enable_http2 must be a bool"): + kernel_client._kernel_http2_kwargs(value) + + +def test_http2_option_rejects_unsupported_kernel(monkeypatch): + monkeypatch.setattr(kernel_client._kernel, "Session", lambda *, host: None) + assert kernel_client._kernel_http2_kwargs(None) == {} + for enabled in [True, False]: + with pytest.raises(kernel_client.NotSupportedError, match="HTTP/2"): + kernel_client._kernel_http2_kwargs(enabled) + + @pytest.mark.parametrize("timeout", [None, 0, 12.5]) def test_open_session_passes_request_timeout_to_kernel(monkeypatch, timeout): captured = {} diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..08b6668a7 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -540,6 +540,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _pool_maxsize=41, use_cloud_fetch=False, max_download_threads=4, + enable_http2=False, enable_geospatial_support=False, ) try: @@ -553,6 +554,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): assert kwargs["max_connections"] == 41 assert kwargs["use_cloud_fetch"] is False assert kwargs["max_download_threads"] == 4 + assert kwargs["enable_http2"] is False assert kwargs["enable_geospatial_support"] is False finally: conn.close() @@ -569,6 +571,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _, kwargs = mock_kernel_client.call_args assert kwargs["max_connections"] is None assert kwargs["max_download_threads"] == 16 + assert kwargs["enable_http2"] is None finally: conn.close() From faa36ab6d7f9a2ba21b9ac6a046e279db674f7ac Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 03:19:04 +0000 Subject: [PATCH 2/9] refactor(kernel): pass HTTP/2 options to the native session Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- README.md | 1 - src/databricks/sql/backend/kernel/client.py | 4 ---- src/databricks/sql/client.py | 3 +-- tests/unit/test_kernel_client.py | 8 -------- 5 files changed, 2 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d86877f..0ace7ab44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. Explicit settings require a kernel wheel with HTTP/2 configuration support. +- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/README.md b/README.md index d33fc41fc..e42865ede 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ Install using `pip install databricks-sql-connector[pyarrow]` The kernel negotiates HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the peer does not support HTTP/2. Pass `enable_http2=False` with `use_kernel=True` to force HTTP/1.1 for authentication, SQL requests, result downloads, and telemetry. -This setting requires a kernel wheel with HTTP/2 configuration support. Install using `pip install databricks-sql-connector[kernel]` diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index ebb658fc2..a25a9063d 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -223,10 +223,6 @@ def _kernel_http2_kwargs(value: Optional[bool]) -> Dict[str, bool]: return {} if not isinstance(value, bool): raise ValueError("enable_http2 must be a bool") - if not _kernel_session_accepts_kwarg("enable_http2"): - raise NotSupportedError( - "enable_http2 requires a databricks-sql-kernel wheel with HTTP/2 configuration support." - ) return {"enable_http2": value} diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index fb8af39a0..4bbd1e9cc 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -197,8 +197,7 @@ def __init__( Kernel backend only. Negotiate HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the peer does not support HTTP/2. Set ``False`` to force HTTP/1.1 for authentication, SQL requests, - result downloads, and telemetry. Requires a kernel wheel with - HTTP/2 configuration support. + result downloads, and telemetry. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 31d0ef7be..15cdaef38 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -438,14 +438,6 @@ def test_http2_option_rejects_non_boolean(value): kernel_client._kernel_http2_kwargs(value) -def test_http2_option_rejects_unsupported_kernel(monkeypatch): - monkeypatch.setattr(kernel_client._kernel, "Session", lambda *, host: None) - assert kernel_client._kernel_http2_kwargs(None) == {} - for enabled in [True, False]: - with pytest.raises(kernel_client.NotSupportedError, match="HTTP/2"): - kernel_client._kernel_http2_kwargs(enabled) - - @pytest.mark.parametrize("timeout", [None, 0, 12.5]) def test_open_session_passes_request_timeout_to_kernel(monkeypatch, timeout): captured = {} From 1b895c6f4ded601dd1a4ef1cdfd72bb7e42e092d Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 20:00:02 +0000 Subject: [PATCH 3/9] docs: describe HTTP/1.1 behavior with proxies Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- README.md | 1 + src/databricks/sql/client.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ace7ab44..9bfe19a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. +- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. Configured proxies force HTTP/1.1. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/README.md b/README.md index e42865ede..8b83453b0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Install using `pip install databricks-sql-connector[pyarrow]` The kernel negotiates HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the peer does not support HTTP/2. Pass `enable_http2=False` with `use_kernel=True` to force HTTP/1.1 for authentication, SQL requests, result downloads, and telemetry. +Configured proxies force HTTP/1.1 for the entire kernel client. Install using `pip install databricks-sql-connector[kernel]` diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 4bbd1e9cc..b462797f8 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -197,7 +197,8 @@ def __init__( Kernel backend only. Negotiate HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the peer does not support HTTP/2. Set ``False`` to force HTTP/1.1 for authentication, SQL requests, - result downloads, and telemetry. + result downloads, and telemetry. Configured proxies force + HTTP/1.1 for the entire kernel client. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns From 12855a7f961646e21ab2fa52bcaf41dabb454289 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 20:08:33 +0000 Subject: [PATCH 4/9] docs: clarify environment proxy protocol settings Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- README.md | 3 ++- src/databricks/sql/client.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bfe19a44..d8da809f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. Configured proxies force HTTP/1.1. +- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. Nonempty HTTP_PROXY, HTTPS_PROXY, ALL_PROXY or lowercase equivalents force HTTP/1.1. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/README.md b/README.md index 8b83453b0..0d6cb43b0 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ Install using `pip install databricks-sql-connector[pyarrow]` The kernel negotiates HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the peer does not support HTTP/2. Pass `enable_http2=False` with `use_kernel=True` to force HTTP/1.1 for authentication, SQL requests, result downloads, and telemetry. -Configured proxies force HTTP/1.1 for the entire kernel client. +Nonempty `HTTP_PROXY`, `HTTPS_PROXY`, or `ALL_PROXY` environment variables +(including lowercase forms) force HTTP/1.1 for the entire kernel client. Install using `pip install databricks-sql-connector[kernel]` diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index b462797f8..e861f9102 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -197,8 +197,9 @@ def __init__( Kernel backend only. Negotiate HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the peer does not support HTTP/2. Set ``False`` to force HTTP/1.1 for authentication, SQL requests, - result downloads, and telemetry. Configured proxies force - HTTP/1.1 for the entire kernel client. + result downloads, and telemetry. Nonempty HTTP_PROXY, HTTPS_PROXY, + ALL_PROXY or lowercase equivalents force HTTP/1.1 for the + entire kernel client. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns From 9a88f321d579d81bcaa4998e701a46c928bf89f8 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 21:41:46 +0000 Subject: [PATCH 5/9] fix: validate HTTP/2 connection option before session open Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 5 ++--- tests/unit/test_kernel_client.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index a25a9063d..58821cc98 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -354,7 +354,7 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") - self._enable_http2 = kwargs.get("enable_http2") + self._http2_kwargs = _kernel_http2_kwargs(kwargs.get("enable_http2")) self._use_cloud_fetch = kwargs.get("use_cloud_fetch", True) self._max_download_threads = kwargs.get("max_download_threads", 16) if ( @@ -494,7 +494,6 @@ def open_session( # kernel's ``retry_*`` kwargs. Empty when at defaults. retry_kwargs = _kernel_retry_kwargs(self._retry_options) telemetry_kwargs = _kernel_telemetry_kwargs(self._telemetry_options) - http2_kwargs = _kernel_http2_kwargs(self._enable_http2) geospatial_kwargs = _kernel_geospatial_kwargs( self._enable_geospatial_support ) @@ -548,7 +547,7 @@ def open_session( **geospatial_kwargs, **max_connections_kwargs, **http_headers_kwargs, - **http2_kwargs, + **self._http2_kwargs, ) except Exception as exc: raise _wrap_kernel_exception("open_session", exc) from exc diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 15cdaef38..819cdcbb6 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -433,9 +433,24 @@ def fake_session(**kwargs): @pytest.mark.parametrize("value", ["false", 0, 1]) -def test_http2_option_rejects_non_boolean(value): +def test_http2_option_rejects_non_boolean_through_connect(monkeypatch, value): + from databricks import sql + + native_session = MagicMock() + auth_kwargs = MagicMock() + monkeypatch.setattr(kernel_client._kernel, "Session", native_session) + monkeypatch.setattr(kernel_client, "kernel_auth_kwargs", auth_kwargs) with pytest.raises(ValueError, match="enable_http2 must be a bool"): - kernel_client._kernel_http2_kwargs(value) + sql.connect( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + access_token="dapi-test", + use_kernel=True, + enable_telemetry=False, + enable_http2=value, + ) + native_session.assert_not_called() + auth_kwargs.assert_not_called() @pytest.mark.parametrize("timeout", [None, 0, 12.5]) From cbbd3421438604687745fcb8fd15eae29ffddd12 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 23:56:31 +0000 Subject: [PATCH 6/9] docs: clarify kernel HTTP protocol options Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- README.md | 9 +++------ src/databricks/sql/client.py | 9 +++------ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8da809f4..234877da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Add kernel-only `enable_http2`: HTTPS negotiates HTTP/2 with HTTP/1.1 fallback; set `enable_http2=False` to force HTTP/1.1 for all kernel-owned HTTPS traffic. Nonempty HTTP_PROXY, HTTPS_PROXY, ALL_PROXY or lowercase equivalents force HTTP/1.1. +- Kernel HTTPS traffic negotiates HTTP/2 with HTTP/1.1 fallback. Set `enable_http2=False` to force HTTP/1.1. Configured proxies also force HTTP/1.1. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/README.md b/README.md index 0d6cb43b0..595505c59 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,6 @@ Install using `pip install databricks-sql-connector[pyarrow]` ### Installing with the Rust kernel backend (`use_kernel=True`) -The kernel negotiates HTTP/2 for HTTPS requests, with HTTP/1.1 fallback when the -peer does not support HTTP/2. Pass `enable_http2=False` with `use_kernel=True` to -force HTTP/1.1 for authentication, SQL requests, result downloads, and telemetry. -Nonempty `HTTP_PROXY`, `HTTPS_PROXY`, or `ALL_PROXY` environment variables -(including lowercase forms) force HTTP/1.1 for the entire kernel client. - Install using `pip install databricks-sql-connector[kernel]` This adds the optional [`databricks-sql-kernel`](https://pypi.org/project/databricks-sql-kernel/) @@ -59,6 +53,9 @@ connection = sql.connect( ``` Notes: +- HTTPS uses HTTP/2 with HTTP/1.1 fallback. `enable_http2=False` or configured + proxy environment variables force HTTP/1.1 for all kernel traffic, including + bypassed destinations. - Requires **Python >= 3.10** (the kernel wheel is published as `cp310-abi3`). On older interpreters the `[kernel]` extra installs nothing and `use_kernel=True` raises an `ImportError`. diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index e861f9102..6d644324a 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -194,12 +194,9 @@ def __init__( Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. :param enable_http2: `bool`, optional (kernel default is True) - Kernel backend only. Negotiate HTTP/2 for HTTPS requests, with - HTTP/1.1 fallback when the peer does not support HTTP/2. Set - ``False`` to force HTTP/1.1 for authentication, SQL requests, - result downloads, and telemetry. Nonempty HTTP_PROXY, HTTPS_PROXY, - ALL_PROXY or lowercase equivalents force HTTP/1.1 for the - entire kernel client. + Kernel backend only. Negotiate HTTP/2 for all kernel HTTPS + traffic, with HTTP/1.1 fallback. ``False`` or configured proxy + environment variables force HTTP/1.1, including bypassed hosts. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns From 5f7db3ef40e5568e237ec68b0e107de767d80b9b Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 19 Nov 2026 17:29:31 +0000 Subject: [PATCH 7/9] test(kernel): simplify HTTP/2 option coverage Signed-off-by: Aakash Saravanan --- README.md | 6 +++--- src/databricks/sql/client.py | 3 ++- tests/unit/test_kernel_client.py | 20 ++------------------ 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 595505c59..1fad84d3c 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ Install using `pip install databricks-sql-connector` Install using `pip install databricks-sql-connector[pyarrow]` ### Installing with the Rust kernel backend (`use_kernel=True`) - Install using `pip install databricks-sql-connector[kernel]` This adds the optional [`databricks-sql-kernel`](https://pypi.org/project/databricks-sql-kernel/) @@ -54,8 +53,9 @@ connection = sql.connect( Notes: - HTTPS uses HTTP/2 with HTTP/1.1 fallback. `enable_http2=False` or configured - proxy environment variables force HTTP/1.1 for all kernel traffic, including - bypassed destinations. + proxy environment variables force HTTP/1.1 for session requests, including + bypassed destinations. Telemetry may reuse an earlier session's HTTP client + for the same host. - Requires **Python >= 3.10** (the kernel wheel is published as `cp310-abi3`). On older interpreters the `[kernel]` extra installs nothing and `use_kernel=True` raises an `ImportError`. diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 6d644324a..38bf02982 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -194,9 +194,10 @@ def __init__( Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. :param enable_http2: `bool`, optional (kernel default is True) - Kernel backend only. Negotiate HTTP/2 for all kernel HTTPS + Kernel backend only. Negotiate HTTP/2 for session HTTPS traffic, with HTTP/1.1 fallback. ``False`` or configured proxy environment variables force HTTP/1.1, including bypassed hosts. + Telemetry may reuse an earlier session's HTTP client for the same host. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 819cdcbb6..750bfe635 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -432,25 +432,9 @@ def fake_session(**kwargs): assert captured["enable_http2"] is enabled -@pytest.mark.parametrize("value", ["false", 0, 1]) -def test_http2_option_rejects_non_boolean_through_connect(monkeypatch, value): - from databricks import sql - - native_session = MagicMock() - auth_kwargs = MagicMock() - monkeypatch.setattr(kernel_client._kernel, "Session", native_session) - monkeypatch.setattr(kernel_client, "kernel_auth_kwargs", auth_kwargs) +def test_http2_option_rejects_non_boolean(): with pytest.raises(ValueError, match="enable_http2 must be a bool"): - sql.connect( - server_hostname="example.cloud.databricks.com", - http_path="/sql/1.0/warehouses/abc", - access_token="dapi-test", - use_kernel=True, - enable_telemetry=False, - enable_http2=value, - ) - native_session.assert_not_called() - auth_kwargs.assert_not_called() + kernel_client._kernel_http2_kwargs("false") @pytest.mark.parametrize("timeout", [None, 0, 12.5]) From d517411f9d00248864a950decd59901f2a16dac7 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 19 Nov 2026 18:12:13 +0000 Subject: [PATCH 8/9] fix(kernel): check support for HTTP/2 options Signed-off-by: Aakash Saravanan --- CONNECTION_PARAMETERS.md | 1 + README.md | 3 +-- src/databricks/sql/backend/kernel/client.py | 5 +++++ src/databricks/sql/client.py | 1 - tests/unit/test_kernel_client.py | 15 +++++++++++++++ 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index f0b3d2ee7..88ee4edd8 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -114,6 +114,7 @@ to change without notice. | Option | Type | Thrift | Kernel | Default Value | Note | | ------------------------------------ | ----------- | :----: | :----: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `enable_http2` | `bool` | ❌ | ✅ | `None` (kernel default `True`) | Negotiate HTTP/2 for HTTPS with HTTP/1.1 fallback. `False` or configured proxies force HTTP/1.1. An explicit value requires a supporting kernel wheel; otherwise raises `NotSupportedError`. | | `_socket_timeout` | `float` (s) | ✅ | ✅ | `900` (Thrift); `120` (kernel) | Thrift: socket send/recv/connect timeout. Kernel: total HTTP request deadline from connect through response-body completion. A positive value is forwarded; unset or `0` selects the kernel's 120s default. On the kernel path, `0` is neither unlimited nor an immediate timeout. | | `_pool_connections` | `int` | ⚠️ | ❌ | `10` | Number of pools in the shared Python HTTP client. On Thrift this affects auxiliary auth/feature-flag/telemetry traffic, not the separate Thrift query transport. Kernel-owned network traffic does not use this client. | | `_pool_maxsize` | `int` | ⚠️ | ⚠️ | `20` (Python); `100` (kernel when unset) | Max idle connections retained per host. It configures the shared Python client but not the Thrift query transport. With kernel ≥ 1.1.0, a positive value also configures the kernel's Rust HTTP pool. Unset or `0` keeps each client's default. | diff --git a/README.md b/README.md index 1fad84d3c..b890bd992 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,7 @@ connection = sql.connect( Notes: - HTTPS uses HTTP/2 with HTTP/1.1 fallback. `enable_http2=False` or configured proxy environment variables force HTTP/1.1 for session requests, including - bypassed destinations. Telemetry may reuse an earlier session's HTTP client - for the same host. + bypassed destinations. - Requires **Python >= 3.10** (the kernel wheel is published as `cp310-abi3`). On older interpreters the `[kernel]` extra installs nothing and `use_kernel=True` raises an `ImportError`. diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 58821cc98..11897bc39 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -223,6 +223,11 @@ def _kernel_http2_kwargs(value: Optional[bool]) -> Dict[str, bool]: return {} if not isinstance(value, bool): raise ValueError("enable_http2 must be a bool") + if not _kernel_session_accepts_kwarg("enable_http2"): + raise NotSupportedError( + "enable_http2 requires a newer databricks-sql-kernel wheel " + "that supports HTTP/2 configuration." + ) return {"enable_http2": value} diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 38bf02982..a932219d2 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -197,7 +197,6 @@ def __init__( Kernel backend only. Negotiate HTTP/2 for session HTTPS traffic, with HTTP/1.1 fallback. ``False`` or configured proxy environment variables force HTTP/1.1, including bypassed hosts. - Telemetry may reuse an earlier session's HTTP client for the same host. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 750bfe635..2cfc769bd 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -437,6 +437,21 @@ def test_http2_option_rejects_non_boolean(): kernel_client._kernel_http2_kwargs("false") +def test_http2_option_requires_kernel_support(monkeypatch): + def old_session(): + pytest.fail("unsupported HTTP/2 option reached Session") + + monkeypatch.setattr(kernel_client._kernel, "Session", old_session) + with pytest.raises(NotSupportedError, match="enable_http2 requires a newer"): + kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + enable_http2=False, + ) + + @pytest.mark.parametrize("timeout", [None, 0, 12.5]) def test_open_session_passes_request_timeout_to_kernel(monkeypatch, timeout): captured = {} From 6bc721fef46357df90a71e95d48a167d47141785 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 02:43:04 +0000 Subject: [PATCH 9/9] refactor(kernel): group HTTP/2 option validation Signed-off-by: Aakash Saravanan --- src/databricks/sql/backend/kernel/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 11897bc39..1ecc2a7f9 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -359,7 +359,6 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") - self._http2_kwargs = _kernel_http2_kwargs(kwargs.get("enable_http2")) self._use_cloud_fetch = kwargs.get("use_cloud_fetch", True) self._max_download_threads = kwargs.get("max_download_threads", 16) if ( @@ -368,6 +367,7 @@ def __init__( or self._max_download_threads <= 0 ): raise ValueError("max_download_threads must be a positive integer") + self._http2_kwargs = _kernel_http2_kwargs(kwargs.get("enable_http2")) # Opt-in native GEOMETRY / GEOGRAPHY support. True requests # the canonical Arrow struct and surfaces as ``{"srid": int, "wkb": # bytes}`` through pyarrow; False requests WKT / EWKT strings. This is