From 15275dfe0a45259bf498b1d567c32d5b043c79e5 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:14:33 +0000 Subject: [PATCH 1/2] feat(kernel): add opt-in certificate revocation checks for kernel connections Expose the kernel's TLS revocation policy to Python connections on the kernel backend (use_kernel=True): - _tls_check_certificate_revocation (default False) checks the server certificate chain against its CRLs. - _tls_accept_undetermined_revocation (default False) accepts a chain whose revocation status cannot be determined. The two flags are carried on SSLOptions. _kernel_tls_kwargs forwards each one to the kernel Session as tls_check_certificate_revocation / tls_accept_undetermined_revocation only when it is True, so default connections still work with kernel wheels that predate the setters. Enabling either flag on the Thrift or SEA backend raises ValueError before any connection is made. Revocation is a kernel-only feature. Tests cover the forwarding table in _kernel_tls_kwargs and the rejection on the non-kernel backends. The existing realkernel routing tests still pass against databricks-sql-kernel 1.4.0. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CHANGELOG.md | 1 + src/databricks/sql/backend/kernel/client.py | 7 ++++ src/databricks/sql/client.py | 5 +++ src/databricks/sql/session.py | 14 +++++++ src/databricks/sql/types.py | 4 ++ tests/unit/test_kernel_client.py | 32 +++++++++++++++ tests/unit/test_session.py | 43 +++++++++++++++++++++ 7 files changed, 106 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..a74ab7f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Add `_tls_check_certificate_revocation` and `_tls_accept_undetermined_revocation` (default `False`) for kernel connections (`use_kernel=True`); they raise `ValueError` on other backends. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index eab9847ca..1d1c07daf 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -1223,6 +1223,13 @@ def _kernel_tls_kwargs(ssl_options: Optional[SSLOptions]) -> Dict[str, Any]: kwargs: Dict[str, Any] = {} + # Certificate revocation is opt-in. Emit only when enabled, so connections + # that don't ask for it keep working with kernel wheels that predate the setters. + if ssl_options.tls_check_certificate_revocation: + kwargs["tls_check_certificate_revocation"] = True + if ssl_options.tls_accept_undetermined_revocation: + kwargs["tls_accept_undetermined_revocation"] = True + # Inverted booleans. Emit only the insecure (skip) direction so the # default secure path stays implicit. ``tls_verify=False`` disables # all chain validation, which subsumes hostname verification — so we diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..5e0a8e3a1 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,11 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + + Certificate revocation (kernel only): ``_tls_check_certificate_revocation=True`` + checks the server chain against its CRLs. ``_tls_accept_undetermined_revocation=True`` + accepts a chain whose status cannot be determined. Both default to ``False`` and + raise ``ValueError`` without ``use_kernel=True``. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..79b453fc4 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -119,6 +119,12 @@ def __init__( tls_client_cert_file=kwargs.get("_tls_client_cert_file"), tls_client_cert_key_file=kwargs.get("_tls_client_cert_key_file"), tls_client_cert_key_password=kwargs.get("_tls_client_cert_key_password"), + tls_check_certificate_revocation=kwargs.get( + "_tls_check_certificate_revocation", False + ), + tls_accept_undetermined_revocation=kwargs.get( + "_tls_accept_undetermined_revocation", False + ), ) # Use the provided HTTP client (created in Connection) @@ -140,6 +146,14 @@ def __init__( # otherwise (OAuth M2M/U2M resolve purely from the raw kwargs # the bridge reads). The Thrift / SEA backends are unchanged. self.use_kernel = kwargs.get("use_kernel", False) + if not self.use_kernel and ( + kwargs.get("_tls_check_certificate_revocation") + or kwargs.get("_tls_accept_undetermined_revocation") + ): + raise ValueError( + "_tls_check_certificate_revocation and _tls_accept_undetermined_revocation " + "require use_kernel=True; the Thrift and SEA backends do not check certificate revocation." + ) if self.use_kernel: access_token = kwargs.get("access_token") self.auth_provider = ( diff --git a/src/databricks/sql/types.py b/src/databricks/sql/types.py index 2d1ddc42a..3418a9716 100644 --- a/src/databricks/sql/types.py +++ b/src/databricks/sql/types.py @@ -40,6 +40,8 @@ def __init__( tls_client_cert_file: Optional[str] = None, tls_client_cert_key_file: Optional[str] = None, tls_client_cert_key_password: Optional[str] = None, + tls_check_certificate_revocation: bool = False, + tls_accept_undetermined_revocation: bool = False, ): self.tls_verify = tls_verify self.tls_verify_hostname = tls_verify_hostname @@ -47,6 +49,8 @@ def __init__( self.tls_client_cert_file = tls_client_cert_file self.tls_client_cert_key_file = tls_client_cert_key_file self.tls_client_cert_key_password = tls_client_cert_key_password + self.tls_check_certificate_revocation = tls_check_certificate_revocation + self.tls_accept_undetermined_revocation = tls_accept_undetermined_revocation def validate_client_identity(self) -> None: """Validate the shape of the mutual-TLS client identity. diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5bad2040b..9dfc05985 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -2508,6 +2508,38 @@ def test_default_ssl_options_emit_no_tls_kwargs(self): def test_none_ssl_options_emit_no_tls_kwargs(self): assert kernel_client._kernel_tls_kwargs(None) == {} + @pytest.mark.parametrize( + "overrides, expected", + [ + ({}, {}), + ({"tls_check_certificate_revocation": False}, {}), + ( + {"tls_check_certificate_revocation": True}, + {"tls_check_certificate_revocation": True}, + ), + ( + {"tls_accept_undetermined_revocation": True}, + {"tls_accept_undetermined_revocation": True}, + ), + ( + { + "tls_check_certificate_revocation": True, + "tls_accept_undetermined_revocation": True, + }, + { + "tls_check_certificate_revocation": True, + "tls_accept_undetermined_revocation": True, + }, + ), + ], + ) + def test_revocation_forwarded_only_when_enabled(self, overrides, expected): + # Opt-in: a default connection emits no revocation kwargs, so it still works + # with kernel wheels that predate the setters. + assert ( + kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides)) == expected + ) + def test_verify_off_maps_to_skip_verify_and_hostname(self): # tls_verify=False disables all chain validation, which subsumes # hostname verification — so both skip flags are emitted, matching diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..ca422a310 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1184,3 +1184,46 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): assert kwargs["auth_options"].get("auth_type") is None finally: conn.close() + + +class TestCertificateRevocationRequiresKernel: + """AC-R8: revocation kwargs are kernel-only. Enabling either on the Thrift or SEA + backend raises ValueError before any connection is made.""" + + @pytest.mark.parametrize( + "option, extra", + [ + ("_tls_check_certificate_revocation", {}), + ("_tls_accept_undetermined_revocation", {}), + ("_tls_check_certificate_revocation", {"use_sea": True}), + ("_tls_accept_undetermined_revocation", {"use_sea": True}), + ], + ) + def test_enabled_on_non_kernel_backend_raises(self, option, extra): + from databricks.sql.session import Session + + with pytest.raises(ValueError, match="require use_kernel=True"): + Session( + "h.example.com", + "/sql/1.0/warehouses/abc", + None, + **{option: True}, + **extra, + ) + + @patch("databricks.sql.session.get_python_sql_connector_auth_provider") + @patch("databricks.sql.session.Session._create_backend") + def test_disabled_on_thrift_is_not_rejected(self, mock_backend, _mock_auth): + # False is the default and must not trip the kernel-only check. + from databricks.sql.session import Session + + Session( + server_hostname="h.example.com", + http_path="/sql/1.0/warehouses/abc", + http_client=MagicMock(), + http_headers=[], + enable_telemetry=False, + _tls_check_certificate_revocation=False, + _tls_accept_undetermined_revocation=False, + ) + mock_backend.assert_called_once() From a549393f9556b08a79da6130987b2bc84a4cafae Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:51:21 +0000 Subject: [PATCH 2/2] fix(kernel): validate revocation flags and gate them on kernel support Require the _tls_check_certificate_revocation and _tls_accept_undetermined_revocation options to be real bools on both the Thrift/SEA and kernel paths, before any connection is made. A truthy non-bool such as "false" previously enabled accept-undetermined. When revocation is requested on the kernel path, check that the installed databricks_sql_kernel.Session declares the tls revocation kwargs and raise NotSupportedError with an upgrade hint otherwise. The default path is unchanged and does not run the check. Adds regression tests for the bool validation on both paths, the kernel-wheel capability error, and the connect() -> kernel ssl_options threading. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/backend/kernel/client.py | 23 ++++++-- src/databricks/sql/session.py | 28 +++++---- tests/unit/test_kernel_client.py | 37 +++++++++++- tests/unit/test_session.py | 64 +++++++++++++++++++++ 4 files changed, 137 insertions(+), 15 deletions(-) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index 1d1c07daf..6a41a1a7c 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -1225,10 +1225,25 @@ def _kernel_tls_kwargs(ssl_options: Optional[SSLOptions]) -> Dict[str, Any]: # Certificate revocation is opt-in. Emit only when enabled, so connections # that don't ask for it keep working with kernel wheels that predate the setters. - if ssl_options.tls_check_certificate_revocation: - kwargs["tls_check_certificate_revocation"] = True - if ssl_options.tls_accept_undetermined_revocation: - kwargs["tls_accept_undetermined_revocation"] = True + for name, enabled in ( + ( + "tls_check_certificate_revocation", + ssl_options.tls_check_certificate_revocation, + ), + ( + "tls_accept_undetermined_revocation", + ssl_options.tls_accept_undetermined_revocation, + ), + ): + if enabled is not True: + continue + if not _kernel_session_accepts_kwarg(name): + raise NotSupportedError( + f"{name} requires a newer databricks-sql-kernel wheel: the installed " + "databricks-sql-kernel does not support certificate revocation. Upgrade " + "databricks-sql-kernel or remove the certificate revocation options." + ) + kwargs[name] = True # Inverted booleans. Emit only the insecure (skip) direction so the # default secure path stays implicit. ``tls_verify=False`` disables diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 79b453fc4..7399b00cb 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -109,6 +109,21 @@ def __init__( if self._spog_headers: all_headers = all_headers + list(self._spog_headers.items()) + # Revocation flags weaken enforcement when set, so they must be real + # bools: a truthy string such as "false" would otherwise be accepted. + check_revocation = kwargs.get("_tls_check_certificate_revocation", False) + accept_undetermined_revocation = kwargs.get( + "_tls_accept_undetermined_revocation", False + ) + for option_name, option_value in ( + ("_tls_check_certificate_revocation", check_revocation), + ("_tls_accept_undetermined_revocation", accept_undetermined_revocation), + ): + if not isinstance(option_value, bool): + raise ValueError( + f"{option_name} must be a bool; got {type(option_value).__name__}" + ) + self.ssl_options = SSLOptions( # Double negation is generally a bad thing, but we have to keep backward compatibility tls_verify=not kwargs.get( @@ -119,12 +134,8 @@ def __init__( tls_client_cert_file=kwargs.get("_tls_client_cert_file"), tls_client_cert_key_file=kwargs.get("_tls_client_cert_key_file"), tls_client_cert_key_password=kwargs.get("_tls_client_cert_key_password"), - tls_check_certificate_revocation=kwargs.get( - "_tls_check_certificate_revocation", False - ), - tls_accept_undetermined_revocation=kwargs.get( - "_tls_accept_undetermined_revocation", False - ), + tls_check_certificate_revocation=check_revocation, + tls_accept_undetermined_revocation=accept_undetermined_revocation, ) # Use the provided HTTP client (created in Connection) @@ -146,10 +157,7 @@ def __init__( # otherwise (OAuth M2M/U2M resolve purely from the raw kwargs # the bridge reads). The Thrift / SEA backends are unchanged. self.use_kernel = kwargs.get("use_kernel", False) - if not self.use_kernel and ( - kwargs.get("_tls_check_certificate_revocation") - or kwargs.get("_tls_accept_undetermined_revocation") - ): + if not self.use_kernel and (check_revocation or accept_undetermined_revocation): raise ValueError( "_tls_check_certificate_revocation and _tls_accept_undetermined_revocation " "require use_kernel=True; the Thrift and SEA backends do not check certificate revocation." diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 9dfc05985..959878211 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -2533,13 +2533,48 @@ def test_none_ssl_options_emit_no_tls_kwargs(self): ), ], ) - def test_revocation_forwarded_only_when_enabled(self, overrides, expected): + def test_revocation_forwarded_only_when_enabled( + self, monkeypatch, overrides, expected + ): # Opt-in: a default connection emits no revocation kwargs, so it still works # with kernel wheels that predate the setters. + monkeypatch.setattr( + kernel_client, "_kernel_session_accepts_kwarg", lambda _: True + ) assert ( kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides)) == expected ) + @pytest.mark.parametrize( + "overrides", + [ + {"tls_check_certificate_revocation": True}, + {"tls_accept_undetermined_revocation": True}, + ], + ) + def test_revocation_on_kernel_without_setters_raises_clear_error( + self, monkeypatch, overrides + ): + # An older kernel wheel would reject the kwarg with an opaque TypeError, + # so name the missing capability and the upgrade path instead. + monkeypatch.setattr( + kernel_client, "_kernel_session_accepts_kwarg", lambda _: False + ) + with pytest.raises( + NotSupportedError, match="does not support certificate revocation" + ): + kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides)) + + def test_default_tls_on_kernel_without_setters_emits_no_revocation_kwargs( + self, monkeypatch + ): + # The capability check runs only when revocation is requested, so the + # default path works unchanged on wheels without the setters. + monkeypatch.setattr( + kernel_client, "_kernel_session_accepts_kwarg", lambda _: False + ) + assert kernel_client._kernel_tls_kwargs(self._ssl_options()) == {} + def test_verify_off_maps_to_skip_verify_and_hostname(self): # tls_verify=False disables all chain validation, which subsumes # hostname verification — so both skip flags are emitted, matching diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index ca422a310..428b4bc1d 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -572,6 +572,46 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): finally: conn.close() + def test_revocation_options_threaded_into_kernel_ssl_options(self): + import sys + import types + + pytest.importorskip( + "pyarrow", + reason="kernel client module imports pyarrow at load", + ) + + fake = types.ModuleType("databricks_sql_kernel") + fake.KernelError = type("KernelError", (Exception,), {}) + fake.Session = MagicMock() + + with patch.dict(sys.modules, {"databricks_sql_kernel": fake}), patch( + "databricks.sql.backend.kernel.client.KernelDatabricksClient" + ) as mock_kernel_client, patch( + "%s.session.get_python_sql_connector_auth_provider" % self.PACKAGE + ): + instance = mock_kernel_client.return_value + instance.open_session.return_value = SessionId( + BackendType.SEA, "sess-id", None + ) + + conn = databricks.sql.connect( + server_hostname="foo", + http_path="/sql/1.0/warehouses/abc", + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + _tls_check_certificate_revocation=True, + _tls_accept_undetermined_revocation=True, + ) + try: + _, kwargs = mock_kernel_client.call_args + ssl_options = kwargs["ssl_options"] + assert ssl_options.tls_check_certificate_revocation is True + assert ssl_options.tls_accept_undetermined_revocation is True + finally: + conn.close() + def test_azure_sp_m2m_kwargs_threaded_into_kernel_auth_options(self): # The Azure SP credentials a user passes to connect() must reach the # kernel auth bridge via auth_options; without this threading the @@ -1227,3 +1267,27 @@ def test_disabled_on_thrift_is_not_rejected(self, mock_backend, _mock_auth): _tls_accept_undetermined_revocation=False, ) mock_backend.assert_called_once() + + @pytest.mark.parametrize("extra", [{}, {"use_sea": True}]) + @pytest.mark.parametrize( + "option", + ["_tls_check_certificate_revocation", "_tls_accept_undetermined_revocation"], + ) + @pytest.mark.parametrize("value", ["false", 1, None]) + @patch("databricks.sql.session.Session._create_backend") + def test_non_bool_revocation_value_is_rejected_before_connecting( + self, mock_backend, value, option, extra + ): + # Truthiness would let "false" enable accept-undetermined, so only real + # bools pass. The check runs before any backend is created, on every path. + from databricks.sql.session import Session + + with pytest.raises(ValueError, match=f"{option} must be a bool"): + Session( + "h.example.com", + "/sql/1.0/warehouses/abc", + None, + **{option: value}, + **extra, + ) + mock_backend.assert_not_called()