diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..a74ab7f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Add `_tls_check_certificate_revocation` and `_tls_accept_undetermined_revocation` (default `False`) for kernel connections (`use_kernel=True`); they raise `ValueError` on other backends. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index eab9847ca..6a41a1a7c 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -1223,6 +1223,28 @@ def _kernel_tls_kwargs(ssl_options: Optional[SSLOptions]) -> Dict[str, Any]: kwargs: Dict[str, Any] = {} + # Certificate revocation is opt-in. Emit only when enabled, so connections + # that don't ask for it keep working with kernel wheels that predate the setters. + for name, enabled in ( + ( + "tls_check_certificate_revocation", + ssl_options.tls_check_certificate_revocation, + ), + ( + "tls_accept_undetermined_revocation", + ssl_options.tls_accept_undetermined_revocation, + ), + ): + if enabled is not True: + continue + if not _kernel_session_accepts_kwarg(name): + raise NotSupportedError( + f"{name} requires a newer databricks-sql-kernel wheel: the installed " + "databricks-sql-kernel does not support certificate revocation. Upgrade " + "databricks-sql-kernel or remove the certificate revocation options." + ) + kwargs[name] = True + # Inverted booleans. Emit only the insecure (skip) direction so the # default secure path stays implicit. ``tls_verify=False`` disables # all chain validation, which subsumes hostname verification — so we diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..5e0a8e3a1 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,11 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + + Certificate revocation (kernel only): ``_tls_check_certificate_revocation=True`` + checks the server chain against its CRLs. ``_tls_accept_undetermined_revocation=True`` + accepts a chain whose status cannot be determined. Both default to ``False`` and + raise ``ValueError`` without ``use_kernel=True``. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..7399b00cb 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -109,6 +109,21 @@ def __init__( if self._spog_headers: all_headers = all_headers + list(self._spog_headers.items()) + # Revocation flags weaken enforcement when set, so they must be real + # bools: a truthy string such as "false" would otherwise be accepted. + check_revocation = kwargs.get("_tls_check_certificate_revocation", False) + accept_undetermined_revocation = kwargs.get( + "_tls_accept_undetermined_revocation", False + ) + for option_name, option_value in ( + ("_tls_check_certificate_revocation", check_revocation), + ("_tls_accept_undetermined_revocation", accept_undetermined_revocation), + ): + if not isinstance(option_value, bool): + raise ValueError( + f"{option_name} must be a bool; got {type(option_value).__name__}" + ) + self.ssl_options = SSLOptions( # Double negation is generally a bad thing, but we have to keep backward compatibility tls_verify=not kwargs.get( @@ -119,6 +134,8 @@ def __init__( tls_client_cert_file=kwargs.get("_tls_client_cert_file"), tls_client_cert_key_file=kwargs.get("_tls_client_cert_key_file"), tls_client_cert_key_password=kwargs.get("_tls_client_cert_key_password"), + tls_check_certificate_revocation=check_revocation, + tls_accept_undetermined_revocation=accept_undetermined_revocation, ) # Use the provided HTTP client (created in Connection) @@ -140,6 +157,11 @@ def __init__( # otherwise (OAuth M2M/U2M resolve purely from the raw kwargs # the bridge reads). The Thrift / SEA backends are unchanged. self.use_kernel = kwargs.get("use_kernel", False) + if not self.use_kernel and (check_revocation or accept_undetermined_revocation): + raise ValueError( + "_tls_check_certificate_revocation and _tls_accept_undetermined_revocation " + "require use_kernel=True; the Thrift and SEA backends do not check certificate revocation." + ) if self.use_kernel: access_token = kwargs.get("access_token") self.auth_provider = ( diff --git a/src/databricks/sql/types.py b/src/databricks/sql/types.py index 2d1ddc42a..3418a9716 100644 --- a/src/databricks/sql/types.py +++ b/src/databricks/sql/types.py @@ -40,6 +40,8 @@ def __init__( tls_client_cert_file: Optional[str] = None, tls_client_cert_key_file: Optional[str] = None, tls_client_cert_key_password: Optional[str] = None, + tls_check_certificate_revocation: bool = False, + tls_accept_undetermined_revocation: bool = False, ): self.tls_verify = tls_verify self.tls_verify_hostname = tls_verify_hostname @@ -47,6 +49,8 @@ def __init__( self.tls_client_cert_file = tls_client_cert_file self.tls_client_cert_key_file = tls_client_cert_key_file self.tls_client_cert_key_password = tls_client_cert_key_password + self.tls_check_certificate_revocation = tls_check_certificate_revocation + self.tls_accept_undetermined_revocation = tls_accept_undetermined_revocation def validate_client_identity(self) -> None: """Validate the shape of the mutual-TLS client identity. diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5bad2040b..959878211 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -2508,6 +2508,73 @@ def test_default_ssl_options_emit_no_tls_kwargs(self): def test_none_ssl_options_emit_no_tls_kwargs(self): assert kernel_client._kernel_tls_kwargs(None) == {} + @pytest.mark.parametrize( + "overrides, expected", + [ + ({}, {}), + ({"tls_check_certificate_revocation": False}, {}), + ( + {"tls_check_certificate_revocation": True}, + {"tls_check_certificate_revocation": True}, + ), + ( + {"tls_accept_undetermined_revocation": True}, + {"tls_accept_undetermined_revocation": True}, + ), + ( + { + "tls_check_certificate_revocation": True, + "tls_accept_undetermined_revocation": True, + }, + { + "tls_check_certificate_revocation": True, + "tls_accept_undetermined_revocation": True, + }, + ), + ], + ) + def test_revocation_forwarded_only_when_enabled( + self, monkeypatch, overrides, expected + ): + # Opt-in: a default connection emits no revocation kwargs, so it still works + # with kernel wheels that predate the setters. + monkeypatch.setattr( + kernel_client, "_kernel_session_accepts_kwarg", lambda _: True + ) + assert ( + kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides)) == expected + ) + + @pytest.mark.parametrize( + "overrides", + [ + {"tls_check_certificate_revocation": True}, + {"tls_accept_undetermined_revocation": True}, + ], + ) + def test_revocation_on_kernel_without_setters_raises_clear_error( + self, monkeypatch, overrides + ): + # An older kernel wheel would reject the kwarg with an opaque TypeError, + # so name the missing capability and the upgrade path instead. + monkeypatch.setattr( + kernel_client, "_kernel_session_accepts_kwarg", lambda _: False + ) + with pytest.raises( + NotSupportedError, match="does not support certificate revocation" + ): + kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides)) + + def test_default_tls_on_kernel_without_setters_emits_no_revocation_kwargs( + self, monkeypatch + ): + # The capability check runs only when revocation is requested, so the + # default path works unchanged on wheels without the setters. + monkeypatch.setattr( + kernel_client, "_kernel_session_accepts_kwarg", lambda _: False + ) + assert kernel_client._kernel_tls_kwargs(self._ssl_options()) == {} + def test_verify_off_maps_to_skip_verify_and_hostname(self): # tls_verify=False disables all chain validation, which subsumes # hostname verification — so both skip flags are emitted, matching diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..428b4bc1d 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -572,6 +572,46 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): finally: conn.close() + def test_revocation_options_threaded_into_kernel_ssl_options(self): + import sys + import types + + pytest.importorskip( + "pyarrow", + reason="kernel client module imports pyarrow at load", + ) + + fake = types.ModuleType("databricks_sql_kernel") + fake.KernelError = type("KernelError", (Exception,), {}) + fake.Session = MagicMock() + + with patch.dict(sys.modules, {"databricks_sql_kernel": fake}), patch( + "databricks.sql.backend.kernel.client.KernelDatabricksClient" + ) as mock_kernel_client, patch( + "%s.session.get_python_sql_connector_auth_provider" % self.PACKAGE + ): + instance = mock_kernel_client.return_value + instance.open_session.return_value = SessionId( + BackendType.SEA, "sess-id", None + ) + + conn = databricks.sql.connect( + server_hostname="foo", + http_path="/sql/1.0/warehouses/abc", + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + _tls_check_certificate_revocation=True, + _tls_accept_undetermined_revocation=True, + ) + try: + _, kwargs = mock_kernel_client.call_args + ssl_options = kwargs["ssl_options"] + assert ssl_options.tls_check_certificate_revocation is True + assert ssl_options.tls_accept_undetermined_revocation is True + finally: + conn.close() + def test_azure_sp_m2m_kwargs_threaded_into_kernel_auth_options(self): # The Azure SP credentials a user passes to connect() must reach the # kernel auth bridge via auth_options; without this threading the @@ -1184,3 +1224,70 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): assert kwargs["auth_options"].get("auth_type") is None finally: conn.close() + + +class TestCertificateRevocationRequiresKernel: + """AC-R8: revocation kwargs are kernel-only. Enabling either on the Thrift or SEA + backend raises ValueError before any connection is made.""" + + @pytest.mark.parametrize( + "option, extra", + [ + ("_tls_check_certificate_revocation", {}), + ("_tls_accept_undetermined_revocation", {}), + ("_tls_check_certificate_revocation", {"use_sea": True}), + ("_tls_accept_undetermined_revocation", {"use_sea": True}), + ], + ) + def test_enabled_on_non_kernel_backend_raises(self, option, extra): + from databricks.sql.session import Session + + with pytest.raises(ValueError, match="require use_kernel=True"): + Session( + "h.example.com", + "/sql/1.0/warehouses/abc", + None, + **{option: True}, + **extra, + ) + + @patch("databricks.sql.session.get_python_sql_connector_auth_provider") + @patch("databricks.sql.session.Session._create_backend") + def test_disabled_on_thrift_is_not_rejected(self, mock_backend, _mock_auth): + # False is the default and must not trip the kernel-only check. + from databricks.sql.session import Session + + Session( + server_hostname="h.example.com", + http_path="/sql/1.0/warehouses/abc", + http_client=MagicMock(), + http_headers=[], + enable_telemetry=False, + _tls_check_certificate_revocation=False, + _tls_accept_undetermined_revocation=False, + ) + mock_backend.assert_called_once() + + @pytest.mark.parametrize("extra", [{}, {"use_sea": True}]) + @pytest.mark.parametrize( + "option", + ["_tls_check_certificate_revocation", "_tls_accept_undetermined_revocation"], + ) + @pytest.mark.parametrize("value", ["false", 1, None]) + @patch("databricks.sql.session.Session._create_backend") + def test_non_bool_revocation_value_is_rejected_before_connecting( + self, mock_backend, value, option, extra + ): + # Truthiness would let "false" enable accept-undetermined, so only real + # bools pass. The check runs before any backend is created, on every path. + from databricks.sql.session import Session + + with pytest.raises(ValueError, match=f"{option} must be a bool"): + Session( + "h.example.com", + "/sql/1.0/warehouses/abc", + None, + **{option: value}, + **extra, + ) + mock_backend.assert_not_called()