Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Release History

# Unreleased
- Add `_tls_check_certificate_revocation` and `_tls_accept_undetermined_revocation` (default `False`) for kernel connections (`use_kernel=True`); they raise `ValueError` on other backends.
- Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`.
- Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads.
- Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629))
Expand Down
22 changes: 22 additions & 0 deletions src/databricks/sql/backend/kernel/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1223,6 +1223,28 @@ def _kernel_tls_kwargs(ssl_options: Optional[SSLOptions]) -> Dict[str, Any]:

kwargs: Dict[str, Any] = {}

# Certificate revocation is opt-in. Emit only when enabled, so connections
# that don't ask for it keep working with kernel wheels that predate the setters.
for name, enabled in (
(
"tls_check_certificate_revocation",
ssl_options.tls_check_certificate_revocation,
),
(
"tls_accept_undetermined_revocation",
ssl_options.tls_accept_undetermined_revocation,
),
):
if enabled is not True:
continue
if not _kernel_session_accepts_kwarg(name):
raise NotSupportedError(
f"{name} requires a newer databricks-sql-kernel wheel: the installed "
"databricks-sql-kernel does not support certificate revocation. Upgrade "
"databricks-sql-kernel or remove the certificate revocation options."
)
kwargs[name] = True

# Inverted booleans. Emit only the insecure (skip) direction so the
# default secure path stays implicit. ``tls_verify=False`` disables
# all chain validation, which subsumes hostname verification — so we
Expand Down
5 changes: 5 additions & 0 deletions src/databricks/sql/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,11 @@ def __init__(
decision. This is an intentional divergence from the
Thrift/SEA paths, where an explicit ``True`` can still
be suppressed by the feature flag.

Certificate revocation (kernel only): ``_tls_check_certificate_revocation=True``
checks the server chain against its CRLs. ``_tls_accept_undetermined_revocation=True``
accepts a chain whose status cannot be determined. Both default to ``False`` and
raise ``ValueError`` without ``use_kernel=True``.
:param enable_geospatial_support: `bool`, optional (default is False)
Kernel backend only. Controls the public representation of
``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns
Expand Down
22 changes: 22 additions & 0 deletions src/databricks/sql/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,21 @@ def __init__(
if self._spog_headers:
all_headers = all_headers + list(self._spog_headers.items())

# Revocation flags weaken enforcement when set, so they must be real
# bools: a truthy string such as "false" would otherwise be accepted.
check_revocation = kwargs.get("_tls_check_certificate_revocation", False)
accept_undetermined_revocation = kwargs.get(
"_tls_accept_undetermined_revocation", False
)
for option_name, option_value in (
("_tls_check_certificate_revocation", check_revocation),
("_tls_accept_undetermined_revocation", accept_undetermined_revocation),
):
if not isinstance(option_value, bool):
raise ValueError(
f"{option_name} must be a bool; got {type(option_value).__name__}"
)

self.ssl_options = SSLOptions(
# Double negation is generally a bad thing, but we have to keep backward compatibility
tls_verify=not kwargs.get(
Expand All @@ -119,6 +134,8 @@ def __init__(
tls_client_cert_file=kwargs.get("_tls_client_cert_file"),
tls_client_cert_key_file=kwargs.get("_tls_client_cert_key_file"),
tls_client_cert_key_password=kwargs.get("_tls_client_cert_key_password"),
tls_check_certificate_revocation=check_revocation,
tls_accept_undetermined_revocation=accept_undetermined_revocation,
)

# Use the provided HTTP client (created in Connection)
Expand All @@ -140,6 +157,11 @@ def __init__(
# otherwise (OAuth M2M/U2M resolve purely from the raw kwargs
# the bridge reads). The Thrift / SEA backends are unchanged.
self.use_kernel = kwargs.get("use_kernel", False)
if not self.use_kernel and (check_revocation or accept_undetermined_revocation):
raise ValueError(
"_tls_check_certificate_revocation and _tls_accept_undetermined_revocation "
"require use_kernel=True; the Thrift and SEA backends do not check certificate revocation."
)
if self.use_kernel:
access_token = kwargs.get("access_token")
self.auth_provider = (
Expand Down
4 changes: 4 additions & 0 deletions src/databricks/sql/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,17 @@ def __init__(
tls_client_cert_file: Optional[str] = None,
tls_client_cert_key_file: Optional[str] = None,
tls_client_cert_key_password: Optional[str] = None,
tls_check_certificate_revocation: bool = False,
tls_accept_undetermined_revocation: bool = False,
):
self.tls_verify = tls_verify
self.tls_verify_hostname = tls_verify_hostname
self.tls_trusted_ca_file = tls_trusted_ca_file
self.tls_client_cert_file = tls_client_cert_file
self.tls_client_cert_key_file = tls_client_cert_key_file
self.tls_client_cert_key_password = tls_client_cert_key_password
self.tls_check_certificate_revocation = tls_check_certificate_revocation
self.tls_accept_undetermined_revocation = tls_accept_undetermined_revocation

def validate_client_identity(self) -> None:
"""Validate the shape of the mutual-TLS client identity.
Expand Down
67 changes: 67 additions & 0 deletions tests/unit/test_kernel_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2508,6 +2508,73 @@ def test_default_ssl_options_emit_no_tls_kwargs(self):
def test_none_ssl_options_emit_no_tls_kwargs(self):
assert kernel_client._kernel_tls_kwargs(None) == {}

@pytest.mark.parametrize(
"overrides, expected",
[
({}, {}),
({"tls_check_certificate_revocation": False}, {}),
(
{"tls_check_certificate_revocation": True},
{"tls_check_certificate_revocation": True},
),
(
{"tls_accept_undetermined_revocation": True},
{"tls_accept_undetermined_revocation": True},
),
(
{
"tls_check_certificate_revocation": True,
"tls_accept_undetermined_revocation": True,
},
{
"tls_check_certificate_revocation": True,
"tls_accept_undetermined_revocation": True,
},
),
],
)
def test_revocation_forwarded_only_when_enabled(
self, monkeypatch, overrides, expected
):
# Opt-in: a default connection emits no revocation kwargs, so it still works
# with kernel wheels that predate the setters.
monkeypatch.setattr(
kernel_client, "_kernel_session_accepts_kwarg", lambda _: True
)
assert (
kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides)) == expected
)

@pytest.mark.parametrize(
"overrides",
[
{"tls_check_certificate_revocation": True},
{"tls_accept_undetermined_revocation": True},
],
)
def test_revocation_on_kernel_without_setters_raises_clear_error(
self, monkeypatch, overrides
):
# An older kernel wheel would reject the kwarg with an opaque TypeError,
# so name the missing capability and the upgrade path instead.
monkeypatch.setattr(
kernel_client, "_kernel_session_accepts_kwarg", lambda _: False
)
with pytest.raises(
NotSupportedError, match="does not support certificate revocation"
):
kernel_client._kernel_tls_kwargs(self._ssl_options(**overrides))

def test_default_tls_on_kernel_without_setters_emits_no_revocation_kwargs(
self, monkeypatch
):
# The capability check runs only when revocation is requested, so the
# default path works unchanged on wheels without the setters.
monkeypatch.setattr(
kernel_client, "_kernel_session_accepts_kwarg", lambda _: False
)
assert kernel_client._kernel_tls_kwargs(self._ssl_options()) == {}

def test_verify_off_maps_to_skip_verify_and_hostname(self):
# tls_verify=False disables all chain validation, which subsumes
# hostname verification — so both skip flags are emitted, matching
Expand Down
107 changes: 107 additions & 0 deletions tests/unit/test_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,46 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self):
finally:
conn.close()

def test_revocation_options_threaded_into_kernel_ssl_options(self):
import sys
import types

pytest.importorskip(
"pyarrow",
reason="kernel client module imports pyarrow at load",
)

fake = types.ModuleType("databricks_sql_kernel")
fake.KernelError = type("KernelError", (Exception,), {})
fake.Session = MagicMock()

with patch.dict(sys.modules, {"databricks_sql_kernel": fake}), patch(
"databricks.sql.backend.kernel.client.KernelDatabricksClient"
) as mock_kernel_client, patch(
"%s.session.get_python_sql_connector_auth_provider" % self.PACKAGE
):
instance = mock_kernel_client.return_value
instance.open_session.return_value = SessionId(
BackendType.SEA, "sess-id", None
)

conn = databricks.sql.connect(
server_hostname="foo",
http_path="/sql/1.0/warehouses/abc",
use_kernel=True,
access_token="dapi-xyz",
enable_telemetry=False,
_tls_check_certificate_revocation=True,
_tls_accept_undetermined_revocation=True,
)
try:
_, kwargs = mock_kernel_client.call_args
ssl_options = kwargs["ssl_options"]
assert ssl_options.tls_check_certificate_revocation is True
assert ssl_options.tls_accept_undetermined_revocation is True
finally:
conn.close()

def test_azure_sp_m2m_kwargs_threaded_into_kernel_auth_options(self):
# The Azure SP credentials a user passes to connect() must reach the
# kernel auth bridge via auth_options; without this threading the
Expand Down Expand Up @@ -1184,3 +1224,70 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift):
assert kwargs["auth_options"].get("auth_type") is None
finally:
conn.close()


class TestCertificateRevocationRequiresKernel:
"""AC-R8: revocation kwargs are kernel-only. Enabling either on the Thrift or SEA
backend raises ValueError before any connection is made."""

@pytest.mark.parametrize(
"option, extra",
[
("_tls_check_certificate_revocation", {}),
("_tls_accept_undetermined_revocation", {}),
("_tls_check_certificate_revocation", {"use_sea": True}),
("_tls_accept_undetermined_revocation", {"use_sea": True}),
],
)
def test_enabled_on_non_kernel_backend_raises(self, option, extra):
from databricks.sql.session import Session

with pytest.raises(ValueError, match="require use_kernel=True"):
Session(
"h.example.com",
"/sql/1.0/warehouses/abc",
None,
**{option: True},
**extra,
)

@patch("databricks.sql.session.get_python_sql_connector_auth_provider")
@patch("databricks.sql.session.Session._create_backend")
def test_disabled_on_thrift_is_not_rejected(self, mock_backend, _mock_auth):
# False is the default and must not trip the kernel-only check.
from databricks.sql.session import Session

Session(
server_hostname="h.example.com",
http_path="/sql/1.0/warehouses/abc",
http_client=MagicMock(),
http_headers=[],
enable_telemetry=False,
_tls_check_certificate_revocation=False,
_tls_accept_undetermined_revocation=False,
)
mock_backend.assert_called_once()

@pytest.mark.parametrize("extra", [{}, {"use_sea": True}])
@pytest.mark.parametrize(
"option",
["_tls_check_certificate_revocation", "_tls_accept_undetermined_revocation"],
)
@pytest.mark.parametrize("value", ["false", 1, None])
@patch("databricks.sql.session.Session._create_backend")
def test_non_bool_revocation_value_is_rejected_before_connecting(
self, mock_backend, value, option, extra
):
# Truthiness would let "false" enable accept-undetermined, so only real
# bools pass. The check runs before any backend is created, on every path.
from databricks.sql.session import Session

with pytest.raises(ValueError, match=f"{option} must be a bool"):
Session(
"h.example.com",
"/sql/1.0/warehouses/abc",
None,
**{option: value},
**extra,
)
mock_backend.assert_not_called()
Loading