From db9a0fe0b78d56d82d1a158788aa18b0d2276f3c Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:25:16 +0000 Subject: [PATCH 01/12] feat(kernel): honor _port and _connection_uri on use_kernel connections The kernel backend (use_kernel=True) ignored the Thrift-style _port and _connection_uri overrides. Resolve both connector-side onto the kernel's single host and http_path, with the Thrift rules: - _port (1..65535; 0 or None mean the default) folds into server_hostname as host:port. The default 443 adds nothing, IPv6 literals are bracketed, and an embedded port equal to _port is accepted. An empty server_hostname with _port raises ValueError. - _connection_uri (http(s)://host[:port]/path?query) replaces host, port, path and query, and wins over _port. Scheme-less, non-http(s), path-less and credential-bearing URIs raise ValueError. URI-only use works. - With OAuth (U2M, Azure service principal, JWT key file, or M2M client id and secret), a _connection_uri whose origin differs from https://server_hostname raises before a kernel session opens. PAT is not affected. A non-default _port with OAuth is not guarded: the token exchange goes to the same host:port, which is documented. - The SPOG x-databricks-org-id header follows the resolved path. Tests cover the resolver tables, the OAuth origin guard, the SPOG re-derivation, and a real-wheel loopback dial proving the kernel reaches the requested port over TLS for _port and for _connection_uri. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CHANGELOG.md | 1 + src/databricks/sql/client.py | 7 + src/databricks/sql/session.py | 139 ++++++++++++++++- tests/unit/test_session.py | 274 ++++++++++++++++++++++++++++++++++ 4 files changed, 419 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..e891ec4f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Honor `_port` and `_connection_uri` on kernel connections (`use_kernel=True`), as Thrift does. `_port` sends the connection to that port (default 443 adds nothing); `_connection_uri` (`http(s)://host[:port]/path?query`) sets host, port, path and query and wins over `_port`; URI-only connections work. Invalid ports, conflicting ports, an empty `server_hostname` with `_port`, and URIs without a scheme or path raise `ValueError`. With OAuth, a `_connection_uri` origin that differs from `server_hostname` raises. With OAuth and a non-default `_port`, the token exchange goes to the same host:port, unlike Thrift (documented, not guarded). - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..c925f466b 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,13 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + + Kernel overrides: _port sends the connection to that port + (the default 443 adds nothing). _connection_uri (https://host[:port]/path?query) + sets host, port, path and query, as on Thrift, and wins over _port. + With OAuth, _connection_uri must have the same origin as + server_hostname. With OAuth and a non-default _port, the token + exchange goes to the same host:port, unlike Thrift. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..e7f140cab 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -1,6 +1,7 @@ import logging import re from functools import cached_property +from urllib.parse import urlsplit from typing import Dict, Tuple, List, Optional, Any, Type, TYPE_CHECKING from databricks.sql.types import SSLOptions @@ -47,6 +48,126 @@ def __getattr__(name: str) -> Any: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +_OAUTH_AUTH_TYPES = ("databricks-oauth", "azure-oauth", "azure-sp-m2m") +_HOSTPORT_RE = re.compile(r"(\[[^\]]*\]|(?!\[)[^:]*)(?::([0-9]+))?") + + +def _port_or_none(value: Any) -> Optional[int]: + """Validate ``_port``. ``None`` and ``0`` mean the default (Thrift: ``port or 443``).""" + if value is None: + return None + text = ( + str(value).strip() + if isinstance(value, (int, str)) and not isinstance(value, bool) + else "" + ) + if not re.fullmatch(r"[0-9]+", text) or not int(text) <= 65535: + raise ValueError( + f"Invalid _port {value!r}: expected an integer between 1 and 65535" + ) + return int(text) or None + + +def _split_host(host: str) -> Tuple[str, str, Optional[int]]: + """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``; ``name`` keeps IPv6 brackets.""" + prefix, sep, text = host.rstrip("/").rpartition("://") + prefix += sep + match = _HOSTPORT_RE.fullmatch(text) + if match is None: + if text.startswith("[") or text.count(":") == 1: + raise ValueError( + f"Invalid server_hostname {host!r}: malformed IPv6 literal or port" + ) + return prefix, text, None # an unbracketed IPv6 literal such as ::1 has no port + name, port_text = match.group(1), match.group(2) + if port_text is None: + return prefix, name, None + if not 1 <= int(port_text) <= 65535: + raise ValueError( + f"Invalid server_hostname {host!r}: port {port_text} is out of range" + ) + return prefix, name, int(port_text) + + +def _uri_host_and_path(uri: str) -> Tuple[str, str]: + """Split a ``_connection_uri`` into the kernel host (scheme and authority) and path (with query).""" + parts = urlsplit(uri) + try: + port = parts.port + except ValueError: + port = 0 + if ( + parts.scheme not in ("http", "https") + or not parts.hostname + or "@" in parts.netloc + or port == 0 + or parts.path in ("", "/") + ): + raise ValueError( + f"Invalid _connection_uri {uri!r}: expected http(s)://host[:port]/path with a " + "warehouse path and no credentials" + ) + return f"{parts.scheme}://{parts.netloc}", parts.path + ( + f"?{parts.query}" if parts.query else "" + ) + + +def _kernel_host_and_path( + server_hostname: str, http_path: Optional[str], kwargs: dict +) -> Tuple[str, Optional[str]]: + """Resolve the kernel ``(host, http_path)``. ``_connection_uri`` replaces both and wins over + ``_port``; ``_port`` folds into the host as ``host:port`` (default 443 adds nothing). + """ + if kwargs.get("_connection_uri"): + return _uri_host_and_path(kwargs["_connection_uri"]) + port = _port_or_none(kwargs.get("_port")) + prefix, name, embedded = _split_host(server_hostname) + if port is None: + return server_hostname, http_path + if not server_hostname: + raise ValueError( + "server_hostname is required on use_kernel=True when _port is set " + "(or pass _connection_uri, which carries its own host)" + ) + if embedded is not None: + if embedded != port: + raise ValueError( + f"Conflicting ports: server_hostname {server_hostname!r} carries {embedded} but _port={port}" + ) + return server_hostname, http_path + if port == 443: + return server_hostname, http_path + if ":" in name and not name.startswith("["): + name = f"[{name}]" + return f"{prefix}{name}:{port}", http_path + + +def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> None: + """AC-P4: with OAuth, a ``_connection_uri`` origin must equal ``https://server_hostname``. + Thrift keeps OAuth on server_hostname; the kernel uses one host for API and token calls. + URI-only use (empty server_hostname) has no server origin to compare and is not checked. + """ + oauth = ( + kwargs.get("auth_type") in _OAUTH_AUTH_TYPES + or kwargs.get("oauth_jwt_key_file") + or (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) + or kwargs.get("credentials_provider") is not None + ) + if not (kwargs.get("_connection_uri") and server_hostname and oauth): + return + _, name, port = _split_host(server_hostname) + uri = urlsplit(kernel_host) + if (uri.scheme, uri.hostname or "", uri.port or 443) != ( + "https", + name.strip("[]").lower(), + port or 443, + ): + raise ValueError( + f"OAuth requires the _connection_uri origin to match https://{server_hostname}; " + f"the kernel would send the credentials to {kernel_host}. Use a PAT or match the origin." + ) + + class Session: def __init__( self, @@ -323,9 +444,23 @@ def _create_backend( "_telemetry_circuit_breaker_enabled" ), } + kernel_host, kernel_http_path = _kernel_host_and_path( + server_hostname, http_path, kwargs + ) + _check_oauth_origin(server_hostname, kernel_host, kwargs) + if kernel_http_path != http_path: + # A _connection_uri can point at another workspace: re-derive the + # SPOG org-id header from the resolved path (a caller header still wins). + base_headers = [ + h for h in all_headers if h not in self._spog_headers.items() + ] + self._spog_headers = self._extract_spog_headers( + kernel_http_path, base_headers + ) + all_headers = base_headers + list(self._spog_headers.items()) return KernelDatabricksClient( - server_hostname=server_hostname, - http_path=http_path, + server_hostname=kernel_host, + http_path=kernel_http_path, http_headers=all_headers, auth_provider=auth_provider, ssl_options=self.ssl_options, diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..ae54f7f43 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -2,6 +2,11 @@ from unittest.mock import patch, MagicMock, Mock, PropertyMock from contextlib import contextmanager import gc +import importlib.metadata as ilm +import socket +import sys +import threading +import types from databricks.sql.thrift_api.TCLIService.ttypes import ( TOpenSessionResp, @@ -10,6 +15,9 @@ ) from databricks.sql.backend.types import SessionId, BackendType from databricks.sql.common.agent import KNOWN_AGENTS +from databricks.sql.session import _kernel_host_and_path, _check_oauth_origin + +import databricks.sql from databricks.sql.exc import ( ReydenThriftUnsupportedError, DatabaseError, @@ -1184,3 +1192,269 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): assert kwargs["auth_options"].get("auth_type") is None finally: conn.close() + + +# ---- kernel _port / _connection_uri overrides (use_kernel=True) ---- + +_P = "/sql/1.0/warehouses/abc" + + +@pytest.mark.parametrize( + "server, hpath, kwargs, expected", + [ + # AC-P1: _port folds into host:port; 443 adds nothing; IPv6 is bracketed. + ("h.io", _P, {}, ("h.io", _P)), + ("h.io", _P, {"_port": 8443}, ("h.io:8443", _P)), + ("h.io", _P, {"_port": "8443"}, ("h.io:8443", _P)), + ("h.io", _P, {"_port": 1}, ("h.io:1", _P)), + ("h.io", _P, {"_port": 65535}, ("h.io:65535", _P)), + ("h.io", _P, {"_port": 443}, ("h.io", _P)), + ("h.io", _P, {"_port": 0}, ("h.io", _P)), + ("::1", _P, {}, ("::1", _P)), + ("::1", _P, {"_port": 8443}, ("[::1]:8443", _P)), + ("[::1]", _P, {"_port": 8443}, ("[::1]:8443", _P)), + ("h.io:8443", _P, {}, ("h.io:8443", _P)), + ("h.io:8443", _P, {"_port": 8443}, ("h.io:8443", _P)), + # AC-P2: _connection_uri replaces host, port, path and query; it wins over _port. + ( + "h.io", + _P, + {"_connection_uri": "https://d.io:8443/sql/1.0/warehouses/xyz"}, + ("https://d.io:8443", "/sql/1.0/warehouses/xyz"), + ), + ( + "h.io", + _P, + {"_connection_uri": "https://h.io/sql/x?o=123"}, + ("https://h.io", "/sql/x?o=123"), + ), + ( + "h.io", + _P, + {"_connection_uri": "http://h.io:8080/sql/x"}, + ("http://h.io:8080", "/sql/x"), + ), + ( + "h.io", + _P, + {"_connection_uri": "https://d.io:9999/p", "_port": 8443}, + ("https://d.io:9999", "/p"), + ), + ( + "", + "", + {"_connection_uri": "https://h.io/sql/x"}, + ("https://h.io", "/sql/x"), + ), # URI-only + ], +) +def test_kernel_host_and_path_accepts(server, hpath, kwargs, expected): + assert _kernel_host_and_path(server, hpath, kwargs) == expected + + +@pytest.mark.parametrize( + "server, kwargs, message", + [ + # AC-P3: ports outside 1..65535, or not integers. + ("h.io", {"_port": -1}, "Invalid _port"), + ("h.io", {"_port": 65536}, "Invalid _port"), + ("h.io", {"_port": "abc"}, "Invalid _port"), + ("h.io", {"_port": True}, "Invalid _port"), + ("h.io", {"_port": 8443.5}, "Invalid _port"), + ("h.io:notaport", {}, "malformed"), + ("h.io:0", {}, "out of range"), + # AC-P3: a supplied port must agree with a port embedded in server_hostname. + ("h.io:7000", {"_port": 8443}, "Conflicting ports"), + ("h.io:8443", {"_port": 443}, "Conflicting ports"), + ("[::1]:7000", {"_port": 8443}, "Conflicting ports"), + # R11: _port needs a server_hostname unless _connection_uri carries the host. + ("", {"_port": 8443}, "server_hostname is required"), + ("", {"_port": 443}, "server_hostname is required"), + # AC-P3: _connection_uri must be http(s) with a host and a warehouse path. + ("h", {"_connection_uri": "d.io/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "ftp://h.io/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https://h.io:8443"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https://h.io:8443/"}, "Invalid _connection_uri"), + ( + "h", + {"_connection_uri": "https://h.io:notaport/sql/x"}, + "Invalid _connection_uri", + ), + ("h", {"_connection_uri": "https://h.io:0/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https://u:p@h.io/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https:///sql/x"}, "Invalid _connection_uri"), + ], +) +def test_kernel_host_and_path_rejects(server, kwargs, message): + with pytest.raises(ValueError, match=message): + _kernel_host_and_path(server, _P, kwargs) + + +@pytest.mark.parametrize( + "server, kwargs, uri, should_raise", + [ + # AC-P4: with OAuth, the _connection_uri origin must equal https://server_hostname. + ( + "h.io", + {"oauth_client_id": "c", "oauth_client_secret": "s"}, + "https://evil.io/sql/x", + True, + ), + ("h.io", {"auth_type": "azure-sp-m2m"}, "https://evil.io/sql/x", True), + ("h.io", {"oauth_jwt_key_file": "/k.pem"}, "https://evil.io/sql/x", True), + ("h.io", {"auth_type": "databricks-oauth"}, "http://h.io/sql/x", True), + ("h.io", {"auth_type": "databricks-oauth"}, "https://h.io:8443/sql/x", True), + ("h.io", {"auth_type": "databricks-oauth"}, "https://H.io/sql/x", False), + ( + "h.io:8443", + {"auth_type": "databricks-oauth"}, + "https://h.io:8443/sql/x", + False, + ), + ("[::1]", {"auth_type": "databricks-oauth"}, "https://[::1]/sql/x", False), + ( + "h.io", + {"access_token": "dapi"}, + "https://evil.io/sql/x", + False, + ), # PAT is not guarded + ( + "", + {"auth_type": "databricks-oauth"}, + "https://h.io/sql/x", + False, + ), # URI-only + ( + "h.io", + {"auth_type": "databricks-oauth", "_port": 8443}, + None, + False, + ), # documented, not guarded + ], +) +def test_oauth_origin_guard(server, kwargs, uri, should_raise): + kwargs = dict(kwargs) + if uri is not None: + kwargs["_connection_uri"] = uri + kernel_host = _kernel_host_and_path(server, _P, kwargs)[0] + if should_raise: + with pytest.raises(ValueError, match="requires the _connection_uri origin"): + _check_oauth_origin(server, kernel_host, kwargs) + else: + _check_oauth_origin(server, kernel_host, kwargs) + + +@pytest.fixture +def fake_kernel(): + """Fake the kernel wheel and client so connect() runs without a warehouse.""" + pytest.importorskip( + "pyarrow", reason="kernel client module imports pyarrow at load" + ) + fake = types.ModuleType("databricks_sql_kernel") + fake.KernelError = type("KernelError", (Exception,), {}) + fake.Session = MagicMock() + with patch.dict(sys.modules, {"databricks_sql_kernel": fake}), patch( + "databricks.sql.backend.kernel.client.KernelDatabricksClient" + ) as client, patch("databricks.sql.session.get_python_sql_connector_auth_provider"): + client.return_value.open_session.return_value = SessionId( + BackendType.SEA, "sess-id", None + ) + yield client, fake + + +def test_oauth_guard_refuses_before_kernel_client_is_built(fake_kernel): + client, fake = fake_kernel + with pytest.raises(ValueError, match="requires the _connection_uri origin"): + databricks.sql.connect( + server_hostname="h.io", + http_path=_P, + use_kernel=True, + enable_telemetry=False, + oauth_client_id="c", + oauth_client_secret="s", + _connection_uri="https://evil.io/sql/x", + ) + client.assert_not_called() + fake.Session.assert_not_called() + + +@pytest.mark.parametrize( + "hp, uri, expected_org", + [ + ("/sql/1.0/warehouses/abc?o=111", "https://d.io/sql/x?o=222", "222"), + ("/sql/1.0/warehouses/abc?o=111", "https://d.io/sql/x", None), + ("/sql/1.0/warehouses/abc", "https://d.io/sql/x?o=222", "222"), + ("/sql/1.0/warehouses/abc?o=111", None, "111"), + ], +) +def test_spog_org_id_follows_resolved_path(fake_kernel, hp, uri, expected_org): + client, _ = fake_kernel + extra = {"_connection_uri": uri} if uri else {} + databricks.sql.connect( + server_hostname="h.io", + http_path=hp, + access_token="dapi", + use_kernel=True, + enable_telemetry=False, + **extra, + ).close() + assert ( + dict(client.call_args.kwargs["http_headers"]).get("x-databricks-org-id") + == expected_org + ) + + +@pytest.mark.realkernel +@pytest.mark.parametrize( + "build", + [ + lambda p: {"server_hostname": "127.0.0.1", "http_path": _P, "_port": p}, + lambda p: { + "server_hostname": "", + "http_path": "", + "_connection_uri": f"https://127.0.0.1:{p}{_P}", + }, + ], + ids=["port", "uri-only"], +) +def test_kernel_dials_over_tls(build): + """Loopback dial on the real kernel wheel: a TCP listener on 127.0.0.1 must receive the + kernel's TLS handshake (first byte 0x16) at the requested port. Nothing completes the + handshake, so connect() is expected to fail; only the dial is asserted.""" + try: + ilm.version("databricks-sql-kernel") + except ilm.PackageNotFoundError: + pytest.skip("databricks-sql-kernel wheel not installed") + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(60) + port = listener.getsockname()[1] + received = {} + + def accept_one(): + try: + sock, _ = listener.accept() + sock.settimeout(10) + received["first_byte"] = sock.recv(1) + sock.close() + except OSError as exc: + received["error"] = repr(exc) + finally: + listener.close() + + thread = threading.Thread(target=accept_one, daemon=True) + thread.start() + try: + databricks.sql.connect( + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + _retry_stop_after_attempts_count=1, + **build(port), + ).close() + except Exception: + pass # expected: nothing behind the listener completes TLS + thread.join(timeout=70) + assert "error" not in received, received + assert received.get("first_byte") == b"\x16", received From d98ca45d249c6ab1c7225f7bfdfb48bd34baeaaf Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:58:26 +0000 Subject: [PATCH 02/12] fix(kernel): type _kernel_host_and_path http_path as str The helper returned Optional[str] for the resolved path, but KernelDatabricksClient takes a str, so mypy fails in check-types on every supported Python. Session.__init__ already types http_path as str, so this is an annotation-only change with no runtime effect. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index e7f140cab..62ae909c2 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -113,8 +113,8 @@ def _uri_host_and_path(uri: str) -> Tuple[str, str]: def _kernel_host_and_path( - server_hostname: str, http_path: Optional[str], kwargs: dict -) -> Tuple[str, Optional[str]]: + server_hostname: str, http_path: str, kwargs: dict +) -> Tuple[str, str]: """Resolve the kernel ``(host, http_path)``. ``_connection_uri`` replaces both and wins over ``_port``; ``_port`` folds into the host as ``host:port`` (default 443 adds nothing). """ From 5bb846e7f0a45de1d4114a08bb0c4c8bd27ca213 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:25:20 +0000 Subject: [PATCH 03/12] fix(kernel): match main for default host resolution and PAT OAuth guard A kernel connection without _port or _connection_uri again passes server_hostname through unchanged, so odd hosts no longer raise on the default path. The OAuth origin guard now skips connections the kernel auth bridge routes to PAT: an access_token with no JWT or client-secret M2M credentials, even when auth_type is databricks-oauth. azure-sp-m2m and M2M/JWT stay guarded. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 13 ++++++++++-- tests/unit/test_session.py | 39 +++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 62ae909c2..fd58de9b2 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -121,9 +121,9 @@ def _kernel_host_and_path( if kwargs.get("_connection_uri"): return _uri_host_and_path(kwargs["_connection_uri"]) port = _port_or_none(kwargs.get("_port")) - prefix, name, embedded = _split_host(server_hostname) if port is None: return server_hostname, http_path + prefix, name, embedded = _split_host(server_hostname) if not server_hostname: raise ValueError( "server_hostname is required on use_kernel=True when _port is set " @@ -153,7 +153,16 @@ def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> or (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) or kwargs.get("credentials_provider") is not None ) - if not (kwargs.get("_connection_uri") and server_hostname and oauth): + # The kernel auth bridge routes an access_token with no JWT or client-secret + # M2M credentials to PAT, even for auth_type='databricks-oauth'; azure-sp-m2m + # is resolved before that branch and stays OAuth. + routed_to_pat = ( + bool(kwargs.get("access_token")) + and kwargs.get("auth_type") != "azure-sp-m2m" + and not kwargs.get("oauth_jwt_key_file") + and not (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) + ) + if routed_to_pat or not (kwargs.get("_connection_uri") and server_hostname and oauth): return _, name, port = _split_host(server_hostname) uri = urlsplit(kernel_host) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index ae54f7f43..0e66d67c9 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1246,6 +1246,11 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): {"_connection_uri": "https://h.io/sql/x"}, ("https://h.io", "/sql/x"), ), # URI-only + # Without _port or _connection_uri the host passes through as on main, even if + # it is not a valid host:port (no port is parsed on the default path). + ("h.io:0", _P, {}, ("h.io:0", _P)), + ("h.io:notaport", _P, {}, ("h.io:notaport", _P)), + ("h.io:", _P, {}, ("h.io:", _P)), ], ) def test_kernel_host_and_path_accepts(server, hpath, kwargs, expected): @@ -1261,8 +1266,8 @@ def test_kernel_host_and_path_accepts(server, hpath, kwargs, expected): ("h.io", {"_port": "abc"}, "Invalid _port"), ("h.io", {"_port": True}, "Invalid _port"), ("h.io", {"_port": 8443.5}, "Invalid _port"), - ("h.io:notaport", {}, "malformed"), - ("h.io:0", {}, "out of range"), + ("h.io:notaport", {"_port": 8443}, "malformed"), + ("h.io:0", {"_port": 8443}, "out of range"), # AC-P3: a supplied port must agree with a port embedded in server_hostname. ("h.io:7000", {"_port": 8443}, "Conflicting ports"), ("h.io:8443", {"_port": 443}, "Conflicting ports"), @@ -1318,6 +1323,36 @@ def test_kernel_host_and_path_rejects(server, kwargs, message): "https://evil.io/sql/x", False, ), # PAT is not guarded + # The auth bridge routes access_token with no M2M/JWT credentials to PAT, + # even when auth_type is databricks-oauth; azure-sp-m2m and M2M/JWT stay guarded. + ( + "h.io", + {"auth_type": "databricks-oauth", "access_token": "dapi"}, + "https://evil.io/sql/x", + False, + ), + ( + "h.io", + {"auth_type": "azure-sp-m2m", "access_token": "dapi"}, + "https://evil.io/sql/x", + True, + ), + ( + "h.io", + {"oauth_jwt_key_file": "/k.pem", "access_token": "dapi"}, + "https://evil.io/sql/x", + True, + ), + ( + "h.io", + { + "oauth_client_id": "c", + "oauth_client_secret": "s", + "access_token": "dapi", + }, + "https://evil.io/sql/x", + True, + ), ( "", {"auth_type": "databricks-oauth"}, From 6a78cef7d57a669e0ddc00fdd2051eea5654aa61 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:46:48 +0000 Subject: [PATCH 04/12] fix(kernel): black formatting and OAuth guard follows the auth bridge Wraps the origin-guard condition to the repo's black line length (check-linting). The guard no longer treats a custom credentials_provider as OAuth: the kernel auth bridge rejects that provider with NotSupportedError, so no OAuth origin is involved. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 8 +++++--- tests/unit/test_session.py | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index fd58de9b2..72adc48f2 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -151,18 +151,20 @@ def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> kwargs.get("auth_type") in _OAUTH_AUTH_TYPES or kwargs.get("oauth_jwt_key_file") or (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) - or kwargs.get("credentials_provider") is not None ) # The kernel auth bridge routes an access_token with no JWT or client-secret # M2M credentials to PAT, even for auth_type='databricks-oauth'; azure-sp-m2m - # is resolved before that branch and stays OAuth. + # is resolved before that branch and stays OAuth. A custom credentials_provider + # is rejected by the bridge, so it is not an OAuth origin here. routed_to_pat = ( bool(kwargs.get("access_token")) and kwargs.get("auth_type") != "azure-sp-m2m" and not kwargs.get("oauth_jwt_key_file") and not (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) ) - if routed_to_pat or not (kwargs.get("_connection_uri") and server_hostname and oauth): + if routed_to_pat or not ( + kwargs.get("_connection_uri") and server_hostname and oauth + ): return _, name, port = _split_host(server_hostname) uri = urlsplit(kernel_host) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 0e66d67c9..77f3234a8 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1337,6 +1337,14 @@ def test_kernel_host_and_path_rejects(server, kwargs, message): "https://evil.io/sql/x", True, ), + # The kernel auth bridge rejects a custom credentials_provider with NotSupportedError, + # so the origin guard does not classify it as OAuth. + ( + "h.io", + {"credentials_provider": object()}, + "https://evil.io/sql/x", + False, + ), ( "h.io", {"oauth_jwt_key_file": "/k.pem", "access_token": "dapi"}, From 8f96c0b659b78fec90586559d06f95b884cdbc80 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:03:52 +0000 Subject: [PATCH 05/12] fix(kernel): keep credentials and resolved host details out of error messages Error messages that echoed a _connection_uri, server_hostname or the OAuth origin now show only scheme and host[:port] through a redaction helper, so userinfo, path, query and fragment never reach exception text or logs. Adds regression tests that the secret does not appear in the message, and a mocked connect() test that the resolved kernel host and path reach KernelDatabricksClient. Removes a duplicate import in the unit test module. No validation semantics change. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 25 +++++++++++---- tests/unit/test_session.py | 57 +++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 72adc48f2..654f19cac 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -68,6 +68,19 @@ def _port_or_none(value: Any) -> Optional[int]: return int(text) or None +def _redact(value: str) -> str: + """Keep only scheme and host[:port] of a URI or host string for an error message. + + Userinfo, path, query and fragment can carry credentials, so they never reach an + exception text or log line. + """ + scheme, sep, rest = value.partition("://") + if not sep: + scheme, rest = "", value + host = rest.split("/", 1)[0].rpartition("@")[2] + return f"{scheme}{sep}{re.split(r'[?#]', host, maxsplit=1)[0]}" + + def _split_host(host: str) -> Tuple[str, str, Optional[int]]: """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``; ``name`` keeps IPv6 brackets.""" prefix, sep, text = host.rstrip("/").rpartition("://") @@ -76,7 +89,7 @@ def _split_host(host: str) -> Tuple[str, str, Optional[int]]: if match is None: if text.startswith("[") or text.count(":") == 1: raise ValueError( - f"Invalid server_hostname {host!r}: malformed IPv6 literal or port" + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port" ) return prefix, text, None # an unbracketed IPv6 literal such as ::1 has no port name, port_text = match.group(1), match.group(2) @@ -84,7 +97,7 @@ def _split_host(host: str) -> Tuple[str, str, Optional[int]]: return prefix, name, None if not 1 <= int(port_text) <= 65535: raise ValueError( - f"Invalid server_hostname {host!r}: port {port_text} is out of range" + f"Invalid server_hostname {_redact(host)!r}: port {port_text} is out of range" ) return prefix, name, int(port_text) @@ -104,7 +117,7 @@ def _uri_host_and_path(uri: str) -> Tuple[str, str]: or parts.path in ("", "/") ): raise ValueError( - f"Invalid _connection_uri {uri!r}: expected http(s)://host[:port]/path with a " + f"Invalid _connection_uri {_redact(uri)!r}: expected http(s)://host[:port]/path with a " "warehouse path and no credentials" ) return f"{parts.scheme}://{parts.netloc}", parts.path + ( @@ -132,7 +145,7 @@ def _kernel_host_and_path( if embedded is not None: if embedded != port: raise ValueError( - f"Conflicting ports: server_hostname {server_hostname!r} carries {embedded} but _port={port}" + f"Conflicting ports: server_hostname {_redact(server_hostname)!r} carries {embedded} but _port={port}" ) return server_hostname, http_path if port == 443: @@ -174,8 +187,8 @@ def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> port or 443, ): raise ValueError( - f"OAuth requires the _connection_uri origin to match https://{server_hostname}; " - f"the kernel would send the credentials to {kernel_host}. Use a PAT or match the origin." + f"OAuth requires the _connection_uri origin to match https://{_redact(server_hostname)}; " + f"the kernel would send the credentials to {_redact(kernel_host)}. Use a PAT or match the origin." ) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 77f3234a8..a210060fe 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -25,8 +25,6 @@ ) from databricks.sql.session import Session -import databricks.sql - class TestSession: """ @@ -1387,6 +1385,61 @@ def test_oauth_origin_guard(server, kwargs, uri, should_raise): _check_oauth_origin(server, kernel_host, kwargs) +@pytest.mark.parametrize( + "uri, secret", + [ + ("https://user:secret@h.io/sql/x", "secret"), + ("https://h.io:8443?token=secret", "secret"), + ], +) +def test_invalid_connection_uri_message_omits_credentials(uri, secret): + with pytest.raises(ValueError, match="Invalid _connection_uri") as exc: + _kernel_host_and_path("h", _P, {"_connection_uri": uri}) + assert secret not in str(exc.value) + + +def test_oauth_origin_message_omits_uri_query(): + kwargs = { + "auth_type": "databricks-oauth", + "_connection_uri": "https://evil.io/sql/x?token=secret", + } + kernel_host = _kernel_host_and_path("h.io", _P, kwargs)[0] + with pytest.raises(ValueError, match="requires the _connection_uri origin") as exc: + _check_oauth_origin("h.io", kernel_host, kwargs) + assert "secret" not in str(exc.value) + assert "https://evil.io" in str(exc.value) + + +@pytest.mark.parametrize( + "conn_kwargs, expected_host, expected_path", + [ + ({"server_hostname": "h.io", "http_path": _P, "_port": 8443}, "h.io:8443", _P), + ( + { + "server_hostname": "", + "http_path": "", + "_connection_uri": f"https://d.io:9999{_P}", + }, + "https://d.io:9999", + _P, + ), + ], +) +def test_resolved_kernel_host_and_path_reach_kernel_client( + fake_kernel, conn_kwargs, expected_host, expected_path +): + client, _ = fake_kernel + databricks.sql.connect( + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + **conn_kwargs, + ) + kwargs = client.call_args.kwargs + assert kwargs["server_hostname"] == expected_host + assert kwargs["http_path"] == expected_path + + @pytest.fixture def fake_kernel(): """Fake the kernel wheel and client so connect() runs without a warehouse.""" From 6250c286fba73c1e8e8d65eab521a5fc18b717c9 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:21:45 +0000 Subject: [PATCH 06/12] fix(kernel): sanitize connection-URI parse errors and wrap long messages urlsplit ran outside the error handling, so a malformed URI whose NFKC-normalized netloc failed raised a ValueError that echoed the raw netloc, password included. The parse is now wrapped and re-raised as a redacted ValueError with from None; a regression row covers it. Long error messages and docstrings are wrapped to the 100-column limit in CONTRIBUTING.md. No validation semantics change. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 21 +++++++++++++++------ tests/unit/test_session.py | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 654f19cac..7cea9140c 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -82,7 +82,7 @@ def _redact(value: str) -> str: def _split_host(host: str) -> Tuple[str, str, Optional[int]]: - """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``; ``name`` keeps IPv6 brackets.""" + """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``; keeps IPv6 brackets.""" prefix, sep, text = host.rstrip("/").rpartition("://") prefix += sep match = _HOSTPORT_RE.fullmatch(text) @@ -103,8 +103,14 @@ def _split_host(host: str) -> Tuple[str, str, Optional[int]]: def _uri_host_and_path(uri: str) -> Tuple[str, str]: - """Split a ``_connection_uri`` into the kernel host (scheme and authority) and path (with query).""" - parts = urlsplit(uri) + """Split a ``_connection_uri`` into the kernel host (scheme, authority) and path.""" + try: + parts = urlsplit(uri) + except ValueError: + # urlsplit echoes the raw netloc, which can carry credentials; re-raise without it. + raise ValueError( + f"Invalid _connection_uri {_redact(uri)!r}: could not be parsed" + ) from None try: port = parts.port except ValueError: @@ -145,7 +151,8 @@ def _kernel_host_and_path( if embedded is not None: if embedded != port: raise ValueError( - f"Conflicting ports: server_hostname {_redact(server_hostname)!r} carries {embedded} but _port={port}" + f"Conflicting ports: server_hostname {_redact(server_hostname)!r} " + f"carries {embedded} but _port={port}" ) return server_hostname, http_path if port == 443: @@ -187,8 +194,10 @@ def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> port or 443, ): raise ValueError( - f"OAuth requires the _connection_uri origin to match https://{_redact(server_hostname)}; " - f"the kernel would send the credentials to {_redact(kernel_host)}. Use a PAT or match the origin." + f"OAuth requires the _connection_uri origin to match " + f"https://{_redact(server_hostname)}; " + f"the kernel would send the credentials to {_redact(kernel_host)}. " + "Use a PAT or match the origin." ) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index a210060fe..990c20a2b 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1390,6 +1390,8 @@ def test_oauth_origin_guard(server, kwargs, uri, should_raise): [ ("https://user:secret@h.io/sql/x", "secret"), ("https://h.io:8443?token=secret", "secret"), + # A fullwidth solidus makes urlsplit's NFKC check raise with the raw netloc in its text. + ("https://user:secret@h.io/sql/x", "secret"), ], ) def test_invalid_connection_uri_message_omits_credentials(uri, secret): From f27c1655d525590fbb08613d13dbb04a5d7bc425 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:43:38 +0000 Subject: [PATCH 07/12] fix(kernel): drop the whole authority from errors when credentials may be present _redact split at the first slash before stripping userinfo, so a password containing a slash left part of the secret in the message. Whenever the raw value contains an @ or a percent-encoded %40, only the scheme is kept. The parse failure is now raised outside the except block, so no chained exception can carry urlsplit's netloc. Regression rows cover an @ after the host, a percent-encoded %40, and passwords containing a slash or a question mark. No validation semantics change. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 12 +++++++++--- tests/unit/test_session.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 7cea9140c..38de34b08 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -77,7 +77,11 @@ def _redact(value: str) -> str: scheme, sep, rest = value.partition("://") if not sep: scheme, rest = "", value - host = rest.split("/", 1)[0].rpartition("@")[2] + # A "/" or "?" can sit inside a password, and "@" can sit in a path, so once any + # userinfo marker is present the whole authority is dropped rather than guessed at. + if "@" in value or "%40" in value.lower(): + return f"{scheme}{sep}" + host = rest.split("/", 1)[0] return f"{scheme}{sep}{re.split(r'[?#]', host, maxsplit=1)[0]}" @@ -107,10 +111,12 @@ def _uri_host_and_path(uri: str) -> Tuple[str, str]: try: parts = urlsplit(uri) except ValueError: - # urlsplit echoes the raw netloc, which can carry credentials; re-raise without it. + parts = None + if parts is None: + # Raised outside the except block so no chained exception carries urlsplit's netloc. raise ValueError( f"Invalid _connection_uri {_redact(uri)!r}: could not be parsed" - ) from None + ) try: port = parts.port except ValueError: diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 990c20a2b..a89e39aa7 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1390,6 +1390,15 @@ def test_oauth_origin_guard(server, kwargs, uri, should_raise): [ ("https://user:secret@h.io/sql/x", "secret"), ("https://h.io:8443?token=secret", "secret"), + # "@" after the host with no path: the URI is rejected, and the "@" must not leak. + ("https://h.io?x=a@secret", "secret"), + # Percent-encoded "@" hides the userinfo marker from a literal "@" check. + ("https://user%3Asecret%40h.io", "secret"), + # A "/" or "?" inside the password must not cut the authority short. + ("https://user:sec/ret@h.io/sql/x", "sec/ret"), + ("https://user:sec?ret@h.io/sql/x", "sec?ret"), + # A "/" inside the password, with "@" after it in the path: netloc stops at "/". + ("https://user:secret/foo@h.io/sql/x", "secret"), # A fullwidth solidus makes urlsplit's NFKC check raise with the raw netloc in its text. ("https://user:secret@h.io/sql/x", "secret"), ], @@ -1398,6 +1407,9 @@ def test_invalid_connection_uri_message_omits_credentials(uri, secret): with pytest.raises(ValueError, match="Invalid _connection_uri") as exc: _kernel_host_and_path("h", _P, {"_connection_uri": uri}) assert secret not in str(exc.value) + # The secret must not survive in a chained cause or context either. + assert exc.value.__cause__ is None + assert exc.value.__context__ is None or secret not in str(exc.value.__context__) def test_oauth_origin_message_omits_uri_query(): From c90459f1fef44a9243fa016aaa41db95a775b962 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:11:53 +0000 Subject: [PATCH 08/12] Embedded host port wins over a differing _port on the kernel path A port embedded in server_hostname now wins over a differing _port, with a warning, as on main, where the kernel connected to the embedded port. A falsy _port (None, 0, '', False) means the default, as on Thrift (port or 443). A trailing '/' is trimmed before port handling. A truthy invalid _port still raises, as Thrift fails on it (LocationParseError). Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CHANGELOG.md | 2 +- src/databricks/sql/client.py | 4 +- src/databricks/sql/session.py | 27 +++++++-- tests/unit/test_session.py | 103 ++++++++++++++++++++++++++++++++-- 4 files changed, 124 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e891ec4f0..5577c7836 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Release History # Unreleased -- Honor `_port` and `_connection_uri` on kernel connections (`use_kernel=True`), as Thrift does. `_port` sends the connection to that port (default 443 adds nothing); `_connection_uri` (`http(s)://host[:port]/path?query`) sets host, port, path and query and wins over `_port`; URI-only connections work. Invalid ports, conflicting ports, an empty `server_hostname` with `_port`, and URIs without a scheme or path raise `ValueError`. With OAuth, a `_connection_uri` origin that differs from `server_hostname` raises. With OAuth and a non-default `_port`, the token exchange goes to the same host:port, unlike Thrift (documented, not guarded). +- Honor `_port` and `_connection_uri` on kernel connections (`use_kernel=True`), as Thrift does. `_port` sends the connection to that port (default 443 adds nothing); `_connection_uri` (`http(s)://host[:port]/path?query`) sets host, port, path and query and wins over `_port`; URI-only connections work. A port embedded in `server_hostname` wins over a differing `_port`, with a warning, as on main. Invalid ports, an empty `server_hostname` with `_port`, and URIs without a scheme or path raise `ValueError`. With OAuth, a `_connection_uri` origin that differs from `server_hostname` raises. With OAuth and a non-default `_port`, the token exchange goes to the same host:port, unlike Thrift (documented, not guarded). - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index c925f466b..bf6a63840 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -195,7 +195,9 @@ def __init__( be suppressed by the feature flag. Kernel overrides: _port sends the connection to that port - (the default 443 adds nothing). _connection_uri (https://host[:port]/path?query) + (the default 443 adds nothing; a falsy _port means the default). A port + embedded in server_hostname wins over a differing _port, with a warning. + _connection_uri (https://host[:port]/path?query) sets host, port, path and query, as on Thrift, and wins over _port. With OAuth, _connection_uri must have the same origin as server_hostname. With OAuth and a non-default _port, the token diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 38de34b08..c9eba9cfd 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -53,8 +53,10 @@ def __getattr__(name: str) -> Any: def _port_or_none(value: Any) -> Optional[int]: - """Validate ``_port``. ``None`` and ``0`` mean the default (Thrift: ``port or 443``).""" - if value is None: + """Validate ``_port``. A falsy value (None, 0, '', False) means the default, as on + Thrift (``port or 443``); a truthy value must be an integer between 1 and 65535. + """ + if not value or (isinstance(value, str) and not value.strip()): return None text = ( str(value).strip() @@ -148,21 +150,34 @@ def _kernel_host_and_path( port = _port_or_none(kwargs.get("_port")) if port is None: return server_hostname, http_path - prefix, name, embedded = _split_host(server_hostname) if not server_hostname: raise ValueError( "server_hostname is required on use_kernel=True when _port is set " "(or pass _connection_uri, which carries its own host)" ) + prefix, name, embedded = _split_host(server_hostname) if embedded is not None: + # A port embedded in server_hostname always wins, as on main (the kernel + # connected to it); a differing _port is ignored with a warning, not an error. if embedded != port: - raise ValueError( - f"Conflicting ports: server_hostname {_redact(server_hostname)!r} " - f"carries {embedded} but _port={port}" + logger.warning( + "server_hostname %s carries port %d, which differs from _port=%d; " + "connecting to port %d. Set the port in one place only.", + _redact(server_hostname), + embedded, + port, + embedded, ) return server_hostname, http_path if port == 443: return server_hostname, http_path + if any(c in name for c in "/?#"): + # Folding _port in would put it after the path, query or fragment, so the + # port could not be applied; reject rather than silently drop it. + raise ValueError( + f"Invalid server_hostname {_redact(server_hostname)!r}: a path, query or " + "fragment cannot be combined with _port; put the port in server_hostname" + ) if ":" in name and not name.startswith("["): name = f"[{name}]" return f"{prefix}{name}:{port}", http_path diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index a89e39aa7..5c9845a13 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -3,6 +3,7 @@ from contextlib import contextmanager import gc import importlib.metadata as ilm +import logging import socket import sys import threading @@ -1208,6 +1209,16 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): ("h.io", _P, {"_port": 65535}, ("h.io:65535", _P)), ("h.io", _P, {"_port": 443}, ("h.io", _P)), ("h.io", _P, {"_port": 0}, ("h.io", _P)), + # Falsy _port means the default, as on Thrift (port or 443); no :443 is added. + ("h.io", _P, {"_port": None}, ("h.io", _P)), + ("h.io", _P, {"_port": ""}, ("h.io", _P)), + ("h.io", _P, {"_port": " "}, ("h.io", _P)), + ("h.io", _P, {"_port": False}, ("h.io", _P)), + ("h.io", _P, {"_port": "0"}, ("h.io", _P)), + # Trailing "/" is trimmed before the port is folded in. + ("h.io/", _P, {"_port": 8443}, ("h.io:8443", _P)), + ("h.io:8443/", _P, {"_port": 8443}, ("h.io:8443/", _P)), + ("h.io/", _P, {}, ("h.io/", _P)), ("::1", _P, {}, ("::1", _P)), ("::1", _P, {"_port": 8443}, ("[::1]:8443", _P)), ("[::1]", _P, {"_port": 8443}, ("[::1]:8443", _P)), @@ -1266,10 +1277,10 @@ def test_kernel_host_and_path_accepts(server, hpath, kwargs, expected): ("h.io", {"_port": 8443.5}, "Invalid _port"), ("h.io:notaport", {"_port": 8443}, "malformed"), ("h.io:0", {"_port": 8443}, "out of range"), - # AC-P3: a supplied port must agree with a port embedded in server_hostname. - ("h.io:7000", {"_port": 8443}, "Conflicting ports"), - ("h.io:8443", {"_port": 443}, "Conflicting ports"), - ("[::1]:7000", {"_port": 8443}, "Conflicting ports"), + # An explicit _port cannot be folded in after a path, query or fragment. + ("h.io/x", {"_port": 8443}, "path, query or fragment"), + ("h.io?x=1", {"_port": 8443}, "path, query or fragment"), + ("h.io#x", {"_port": 8443}, "path, query or fragment"), # R11: _port needs a server_hostname unless _connection_uri carries the host. ("", {"_port": 8443}, "server_hostname is required"), ("", {"_port": 443}, "server_hostname is required"), @@ -1293,6 +1304,33 @@ def test_kernel_host_and_path_rejects(server, kwargs, message): _kernel_host_and_path(server, _P, kwargs) +@pytest.mark.parametrize( + "server, port, embedded", + [ + ("h.io:7000", 8443, 7000), + ("h.io:8443", 443, 8443), + ("[::1]:7000", 8443, 7000), + ("h.io:443", 8443, 443), + ], +) +def test_embedded_port_wins_over_differing_explicit_port( + server, port, embedded, caplog +): + # A port embedded in server_hostname always wins (as on main); a differing _port + # is not an error. The warning names both ports, and the host is passed unchanged. + with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): + host, path = _kernel_host_and_path(server, _P, {"_port": port}) + assert (host, path) == (server, _P) + assert f"carries port {embedded}" in caplog.text + assert f"_port={port}" in caplog.text + + +def test_embedded_port_equal_to_explicit_port_does_not_warn(caplog): + with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): + _kernel_host_and_path("h.io:8443", _P, {"_port": 8443}) + assert "differs from _port" not in caplog.text + + @pytest.mark.parametrize( "server, kwargs, uri, should_raise", [ @@ -1568,3 +1606,60 @@ def accept_one(): thread.join(timeout=70) assert "error" not in received, received assert received.get("first_byte") == b"\x16", received + + +@pytest.mark.realkernel +def test_kernel_embedded_port_wins_over_explicit_port(): + """Loopback on the real kernel wheel: server_hostname 'h:P1' with an explicit _port P2 + dials P1 (TLS first byte 0x16) and never P2.""" + try: + ilm.version("databricks-sql-kernel") + except ilm.PackageNotFoundError: + pytest.skip("databricks-sql-kernel wheel not installed") + + def listen(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + sock.settimeout(15) + return sock + + embedded, explicit = listen(), listen() + embedded_port = embedded.getsockname()[1] + explicit_port = explicit.getsockname()[1] + received = {} + + def accept_one(name, sock): + try: + conn, _ = sock.accept() + conn.settimeout(10) + received[name] = conn.recv(1) + conn.close() + except OSError as exc: + received[name + "_error"] = repr(exc) + finally: + sock.close() + + threads = [ + threading.Thread(target=accept_one, args=("embedded", embedded), daemon=True), + threading.Thread(target=accept_one, args=("explicit", explicit), daemon=True), + ] + for thread in threads: + thread.start() + try: + databricks.sql.connect( + server_hostname=f"127.0.0.1:{embedded_port}", + http_path=_P, + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + _retry_stop_after_attempts_count=1, + _port=explicit_port, + ).close() + except Exception: + pass # expected: nothing behind the listener completes TLS + for thread in threads: + thread.join(timeout=70) + assert received.get("embedded") == b"\x16", received + assert "explicit" not in received, received + assert "explicit_error" in received, received From bc7c09ebb170660f96e383b9b6034609a299c688 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:21:15 +0000 Subject: [PATCH 09/12] Reject a path, query or fragment host combined with _port; document kernel port rules Check the host for a path, query or fragment before the port is parsed, so 'h.io/path:8443' with _port cannot read as port 8443. Only explicit _port is affected; a host without _port passes through unchanged, as on main. Add Thrift tests pinning that host and _port reach ThriftDatabricksClient unchanged. Update CONNECTION_PARAMETERS.md: _port and _connection_uri are now kernel options, with the embedded-port and falsy rules and the OAuth origin rule. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- CONNECTION_PARAMETERS.md | 4 ++-- src/databricks/sql/session.py | 15 +++++++------- tests/unit/test_session.py | 38 ++++++++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index f0b3d2ee7..637577b4c 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -73,8 +73,8 @@ to change without notice. | ----------------- | ----- | :----: | :----: | ------------- | ----------------------------------------------------------------------------------------------------------------------- | | `server_hostname` | `str` | ✅ | ✅ | — (required) | Workspace hostname, e.g. `dbc-12345.cloud.databricks.com`. | | `http_path` | `str` | ✅ | ✅ | — (required) | Thrift accepts a SQL-warehouse path **or** an all-purpose-cluster path; Kernel requires a warehouse/endpoint path. | -| `_port` | `int` | ✅ | — | `443` | TCP port (advanced). Not threaded to the kernel; it derives host/port from `server_hostname` + `http_path`. | -| `_connection_uri` | `str` | ✅ | — | `None` | Thrift-only internal override of `server_hostname`/`http_path`. No kernel equivalent. | +| `_port` | `int` | ✅ | ✅ | `443` | TCP port (advanced). On Kernel a falsy value means the default, and a port embedded in `server_hostname` wins over a differing `_port` (with a warning). | +| `_connection_uri` | `str` | ✅ | ✅ | `None` | Overrides `server_hostname`, `_port` and `http_path` on both, and wins over `_port`. On Kernel with OAuth, the URI origin must match `server_hostname`, and OAuth uses the same host:port as the connection (Thrift keeps OAuth on the bare host). | | `http_headers` | `List[Tuple[str, str]]` | ✅ | ⚠️ | `None` | Extra headers sent on requests. Kernel filters `Authorization` and `x-databricks-org-id` because it manages them itself; other headers are forwarded. | | `user_agent_entry`| `str` | ✅ | ✅ | `None` | Custom tag folded into the composed `User-Agent` on both. (`_user_agent_entry` is a deprecated alias that warns.) | diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index c9eba9cfd..f83e5808b 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -155,6 +155,14 @@ def _kernel_host_and_path( "server_hostname is required on use_kernel=True when _port is set " "(or pass _connection_uri, which carries its own host)" ) + # Checked before the port is parsed: 'h.io/path:8443' would otherwise read as host + # 'h.io/path' with port 8443. Only the text after any scheme is checked. + authority = server_hostname.rstrip("/").rpartition("://")[2] + if any(c in authority for c in "/?#"): + raise ValueError( + f"Invalid server_hostname {_redact(server_hostname)!r}: a path, query or " + "fragment cannot be combined with _port; put the port in server_hostname" + ) prefix, name, embedded = _split_host(server_hostname) if embedded is not None: # A port embedded in server_hostname always wins, as on main (the kernel @@ -171,13 +179,6 @@ def _kernel_host_and_path( return server_hostname, http_path if port == 443: return server_hostname, http_path - if any(c in name for c in "/?#"): - # Folding _port in would put it after the path, query or fragment, so the - # port could not be applied; reject rather than silently drop it. - raise ValueError( - f"Invalid server_hostname {_redact(server_hostname)!r}: a path, query or " - "fragment cannot be combined with _port; put the port in server_hostname" - ) if ":" in name and not name.startswith("["): name = f"[{name}]" return f"{prefix}{name}:{port}", http_path diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 5c9845a13..517fbe834 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1219,6 +1219,10 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): ("h.io/", _P, {"_port": 8443}, ("h.io:8443", _P)), ("h.io:8443/", _P, {"_port": 8443}, ("h.io:8443/", _P)), ("h.io/", _P, {}, ("h.io/", _P)), + # Without _port, a host with a path passes through unchanged, as on main. + ("h.io/x", _P, {}, ("h.io/x", _P)), + ("https://h.io:8443/", _P, {"_port": 8443}, ("https://h.io:8443/", _P)), + ("https://h.io", _P, {"_port": 8443}, ("https://h.io:8443", _P)), ("::1", _P, {}, ("::1", _P)), ("::1", _P, {"_port": 8443}, ("[::1]:8443", _P)), ("[::1]", _P, {"_port": 8443}, ("[::1]:8443", _P)), @@ -1277,10 +1281,15 @@ def test_kernel_host_and_path_accepts(server, hpath, kwargs, expected): ("h.io", {"_port": 8443.5}, "Invalid _port"), ("h.io:notaport", {"_port": 8443}, "malformed"), ("h.io:0", {"_port": 8443}, "out of range"), - # An explicit _port cannot be folded in after a path, query or fragment. + # An explicit _port with a path, query or fragment is rejected, even when the + # path looks like a port ('h.io/path:8443') or the port is the default (443). ("h.io/x", {"_port": 8443}, "path, query or fragment"), + ("h.io/path:8443", {"_port": 8443}, "path, query or fragment"), + ("h.io/x", {"_port": 443}, "path, query or fragment"), ("h.io?x=1", {"_port": 8443}, "path, query or fragment"), ("h.io#x", {"_port": 8443}, "path, query or fragment"), + ("https://h.io/x", {"_port": 8443}, "path, query or fragment"), + ("h.io/x/", {"_port": 8443}, "path, query or fragment"), # R11: _port needs a server_hostname unless _connection_uri carries the host. ("", {"_port": 8443}, "server_hostname is required"), ("", {"_port": 443}, "server_hostname is required"), @@ -1331,6 +1340,33 @@ def test_embedded_port_equal_to_explicit_port_does_not_warn(caplog): assert "differs from _port" not in caplog.text +@pytest.mark.parametrize( + "server, port", + [ + ("h.io:8443", 443), + ("h.io", 8443), + ("h.io/x", 8443), + ("h.io", ""), + ], +) +@patch("databricks.sql.session.ThriftDatabricksClient") +def test_thrift_receives_host_and_port_unchanged(mock_client_class, server, port): + # Thrift is unchanged: the host and _port reach ThriftDatabricksClient verbatim, and + # ThriftDatabricksClient applies its own `port or 443` to the URL. The kernel-only + # folding above does not apply here. + connection = databricks.sql.connect( + server_hostname=server, + http_path=_P, + access_token="tok", + enable_telemetry=False, + _port=port, + ) + call_kwargs = mock_client_class.call_args.kwargs + assert call_kwargs["server_hostname"] == server + assert call_kwargs["port"] == port + connection.close() + + @pytest.mark.parametrize( "server, kwargs, uri, should_raise", [ From bdeef64faa14f5cca506f189e9ccf50ecc219d2e Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:44:20 +0000 Subject: [PATCH 10/12] kernel: parse host:port with urllib and check the JWT issuer origin Replace the _HOSTPORT_RE regex, the _port regex and the manual bracketing in the kernel host folding with urllib.parse.urlsplit, int() and an ipaddress pre-check for bare IPv6 literals. The original host string still passes through unchanged unless a port is folded in; folded IPv6 hosts are bracketed. _check_oauth_origin treats an access_token that decodes as a JWT whose issuer is not server_hostname as OAuth. The kernel wraps every provider with the connection host, so such a token would be exchanged at the _connection_uri origin, while Thrift exchanges it at server_hostname. A mismatched origin now raises before any request. PATs and same-host issuers are unaffected. The port-conflict warning is skipped when _port is the default 443, matching Node. Non-default ports still warn on a mismatch. Behavior-table tests in test_session.py pass unchanged. One warning test changed: test_embedded_port_wins_over_differing_explicit_port lost its ('h.io:8443', 443) row, which asserted the 443 warning. That case moved to test_embedded_port_with_default_443_does_not_warn under the new rule. Known differences from bc7c09e: the OAuth origin check now compares the parsed hostname, so a server_hostname such as 'h.io/x' or 'user@h' with a matching _connection_uri origin passes where it was previously refused. The error text for 'h:70000', 'h:abc' and '[::1' is now one generic malformed-or-out-of-range message. 'user:pass@h' with _port is still refused. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 103 +++++++++++++++++++++++++--------- tests/unit/test_session.py | 96 ++++++++++++++++++++++++++++++- 2 files changed, 171 insertions(+), 28 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index f83e5808b..cf61fbac4 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -1,3 +1,4 @@ +import ipaddress import logging import re from functools import cached_property @@ -6,8 +7,10 @@ from databricks.sql.types import SSLOptions from databricks.sql.auth.auth import get_python_sql_connector_auth_provider +from databricks.sql.auth.auth_utils import decode_token, is_same_host from databricks.sql.auth.authenticators import AccessTokenAuthProvider from databricks.sql.auth.common import ClientContext +from databricks.sql.common.url_utils import normalize_host_with_protocol from databricks.sql.exc import SessionAlreadyClosedError, DatabaseError, RequestError from databricks.sql import __version__ from databricks.sql import USER_AGENT_NAME @@ -49,7 +52,6 @@ def __getattr__(name: str) -> Any: _OAUTH_AUTH_TYPES = ("databricks-oauth", "azure-oauth", "azure-sp-m2m") -_HOSTPORT_RE = re.compile(r"(\[[^\]]*\]|(?!\[)[^:]*)(?::([0-9]+))?") def _port_or_none(value: Any) -> Optional[int]: @@ -63,7 +65,8 @@ def _port_or_none(value: Any) -> Optional[int]: if isinstance(value, (int, str)) and not isinstance(value, bool) else "" ) - if not re.fullmatch(r"[0-9]+", text) or not int(text) <= 65535: + # isdigit with isascii rejects signs, whitespace and underscores that int() would accept. + if not (text.isascii() and text.isdigit()) or int(text) > 65535: raise ValueError( f"Invalid _port {value!r}: expected an integer between 1 and 65535" ) @@ -83,29 +86,57 @@ def _redact(value: str) -> str: # userinfo marker is present the whole authority is dropped rather than guessed at. if "@" in value or "%40" in value.lower(): return f"{scheme}{sep}" - host = rest.split("/", 1)[0] - return f"{scheme}{sep}{re.split(r'[?#]', host, maxsplit=1)[0]}" + try: + host = urlsplit(f"//{rest}").netloc + except ValueError: + # urlsplit refuses some hosts (bad IPv6, NFKC-unsafe text); show none of them. + return f"{scheme}{sep}" + return f"{scheme}{sep}{host}" + + +def _is_bare_ipv6(text: str) -> bool: + """True for an unbracketed IPv6 literal such as ``::1``, which urlsplit reads as a port.""" + try: + return ipaddress.ip_address(text).version == 6 + except ValueError: + return False def _split_host(host: str) -> Tuple[str, str, Optional[int]]: - """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``; keeps IPv6 brackets.""" - prefix, sep, text = host.rstrip("/").rpartition("://") + """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``. ``name`` keeps the + input's case and any userinfo, and a bare IPv6 literal gets its brackets. + """ + prefix, sep, rest = host.rstrip("/").rpartition("://") prefix += sep - match = _HOSTPORT_RE.fullmatch(text) - if match is None: - if text.startswith("[") or text.count(":") == 1: - raise ValueError( - f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port" - ) - return prefix, text, None # an unbracketed IPv6 literal such as ::1 has no port - name, port_text = match.group(1), match.group(2) - if port_text is None: - return prefix, name, None - if not 1 <= int(port_text) <= 65535: + if _is_bare_ipv6(rest): + return prefix, f"[{rest}]", None + # urlsplit reads a trailing ':' as an empty port; with an explicit _port that is an error. + if rest.endswith(":"): + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port" + ) + try: + parts = urlsplit(f"//{rest}") + port = parts.port + except ValueError: + # ``from None`` drops the urlsplit exception, whose text can quote the raw netloc. + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port; " + "a port must be 1-65535" + ) from None + if parts.password is not None: + # A colon before '@' is userinfo, not a port, but a credential-bearing host is + # still refused when a port is requested (as on main), so it is never folded. + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port" + ) + if port is None: + return prefix, parts.netloc, None + if not 1 <= port <= 65535: raise ValueError( - f"Invalid server_hostname {_redact(host)!r}: port {port_text} is out of range" + f"Invalid server_hostname {_redact(host)!r}: port {port} is out of range" ) - return prefix, name, int(port_text) + return prefix, parts.netloc.rpartition(":")[0], port def _uri_host_and_path(uri: str) -> Tuple[str, str]: @@ -167,7 +198,8 @@ def _kernel_host_and_path( if embedded is not None: # A port embedded in server_hostname always wins, as on main (the kernel # connected to it); a differing _port is ignored with a warning, not an error. - if embedded != port: + # The default 443 is not a choice worth warning about, so it is never flagged. + if port != 443 and embedded != port: logger.warning( "server_hostname %s carries port %d, which differs from _port=%d; " "connecting to port %d. Set the port in one place only.", @@ -179,40 +211,57 @@ def _kernel_host_and_path( return server_hostname, http_path if port == 443: return server_hostname, http_path - if ":" in name and not name.startswith("["): - name = f"[{name}]" return f"{prefix}{name}:{port}", http_path +def _is_foreign_jwt(access_token: str, server_hostname: str) -> bool: + """True when ``access_token`` is a JWT whose issuer is not ``server_hostname``. Thrift + token-exchanges such a token at server_hostname (TokenFederationProvider); the kernel + wraps every provider with the connection host, so it exchanges at the connection host. + """ + claims = decode_token(access_token) + if not claims: + return False + issuer = claims.get("iss", "") + return not is_same_host(issuer, normalize_host_with_protocol(server_hostname)) + + def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> None: """AC-P4: with OAuth, a ``_connection_uri`` origin must equal ``https://server_hostname``. Thrift keeps OAuth on server_hostname; the kernel uses one host for API and token calls. URI-only use (empty server_hostname) has no server origin to compare and is not checked. """ + if not (kwargs.get("_connection_uri") and server_hostname): + return + access_token = kwargs.get("access_token") + # A JWT from a foreign issuer is exchanged, so it is an OAuth credential even though + # it arrives as access_token. + foreign_jwt = bool(access_token) and _is_foreign_jwt(access_token, server_hostname) oauth = ( kwargs.get("auth_type") in _OAUTH_AUTH_TYPES or kwargs.get("oauth_jwt_key_file") or (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) + or foreign_jwt ) # The kernel auth bridge routes an access_token with no JWT or client-secret # M2M credentials to PAT, even for auth_type='databricks-oauth'; azure-sp-m2m # is resolved before that branch and stays OAuth. A custom credentials_provider # is rejected by the bridge, so it is not an OAuth origin here. routed_to_pat = ( - bool(kwargs.get("access_token")) + bool(access_token) + and not foreign_jwt and kwargs.get("auth_type") != "azure-sp-m2m" and not kwargs.get("oauth_jwt_key_file") and not (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) ) - if routed_to_pat or not ( - kwargs.get("_connection_uri") and server_hostname and oauth - ): + if routed_to_pat or not oauth: return _, name, port = _split_host(server_hostname) uri = urlsplit(kernel_host) + server_host = urlsplit(f"//{name}").hostname or "" if (uri.scheme, uri.hostname or "", uri.port or 443) != ( "https", - name.strip("[]").lower(), + server_host, port or 443, ): raise ValueError( diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 517fbe834..047fa9bb5 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -3,6 +3,7 @@ from contextlib import contextmanager import gc import importlib.metadata as ilm +import jwt import logging import socket import sys @@ -1317,7 +1318,6 @@ def test_kernel_host_and_path_rejects(server, kwargs, message): "server, port, embedded", [ ("h.io:7000", 8443, 7000), - ("h.io:8443", 443, 8443), ("[::1]:7000", 8443, 7000), ("h.io:443", 8443, 443), ], @@ -1334,12 +1334,61 @@ def test_embedded_port_wins_over_differing_explicit_port( assert f"_port={port}" in caplog.text +@pytest.mark.parametrize("server", ["h.io:8443", "[::1]:8443", "h.io:7000"]) +def test_embedded_port_with_default_443_does_not_warn(server, caplog): + # An explicit _port of the default 443 is not a conflict worth a warning (aligned + # with Node), so only a non-default _port that differs from the embedded port logs. + with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): + host, path = _kernel_host_and_path(server, _P, {"_port": 443}) + assert (host, path) == (server, _P) + assert "carries port" not in caplog.text + + def test_embedded_port_equal_to_explicit_port_does_not_warn(caplog): with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): _kernel_host_and_path("h.io:8443", _P, {"_port": 8443}) assert "differs from _port" not in caplog.text +@pytest.mark.parametrize( + "server, kwargs, expected", + [ + # urlsplit edge cases: the original string passes through unless a port is folded. + ("h:", {}, ("h:", _P)), + ("user@h", {}, ("user@h", _P)), + ("user@h", {"_port": 8443}, ("user@h:8443", _P)), + ("[::1", {}, ("[::1", _P)), + ("::1", {"_port": 443}, ("::1", _P)), + ("H.Example.COM", {}, ("H.Example.COM", _P)), + ("H.Example.COM", {"_port": 8443}, ("H.Example.COM:8443", _P)), + ("h:8443/", {"_port": 8443}, ("h:8443/", _P)), + ("h/x", {}, ("h/x", _P)), + ("h?q", {}, ("h?q", _P)), + ], +) +def test_kernel_host_and_path_stdlib_edges_accept(server, kwargs, expected): + assert _kernel_host_and_path(server, _P, kwargs) == expected + + +@pytest.mark.parametrize( + "server, kwargs, message", + [ + # Trailing ':' is an empty port, which is malformed with an explicit _port. + ("h:", {"_port": 8443}, "malformed"), + ("[::1", {"_port": 8443}, "malformed"), + ("h:abc", {"_port": 8443}, "malformed"), + # A colon in userinfo is not a port, but a credential-bearing host is never folded. + ("user:pass@h", {"_port": 8443}, "malformed"), + ("h:70000", {"_port": 8443}, "1-65535"), + ("h/x", {"_port": 8443}, "path, query or fragment"), + ("h?q", {"_port": 8443}, "path, query or fragment"), + ], +) +def test_kernel_host_and_path_stdlib_edges_reject(server, kwargs, message): + with pytest.raises(ValueError, match=message): + _kernel_host_and_path(server, _P, kwargs) + + @pytest.mark.parametrize( "server, port", [ @@ -1459,6 +1508,51 @@ def test_oauth_origin_guard(server, kwargs, uri, should_raise): _check_oauth_origin(server, kernel_host, kwargs) +def _jwt(issuer): + # Unverified on purpose: the guard only reads the iss claim, so the key is a placeholder. + return jwt.encode({"iss": issuer, "sub": "user"}, "k" * 32, algorithm="HS256") + + +@pytest.mark.parametrize( + "server, uri, token, should_raise", + [ + # A foreign-issuer JWT is exchanged at the kernel's connection host, so a + # mismatched _connection_uri origin must fail before any request. + ( + "h.io", + "https://evil.io/sql/x", + _jwt("https://login.microsoftonline.com/tenant/v2.0"), + True, + ), + # The same foreign JWT is fine when the _connection_uri origin matches. + ( + "h.io", + "https://h.io/sql/x", + _jwt("https://login.microsoftonline.com/tenant/v2.0"), + False, + ), + # An issuer on the server host is not exchanged (Thrift parity), so it passes. + ("h.io", "https://h.io/sql/x", _jwt("https://h.io/oidc"), False), + # A PAT is not a JWT and is not guarded. + ("h.io", "https://evil.io/sql/x", "dapi-xyz", False), + ], +) +def test_foreign_issuer_access_token_origin_guard(server, uri, token, should_raise): + kwargs = {"access_token": token, "_connection_uri": uri} + kernel_host = _kernel_host_and_path(server, _P, kwargs)[0] + if should_raise: + with pytest.raises(ValueError, match="requires the _connection_uri origin"): + _check_oauth_origin(server, kernel_host, kwargs) + else: + _check_oauth_origin(server, kernel_host, kwargs) + + +def test_foreign_issuer_access_token_without_connection_uri_is_unaffected(): + # No _connection_uri: the kernel host is the server_hostname, so there is no origin to check. + kwargs = {"access_token": _jwt("https://login.microsoftonline.com/tenant/v2.0")} + _check_oauth_origin("h.io", "h.io", kwargs) + + @pytest.mark.parametrize( "uri, secret", [ From f242c0f7151bef3624c4d9ae7c6167e7fad28219 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:05:38 +0000 Subject: [PATCH 11/12] kernel: skip ip_address for host names on the host:port path _is_bare_ipv6 now returns early unless the text has at least two colons and no leading bracket. ip_address raised a ValueError for every host name and host:port string, which made the folding path about 2x slower than bc7c09e. A bare IPv6 literal always has two or more colons, so behavior is unchanged; the session tests and the realkernel loopback tests pass. Microbenchmark, median us per call (20000 calls x 7 repeats): - bare host, _port 8443: 1.305 base, 1.261 new - host:port, _port 8443: 1.364 base, 1.393 new (+2%) - IPv6 bracketed, _port 8443: 1.340 base, 1.355 new (+1%) - IPv6 bare, _port 8443: 1.413 base, 2.642 new (+87%, the successful ip_address parse) Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cf61fbac4..80e7c22cc 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -96,6 +96,10 @@ def _redact(value: str) -> str: def _is_bare_ipv6(text: str) -> bool: """True for an unbracketed IPv6 literal such as ``::1``, which urlsplit reads as a port.""" + # A bare IPv6 literal has at least two colons and no brackets. Checking that first skips + # the costly ValueError that ip_address raises for names and host:port strings. + if text.count(":") < 2 or text.startswith("["): + return False try: return ipaddress.ip_address(text).version == 6 except ValueError: @@ -124,7 +128,7 @@ def _split_host(host: str) -> Tuple[str, str, Optional[int]]: f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port; " "a port must be 1-65535" ) from None - if parts.password is not None: + if "@" in parts.netloc and parts.password is not None: # A colon before '@' is userinfo, not a port, but a credential-bearing host is # still refused when a port is requested (as on main), so it is never folded. raise ValueError( From 5082c7ee8b454e82ef9242de0683c50c01aff24c Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:48:28 +0000 Subject: [PATCH 12/12] kernel: drop bare IPv6 support on the host:port path Remove _is_bare_ipv6 and the ipaddress import. Bare IPv6 hosts such as ::1 never worked on the kernel path: the kernel prepends https:// and its URL parser rejects an unbracketed IPv6 literal. IPv6 literals must be bracketed ([::1]). With _port set, urlsplit reads everything after the first colon of a bare IPv6 literal as the port. That text always contains another colon, so the integer parse fails and the existing "malformed IPv6 literal or port" ValueError is raised before any request. This includes _port=443. Without _port the host still passes through unchanged, as on main. Tests: the ("::1", 8443) -> "[::1]:8443" row is removed. ::1 with _port 8443 or 443, and 2001:db8::1 with _port 8443, are now in the reject table. Co-authored-by: Isaac Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- src/databricks/sql/session.py | 18 ++---------------- tests/unit/test_session.py | 10 +++++++--- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index 80e7c22cc..e8919bcb4 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -1,4 +1,3 @@ -import ipaddress import logging import re from functools import cached_property @@ -94,26 +93,13 @@ def _redact(value: str) -> str: return f"{scheme}{sep}{host}" -def _is_bare_ipv6(text: str) -> bool: - """True for an unbracketed IPv6 literal such as ``::1``, which urlsplit reads as a port.""" - # A bare IPv6 literal has at least two colons and no brackets. Checking that first skips - # the costly ValueError that ip_address raises for names and host:port strings. - if text.count(":") < 2 or text.startswith("["): - return False - try: - return ipaddress.ip_address(text).version == 6 - except ValueError: - return False - - def _split_host(host: str) -> Tuple[str, str, Optional[int]]: """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``. ``name`` keeps the - input's case and any userinfo, and a bare IPv6 literal gets its brackets. + input's case and any userinfo. IPv6 literals must be bracketed: urlsplit reads a bare + ``::1`` as a malformed port, so it is rejected. """ prefix, sep, rest = host.rstrip("/").rpartition("://") prefix += sep - if _is_bare_ipv6(rest): - return prefix, f"[{rest}]", None # urlsplit reads a trailing ':' as an empty port; with an explicit _port that is an error. if rest.endswith(":"): raise ValueError( diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 047fa9bb5..82f1a4ba3 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1202,7 +1202,7 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): @pytest.mark.parametrize( "server, hpath, kwargs, expected", [ - # AC-P1: _port folds into host:port; 443 adds nothing; IPv6 is bracketed. + # AC-P1: _port folds into host:port; 443 adds nothing; a bracketed IPv6 keeps its brackets. ("h.io", _P, {}, ("h.io", _P)), ("h.io", _P, {"_port": 8443}, ("h.io:8443", _P)), ("h.io", _P, {"_port": "8443"}, ("h.io:8443", _P)), @@ -1224,8 +1224,9 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): ("h.io/x", _P, {}, ("h.io/x", _P)), ("https://h.io:8443/", _P, {"_port": 8443}, ("https://h.io:8443/", _P)), ("https://h.io", _P, {"_port": 8443}, ("https://h.io:8443", _P)), + # Without _port a bare IPv6 host passes through, as on main; with _port it is + # rejected (see test_kernel_host_and_path_stdlib_edges_reject). Bracket it. ("::1", _P, {}, ("::1", _P)), - ("::1", _P, {"_port": 8443}, ("[::1]:8443", _P)), ("[::1]", _P, {"_port": 8443}, ("[::1]:8443", _P)), ("h.io:8443", _P, {}, ("h.io:8443", _P)), ("h.io:8443", _P, {"_port": 8443}, ("h.io:8443", _P)), @@ -1358,7 +1359,6 @@ def test_embedded_port_equal_to_explicit_port_does_not_warn(caplog): ("user@h", {}, ("user@h", _P)), ("user@h", {"_port": 8443}, ("user@h:8443", _P)), ("[::1", {}, ("[::1", _P)), - ("::1", {"_port": 443}, ("::1", _P)), ("H.Example.COM", {}, ("H.Example.COM", _P)), ("H.Example.COM", {"_port": 8443}, ("H.Example.COM:8443", _P)), ("h:8443/", {"_port": 8443}, ("h:8443/", _P)), @@ -1376,6 +1376,10 @@ def test_kernel_host_and_path_stdlib_edges_accept(server, kwargs, expected): # Trailing ':' is an empty port, which is malformed with an explicit _port. ("h:", {"_port": 8443}, "malformed"), ("[::1", {"_port": 8443}, "malformed"), + # Bare IPv6 is not supported with _port: urlsplit reads it as a malformed port. + ("::1", {"_port": 8443}, "malformed"), + ("::1", {"_port": 443}, "malformed"), + ("2001:db8::1", {"_port": 8443}, "malformed"), ("h:abc", {"_port": 8443}, "malformed"), # A colon in userinfo is not a port, but a credential-bearing host is never folded. ("user:pass@h", {"_port": 8443}, "malformed"),