diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..5577c7836 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Honor `_port` and `_connection_uri` on kernel connections (`use_kernel=True`), as Thrift does. `_port` sends the connection to that port (default 443 adds nothing); `_connection_uri` (`http(s)://host[:port]/path?query`) sets host, port, path and query and wins over `_port`; URI-only connections work. A port embedded in `server_hostname` wins over a differing `_port`, with a warning, as on main. Invalid ports, an empty `server_hostname` with `_port`, and URIs without a scheme or path raise `ValueError`. With OAuth, a `_connection_uri` origin that differs from `server_hostname` raises. With OAuth and a non-default `_port`, the token exchange goes to the same host:port, unlike Thrift (documented, not guarded). - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index f0b3d2ee7..637577b4c 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -73,8 +73,8 @@ to change without notice. | ----------------- | ----- | :----: | :----: | ------------- | ----------------------------------------------------------------------------------------------------------------------- | | `server_hostname` | `str` | ✅ | ✅ | — (required) | Workspace hostname, e.g. `dbc-12345.cloud.databricks.com`. | | `http_path` | `str` | ✅ | ✅ | — (required) | Thrift accepts a SQL-warehouse path **or** an all-purpose-cluster path; Kernel requires a warehouse/endpoint path. | -| `_port` | `int` | ✅ | — | `443` | TCP port (advanced). Not threaded to the kernel; it derives host/port from `server_hostname` + `http_path`. | -| `_connection_uri` | `str` | ✅ | — | `None` | Thrift-only internal override of `server_hostname`/`http_path`. No kernel equivalent. | +| `_port` | `int` | ✅ | ✅ | `443` | TCP port (advanced). On Kernel a falsy value means the default, and a port embedded in `server_hostname` wins over a differing `_port` (with a warning). | +| `_connection_uri` | `str` | ✅ | ✅ | `None` | Overrides `server_hostname`, `_port` and `http_path` on both, and wins over `_port`. On Kernel with OAuth, the URI origin must match `server_hostname`, and OAuth uses the same host:port as the connection (Thrift keeps OAuth on the bare host). | | `http_headers` | `List[Tuple[str, str]]` | ✅ | ⚠️ | `None` | Extra headers sent on requests. Kernel filters `Authorization` and `x-databricks-org-id` because it manages them itself; other headers are forwarded. | | `user_agent_entry`| `str` | ✅ | ✅ | `None` | Custom tag folded into the composed `User-Agent` on both. (`_user_agent_entry` is a deprecated alias that warns.) | diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..bf6a63840 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,15 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + + Kernel overrides: _port sends the connection to that port + (the default 443 adds nothing; a falsy _port means the default). A port + embedded in server_hostname wins over a differing _port, with a warning. + _connection_uri (https://host[:port]/path?query) + sets host, port, path and query, as on Thrift, and wins over _port. + With OAuth, _connection_uri must have the same origin as + server_hostname. With OAuth and a non-default _port, the token + exchange goes to the same host:port, unlike Thrift. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..e8919bcb4 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -1,12 +1,15 @@ import logging import re from functools import cached_property +from urllib.parse import urlsplit from typing import Dict, Tuple, List, Optional, Any, Type, TYPE_CHECKING from databricks.sql.types import SSLOptions from databricks.sql.auth.auth import get_python_sql_connector_auth_provider +from databricks.sql.auth.auth_utils import decode_token, is_same_host from databricks.sql.auth.authenticators import AccessTokenAuthProvider from databricks.sql.auth.common import ClientContext +from databricks.sql.common.url_utils import normalize_host_with_protocol from databricks.sql.exc import SessionAlreadyClosedError, DatabaseError, RequestError from databricks.sql import __version__ from databricks.sql import USER_AGENT_NAME @@ -47,6 +50,218 @@ def __getattr__(name: str) -> Any: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +_OAUTH_AUTH_TYPES = ("databricks-oauth", "azure-oauth", "azure-sp-m2m") + + +def _port_or_none(value: Any) -> Optional[int]: + """Validate ``_port``. A falsy value (None, 0, '', False) means the default, as on + Thrift (``port or 443``); a truthy value must be an integer between 1 and 65535. + """ + if not value or (isinstance(value, str) and not value.strip()): + return None + text = ( + str(value).strip() + if isinstance(value, (int, str)) and not isinstance(value, bool) + else "" + ) + # isdigit with isascii rejects signs, whitespace and underscores that int() would accept. + if not (text.isascii() and text.isdigit()) or int(text) > 65535: + raise ValueError( + f"Invalid _port {value!r}: expected an integer between 1 and 65535" + ) + return int(text) or None + + +def _redact(value: str) -> str: + """Keep only scheme and host[:port] of a URI or host string for an error message. + + Userinfo, path, query and fragment can carry credentials, so they never reach an + exception text or log line. + """ + scheme, sep, rest = value.partition("://") + if not sep: + scheme, rest = "", value + # A "/" or "?" can sit inside a password, and "@" can sit in a path, so once any + # userinfo marker is present the whole authority is dropped rather than guessed at. + if "@" in value or "%40" in value.lower(): + return f"{scheme}{sep}" + try: + host = urlsplit(f"//{rest}").netloc + except ValueError: + # urlsplit refuses some hosts (bad IPv6, NFKC-unsafe text); show none of them. + return f"{scheme}{sep}" + return f"{scheme}{sep}{host}" + + +def _split_host(host: str) -> Tuple[str, str, Optional[int]]: + """Split ``[scheme://]name[:port]`` into ``(prefix, name, port)``. ``name`` keeps the + input's case and any userinfo. IPv6 literals must be bracketed: urlsplit reads a bare + ``::1`` as a malformed port, so it is rejected. + """ + prefix, sep, rest = host.rstrip("/").rpartition("://") + prefix += sep + # urlsplit reads a trailing ':' as an empty port; with an explicit _port that is an error. + if rest.endswith(":"): + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port" + ) + try: + parts = urlsplit(f"//{rest}") + port = parts.port + except ValueError: + # ``from None`` drops the urlsplit exception, whose text can quote the raw netloc. + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port; " + "a port must be 1-65535" + ) from None + if "@" in parts.netloc and parts.password is not None: + # A colon before '@' is userinfo, not a port, but a credential-bearing host is + # still refused when a port is requested (as on main), so it is never folded. + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: malformed IPv6 literal or port" + ) + if port is None: + return prefix, parts.netloc, None + if not 1 <= port <= 65535: + raise ValueError( + f"Invalid server_hostname {_redact(host)!r}: port {port} is out of range" + ) + return prefix, parts.netloc.rpartition(":")[0], port + + +def _uri_host_and_path(uri: str) -> Tuple[str, str]: + """Split a ``_connection_uri`` into the kernel host (scheme, authority) and path.""" + try: + parts = urlsplit(uri) + except ValueError: + parts = None + if parts is None: + # Raised outside the except block so no chained exception carries urlsplit's netloc. + raise ValueError( + f"Invalid _connection_uri {_redact(uri)!r}: could not be parsed" + ) + try: + port = parts.port + except ValueError: + port = 0 + if ( + parts.scheme not in ("http", "https") + or not parts.hostname + or "@" in parts.netloc + or port == 0 + or parts.path in ("", "/") + ): + raise ValueError( + f"Invalid _connection_uri {_redact(uri)!r}: expected http(s)://host[:port]/path with a " + "warehouse path and no credentials" + ) + return f"{parts.scheme}://{parts.netloc}", parts.path + ( + f"?{parts.query}" if parts.query else "" + ) + + +def _kernel_host_and_path( + server_hostname: str, http_path: str, kwargs: dict +) -> Tuple[str, str]: + """Resolve the kernel ``(host, http_path)``. ``_connection_uri`` replaces both and wins over + ``_port``; ``_port`` folds into the host as ``host:port`` (default 443 adds nothing). + """ + if kwargs.get("_connection_uri"): + return _uri_host_and_path(kwargs["_connection_uri"]) + port = _port_or_none(kwargs.get("_port")) + if port is None: + return server_hostname, http_path + if not server_hostname: + raise ValueError( + "server_hostname is required on use_kernel=True when _port is set " + "(or pass _connection_uri, which carries its own host)" + ) + # Checked before the port is parsed: 'h.io/path:8443' would otherwise read as host + # 'h.io/path' with port 8443. Only the text after any scheme is checked. + authority = server_hostname.rstrip("/").rpartition("://")[2] + if any(c in authority for c in "/?#"): + raise ValueError( + f"Invalid server_hostname {_redact(server_hostname)!r}: a path, query or " + "fragment cannot be combined with _port; put the port in server_hostname" + ) + prefix, name, embedded = _split_host(server_hostname) + if embedded is not None: + # A port embedded in server_hostname always wins, as on main (the kernel + # connected to it); a differing _port is ignored with a warning, not an error. + # The default 443 is not a choice worth warning about, so it is never flagged. + if port != 443 and embedded != port: + logger.warning( + "server_hostname %s carries port %d, which differs from _port=%d; " + "connecting to port %d. Set the port in one place only.", + _redact(server_hostname), + embedded, + port, + embedded, + ) + return server_hostname, http_path + if port == 443: + return server_hostname, http_path + return f"{prefix}{name}:{port}", http_path + + +def _is_foreign_jwt(access_token: str, server_hostname: str) -> bool: + """True when ``access_token`` is a JWT whose issuer is not ``server_hostname``. Thrift + token-exchanges such a token at server_hostname (TokenFederationProvider); the kernel + wraps every provider with the connection host, so it exchanges at the connection host. + """ + claims = decode_token(access_token) + if not claims: + return False + issuer = claims.get("iss", "") + return not is_same_host(issuer, normalize_host_with_protocol(server_hostname)) + + +def _check_oauth_origin(server_hostname: str, kernel_host: str, kwargs: dict) -> None: + """AC-P4: with OAuth, a ``_connection_uri`` origin must equal ``https://server_hostname``. + Thrift keeps OAuth on server_hostname; the kernel uses one host for API and token calls. + URI-only use (empty server_hostname) has no server origin to compare and is not checked. + """ + if not (kwargs.get("_connection_uri") and server_hostname): + return + access_token = kwargs.get("access_token") + # A JWT from a foreign issuer is exchanged, so it is an OAuth credential even though + # it arrives as access_token. + foreign_jwt = bool(access_token) and _is_foreign_jwt(access_token, server_hostname) + oauth = ( + kwargs.get("auth_type") in _OAUTH_AUTH_TYPES + or kwargs.get("oauth_jwt_key_file") + or (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) + or foreign_jwt + ) + # The kernel auth bridge routes an access_token with no JWT or client-secret + # M2M credentials to PAT, even for auth_type='databricks-oauth'; azure-sp-m2m + # is resolved before that branch and stays OAuth. A custom credentials_provider + # is rejected by the bridge, so it is not an OAuth origin here. + routed_to_pat = ( + bool(access_token) + and not foreign_jwt + and kwargs.get("auth_type") != "azure-sp-m2m" + and not kwargs.get("oauth_jwt_key_file") + and not (kwargs.get("oauth_client_id") and kwargs.get("oauth_client_secret")) + ) + if routed_to_pat or not oauth: + return + _, name, port = _split_host(server_hostname) + uri = urlsplit(kernel_host) + server_host = urlsplit(f"//{name}").hostname or "" + if (uri.scheme, uri.hostname or "", uri.port or 443) != ( + "https", + server_host, + port or 443, + ): + raise ValueError( + f"OAuth requires the _connection_uri origin to match " + f"https://{_redact(server_hostname)}; " + f"the kernel would send the credentials to {_redact(kernel_host)}. " + "Use a PAT or match the origin." + ) + + class Session: def __init__( self, @@ -323,9 +538,23 @@ def _create_backend( "_telemetry_circuit_breaker_enabled" ), } + kernel_host, kernel_http_path = _kernel_host_and_path( + server_hostname, http_path, kwargs + ) + _check_oauth_origin(server_hostname, kernel_host, kwargs) + if kernel_http_path != http_path: + # A _connection_uri can point at another workspace: re-derive the + # SPOG org-id header from the resolved path (a caller header still wins). + base_headers = [ + h for h in all_headers if h not in self._spog_headers.items() + ] + self._spog_headers = self._extract_spog_headers( + kernel_http_path, base_headers + ) + all_headers = base_headers + list(self._spog_headers.items()) return KernelDatabricksClient( - server_hostname=server_hostname, - http_path=http_path, + server_hostname=kernel_host, + http_path=kernel_http_path, http_headers=all_headers, auth_provider=auth_provider, ssl_options=self.ssl_options, diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..82f1a4ba3 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -2,6 +2,13 @@ from unittest.mock import patch, MagicMock, Mock, PropertyMock from contextlib import contextmanager import gc +import importlib.metadata as ilm +import jwt +import logging +import socket +import sys +import threading +import types from databricks.sql.thrift_api.TCLIService.ttypes import ( TOpenSessionResp, @@ -10,6 +17,9 @@ ) from databricks.sql.backend.types import SessionId, BackendType from databricks.sql.common.agent import KNOWN_AGENTS +from databricks.sql.session import _kernel_host_and_path, _check_oauth_origin + +import databricks.sql from databricks.sql.exc import ( ReydenThriftUnsupportedError, DatabaseError, @@ -17,8 +27,6 @@ ) from databricks.sql.session import Session -import databricks.sql - class TestSession: """ @@ -1184,3 +1192,608 @@ def test_pat_recovery_does_not_inject_auth_type(self, mock_thrift): assert kwargs["auth_options"].get("auth_type") is None finally: conn.close() + + +# ---- kernel _port / _connection_uri overrides (use_kernel=True) ---- + +_P = "/sql/1.0/warehouses/abc" + + +@pytest.mark.parametrize( + "server, hpath, kwargs, expected", + [ + # AC-P1: _port folds into host:port; 443 adds nothing; a bracketed IPv6 keeps its brackets. + ("h.io", _P, {}, ("h.io", _P)), + ("h.io", _P, {"_port": 8443}, ("h.io:8443", _P)), + ("h.io", _P, {"_port": "8443"}, ("h.io:8443", _P)), + ("h.io", _P, {"_port": 1}, ("h.io:1", _P)), + ("h.io", _P, {"_port": 65535}, ("h.io:65535", _P)), + ("h.io", _P, {"_port": 443}, ("h.io", _P)), + ("h.io", _P, {"_port": 0}, ("h.io", _P)), + # Falsy _port means the default, as on Thrift (port or 443); no :443 is added. + ("h.io", _P, {"_port": None}, ("h.io", _P)), + ("h.io", _P, {"_port": ""}, ("h.io", _P)), + ("h.io", _P, {"_port": " "}, ("h.io", _P)), + ("h.io", _P, {"_port": False}, ("h.io", _P)), + ("h.io", _P, {"_port": "0"}, ("h.io", _P)), + # Trailing "/" is trimmed before the port is folded in. + ("h.io/", _P, {"_port": 8443}, ("h.io:8443", _P)), + ("h.io:8443/", _P, {"_port": 8443}, ("h.io:8443/", _P)), + ("h.io/", _P, {}, ("h.io/", _P)), + # Without _port, a host with a path passes through unchanged, as on main. + ("h.io/x", _P, {}, ("h.io/x", _P)), + ("https://h.io:8443/", _P, {"_port": 8443}, ("https://h.io:8443/", _P)), + ("https://h.io", _P, {"_port": 8443}, ("https://h.io:8443", _P)), + # Without _port a bare IPv6 host passes through, as on main; with _port it is + # rejected (see test_kernel_host_and_path_stdlib_edges_reject). Bracket it. + ("::1", _P, {}, ("::1", _P)), + ("[::1]", _P, {"_port": 8443}, ("[::1]:8443", _P)), + ("h.io:8443", _P, {}, ("h.io:8443", _P)), + ("h.io:8443", _P, {"_port": 8443}, ("h.io:8443", _P)), + # AC-P2: _connection_uri replaces host, port, path and query; it wins over _port. + ( + "h.io", + _P, + {"_connection_uri": "https://d.io:8443/sql/1.0/warehouses/xyz"}, + ("https://d.io:8443", "/sql/1.0/warehouses/xyz"), + ), + ( + "h.io", + _P, + {"_connection_uri": "https://h.io/sql/x?o=123"}, + ("https://h.io", "/sql/x?o=123"), + ), + ( + "h.io", + _P, + {"_connection_uri": "http://h.io:8080/sql/x"}, + ("http://h.io:8080", "/sql/x"), + ), + ( + "h.io", + _P, + {"_connection_uri": "https://d.io:9999/p", "_port": 8443}, + ("https://d.io:9999", "/p"), + ), + ( + "", + "", + {"_connection_uri": "https://h.io/sql/x"}, + ("https://h.io", "/sql/x"), + ), # URI-only + # Without _port or _connection_uri the host passes through as on main, even if + # it is not a valid host:port (no port is parsed on the default path). + ("h.io:0", _P, {}, ("h.io:0", _P)), + ("h.io:notaport", _P, {}, ("h.io:notaport", _P)), + ("h.io:", _P, {}, ("h.io:", _P)), + ], +) +def test_kernel_host_and_path_accepts(server, hpath, kwargs, expected): + assert _kernel_host_and_path(server, hpath, kwargs) == expected + + +@pytest.mark.parametrize( + "server, kwargs, message", + [ + # AC-P3: ports outside 1..65535, or not integers. + ("h.io", {"_port": -1}, "Invalid _port"), + ("h.io", {"_port": 65536}, "Invalid _port"), + ("h.io", {"_port": "abc"}, "Invalid _port"), + ("h.io", {"_port": True}, "Invalid _port"), + ("h.io", {"_port": 8443.5}, "Invalid _port"), + ("h.io:notaport", {"_port": 8443}, "malformed"), + ("h.io:0", {"_port": 8443}, "out of range"), + # An explicit _port with a path, query or fragment is rejected, even when the + # path looks like a port ('h.io/path:8443') or the port is the default (443). + ("h.io/x", {"_port": 8443}, "path, query or fragment"), + ("h.io/path:8443", {"_port": 8443}, "path, query or fragment"), + ("h.io/x", {"_port": 443}, "path, query or fragment"), + ("h.io?x=1", {"_port": 8443}, "path, query or fragment"), + ("h.io#x", {"_port": 8443}, "path, query or fragment"), + ("https://h.io/x", {"_port": 8443}, "path, query or fragment"), + ("h.io/x/", {"_port": 8443}, "path, query or fragment"), + # R11: _port needs a server_hostname unless _connection_uri carries the host. + ("", {"_port": 8443}, "server_hostname is required"), + ("", {"_port": 443}, "server_hostname is required"), + # AC-P3: _connection_uri must be http(s) with a host and a warehouse path. + ("h", {"_connection_uri": "d.io/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "ftp://h.io/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https://h.io:8443"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https://h.io:8443/"}, "Invalid _connection_uri"), + ( + "h", + {"_connection_uri": "https://h.io:notaport/sql/x"}, + "Invalid _connection_uri", + ), + ("h", {"_connection_uri": "https://h.io:0/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https://u:p@h.io/sql/x"}, "Invalid _connection_uri"), + ("h", {"_connection_uri": "https:///sql/x"}, "Invalid _connection_uri"), + ], +) +def test_kernel_host_and_path_rejects(server, kwargs, message): + with pytest.raises(ValueError, match=message): + _kernel_host_and_path(server, _P, kwargs) + + +@pytest.mark.parametrize( + "server, port, embedded", + [ + ("h.io:7000", 8443, 7000), + ("[::1]:7000", 8443, 7000), + ("h.io:443", 8443, 443), + ], +) +def test_embedded_port_wins_over_differing_explicit_port( + server, port, embedded, caplog +): + # A port embedded in server_hostname always wins (as on main); a differing _port + # is not an error. The warning names both ports, and the host is passed unchanged. + with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): + host, path = _kernel_host_and_path(server, _P, {"_port": port}) + assert (host, path) == (server, _P) + assert f"carries port {embedded}" in caplog.text + assert f"_port={port}" in caplog.text + + +@pytest.mark.parametrize("server", ["h.io:8443", "[::1]:8443", "h.io:7000"]) +def test_embedded_port_with_default_443_does_not_warn(server, caplog): + # An explicit _port of the default 443 is not a conflict worth a warning (aligned + # with Node), so only a non-default _port that differs from the embedded port logs. + with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): + host, path = _kernel_host_and_path(server, _P, {"_port": 443}) + assert (host, path) == (server, _P) + assert "carries port" not in caplog.text + + +def test_embedded_port_equal_to_explicit_port_does_not_warn(caplog): + with caplog.at_level(logging.WARNING, logger="databricks.sql.session"): + _kernel_host_and_path("h.io:8443", _P, {"_port": 8443}) + assert "differs from _port" not in caplog.text + + +@pytest.mark.parametrize( + "server, kwargs, expected", + [ + # urlsplit edge cases: the original string passes through unless a port is folded. + ("h:", {}, ("h:", _P)), + ("user@h", {}, ("user@h", _P)), + ("user@h", {"_port": 8443}, ("user@h:8443", _P)), + ("[::1", {}, ("[::1", _P)), + ("H.Example.COM", {}, ("H.Example.COM", _P)), + ("H.Example.COM", {"_port": 8443}, ("H.Example.COM:8443", _P)), + ("h:8443/", {"_port": 8443}, ("h:8443/", _P)), + ("h/x", {}, ("h/x", _P)), + ("h?q", {}, ("h?q", _P)), + ], +) +def test_kernel_host_and_path_stdlib_edges_accept(server, kwargs, expected): + assert _kernel_host_and_path(server, _P, kwargs) == expected + + +@pytest.mark.parametrize( + "server, kwargs, message", + [ + # Trailing ':' is an empty port, which is malformed with an explicit _port. + ("h:", {"_port": 8443}, "malformed"), + ("[::1", {"_port": 8443}, "malformed"), + # Bare IPv6 is not supported with _port: urlsplit reads it as a malformed port. + ("::1", {"_port": 8443}, "malformed"), + ("::1", {"_port": 443}, "malformed"), + ("2001:db8::1", {"_port": 8443}, "malformed"), + ("h:abc", {"_port": 8443}, "malformed"), + # A colon in userinfo is not a port, but a credential-bearing host is never folded. + ("user:pass@h", {"_port": 8443}, "malformed"), + ("h:70000", {"_port": 8443}, "1-65535"), + ("h/x", {"_port": 8443}, "path, query or fragment"), + ("h?q", {"_port": 8443}, "path, query or fragment"), + ], +) +def test_kernel_host_and_path_stdlib_edges_reject(server, kwargs, message): + with pytest.raises(ValueError, match=message): + _kernel_host_and_path(server, _P, kwargs) + + +@pytest.mark.parametrize( + "server, port", + [ + ("h.io:8443", 443), + ("h.io", 8443), + ("h.io/x", 8443), + ("h.io", ""), + ], +) +@patch("databricks.sql.session.ThriftDatabricksClient") +def test_thrift_receives_host_and_port_unchanged(mock_client_class, server, port): + # Thrift is unchanged: the host and _port reach ThriftDatabricksClient verbatim, and + # ThriftDatabricksClient applies its own `port or 443` to the URL. The kernel-only + # folding above does not apply here. + connection = databricks.sql.connect( + server_hostname=server, + http_path=_P, + access_token="tok", + enable_telemetry=False, + _port=port, + ) + call_kwargs = mock_client_class.call_args.kwargs + assert call_kwargs["server_hostname"] == server + assert call_kwargs["port"] == port + connection.close() + + +@pytest.mark.parametrize( + "server, kwargs, uri, should_raise", + [ + # AC-P4: with OAuth, the _connection_uri origin must equal https://server_hostname. + ( + "h.io", + {"oauth_client_id": "c", "oauth_client_secret": "s"}, + "https://evil.io/sql/x", + True, + ), + ("h.io", {"auth_type": "azure-sp-m2m"}, "https://evil.io/sql/x", True), + ("h.io", {"oauth_jwt_key_file": "/k.pem"}, "https://evil.io/sql/x", True), + ("h.io", {"auth_type": "databricks-oauth"}, "http://h.io/sql/x", True), + ("h.io", {"auth_type": "databricks-oauth"}, "https://h.io:8443/sql/x", True), + ("h.io", {"auth_type": "databricks-oauth"}, "https://H.io/sql/x", False), + ( + "h.io:8443", + {"auth_type": "databricks-oauth"}, + "https://h.io:8443/sql/x", + False, + ), + ("[::1]", {"auth_type": "databricks-oauth"}, "https://[::1]/sql/x", False), + ( + "h.io", + {"access_token": "dapi"}, + "https://evil.io/sql/x", + False, + ), # PAT is not guarded + # The auth bridge routes access_token with no M2M/JWT credentials to PAT, + # even when auth_type is databricks-oauth; azure-sp-m2m and M2M/JWT stay guarded. + ( + "h.io", + {"auth_type": "databricks-oauth", "access_token": "dapi"}, + "https://evil.io/sql/x", + False, + ), + ( + "h.io", + {"auth_type": "azure-sp-m2m", "access_token": "dapi"}, + "https://evil.io/sql/x", + True, + ), + # The kernel auth bridge rejects a custom credentials_provider with NotSupportedError, + # so the origin guard does not classify it as OAuth. + ( + "h.io", + {"credentials_provider": object()}, + "https://evil.io/sql/x", + False, + ), + ( + "h.io", + {"oauth_jwt_key_file": "/k.pem", "access_token": "dapi"}, + "https://evil.io/sql/x", + True, + ), + ( + "h.io", + { + "oauth_client_id": "c", + "oauth_client_secret": "s", + "access_token": "dapi", + }, + "https://evil.io/sql/x", + True, + ), + ( + "", + {"auth_type": "databricks-oauth"}, + "https://h.io/sql/x", + False, + ), # URI-only + ( + "h.io", + {"auth_type": "databricks-oauth", "_port": 8443}, + None, + False, + ), # documented, not guarded + ], +) +def test_oauth_origin_guard(server, kwargs, uri, should_raise): + kwargs = dict(kwargs) + if uri is not None: + kwargs["_connection_uri"] = uri + kernel_host = _kernel_host_and_path(server, _P, kwargs)[0] + if should_raise: + with pytest.raises(ValueError, match="requires the _connection_uri origin"): + _check_oauth_origin(server, kernel_host, kwargs) + else: + _check_oauth_origin(server, kernel_host, kwargs) + + +def _jwt(issuer): + # Unverified on purpose: the guard only reads the iss claim, so the key is a placeholder. + return jwt.encode({"iss": issuer, "sub": "user"}, "k" * 32, algorithm="HS256") + + +@pytest.mark.parametrize( + "server, uri, token, should_raise", + [ + # A foreign-issuer JWT is exchanged at the kernel's connection host, so a + # mismatched _connection_uri origin must fail before any request. + ( + "h.io", + "https://evil.io/sql/x", + _jwt("https://login.microsoftonline.com/tenant/v2.0"), + True, + ), + # The same foreign JWT is fine when the _connection_uri origin matches. + ( + "h.io", + "https://h.io/sql/x", + _jwt("https://login.microsoftonline.com/tenant/v2.0"), + False, + ), + # An issuer on the server host is not exchanged (Thrift parity), so it passes. + ("h.io", "https://h.io/sql/x", _jwt("https://h.io/oidc"), False), + # A PAT is not a JWT and is not guarded. + ("h.io", "https://evil.io/sql/x", "dapi-xyz", False), + ], +) +def test_foreign_issuer_access_token_origin_guard(server, uri, token, should_raise): + kwargs = {"access_token": token, "_connection_uri": uri} + kernel_host = _kernel_host_and_path(server, _P, kwargs)[0] + if should_raise: + with pytest.raises(ValueError, match="requires the _connection_uri origin"): + _check_oauth_origin(server, kernel_host, kwargs) + else: + _check_oauth_origin(server, kernel_host, kwargs) + + +def test_foreign_issuer_access_token_without_connection_uri_is_unaffected(): + # No _connection_uri: the kernel host is the server_hostname, so there is no origin to check. + kwargs = {"access_token": _jwt("https://login.microsoftonline.com/tenant/v2.0")} + _check_oauth_origin("h.io", "h.io", kwargs) + + +@pytest.mark.parametrize( + "uri, secret", + [ + ("https://user:secret@h.io/sql/x", "secret"), + ("https://h.io:8443?token=secret", "secret"), + # "@" after the host with no path: the URI is rejected, and the "@" must not leak. + ("https://h.io?x=a@secret", "secret"), + # Percent-encoded "@" hides the userinfo marker from a literal "@" check. + ("https://user%3Asecret%40h.io", "secret"), + # A "/" or "?" inside the password must not cut the authority short. + ("https://user:sec/ret@h.io/sql/x", "sec/ret"), + ("https://user:sec?ret@h.io/sql/x", "sec?ret"), + # A "/" inside the password, with "@" after it in the path: netloc stops at "/". + ("https://user:secret/foo@h.io/sql/x", "secret"), + # A fullwidth solidus makes urlsplit's NFKC check raise with the raw netloc in its text. + ("https://user:secret@h.io/sql/x", "secret"), + ], +) +def test_invalid_connection_uri_message_omits_credentials(uri, secret): + with pytest.raises(ValueError, match="Invalid _connection_uri") as exc: + _kernel_host_and_path("h", _P, {"_connection_uri": uri}) + assert secret not in str(exc.value) + # The secret must not survive in a chained cause or context either. + assert exc.value.__cause__ is None + assert exc.value.__context__ is None or secret not in str(exc.value.__context__) + + +def test_oauth_origin_message_omits_uri_query(): + kwargs = { + "auth_type": "databricks-oauth", + "_connection_uri": "https://evil.io/sql/x?token=secret", + } + kernel_host = _kernel_host_and_path("h.io", _P, kwargs)[0] + with pytest.raises(ValueError, match="requires the _connection_uri origin") as exc: + _check_oauth_origin("h.io", kernel_host, kwargs) + assert "secret" not in str(exc.value) + assert "https://evil.io" in str(exc.value) + + +@pytest.mark.parametrize( + "conn_kwargs, expected_host, expected_path", + [ + ({"server_hostname": "h.io", "http_path": _P, "_port": 8443}, "h.io:8443", _P), + ( + { + "server_hostname": "", + "http_path": "", + "_connection_uri": f"https://d.io:9999{_P}", + }, + "https://d.io:9999", + _P, + ), + ], +) +def test_resolved_kernel_host_and_path_reach_kernel_client( + fake_kernel, conn_kwargs, expected_host, expected_path +): + client, _ = fake_kernel + databricks.sql.connect( + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + **conn_kwargs, + ) + kwargs = client.call_args.kwargs + assert kwargs["server_hostname"] == expected_host + assert kwargs["http_path"] == expected_path + + +@pytest.fixture +def fake_kernel(): + """Fake the kernel wheel and client so connect() runs without a warehouse.""" + pytest.importorskip( + "pyarrow", reason="kernel client module imports pyarrow at load" + ) + fake = types.ModuleType("databricks_sql_kernel") + fake.KernelError = type("KernelError", (Exception,), {}) + fake.Session = MagicMock() + with patch.dict(sys.modules, {"databricks_sql_kernel": fake}), patch( + "databricks.sql.backend.kernel.client.KernelDatabricksClient" + ) as client, patch("databricks.sql.session.get_python_sql_connector_auth_provider"): + client.return_value.open_session.return_value = SessionId( + BackendType.SEA, "sess-id", None + ) + yield client, fake + + +def test_oauth_guard_refuses_before_kernel_client_is_built(fake_kernel): + client, fake = fake_kernel + with pytest.raises(ValueError, match="requires the _connection_uri origin"): + databricks.sql.connect( + server_hostname="h.io", + http_path=_P, + use_kernel=True, + enable_telemetry=False, + oauth_client_id="c", + oauth_client_secret="s", + _connection_uri="https://evil.io/sql/x", + ) + client.assert_not_called() + fake.Session.assert_not_called() + + +@pytest.mark.parametrize( + "hp, uri, expected_org", + [ + ("/sql/1.0/warehouses/abc?o=111", "https://d.io/sql/x?o=222", "222"), + ("/sql/1.0/warehouses/abc?o=111", "https://d.io/sql/x", None), + ("/sql/1.0/warehouses/abc", "https://d.io/sql/x?o=222", "222"), + ("/sql/1.0/warehouses/abc?o=111", None, "111"), + ], +) +def test_spog_org_id_follows_resolved_path(fake_kernel, hp, uri, expected_org): + client, _ = fake_kernel + extra = {"_connection_uri": uri} if uri else {} + databricks.sql.connect( + server_hostname="h.io", + http_path=hp, + access_token="dapi", + use_kernel=True, + enable_telemetry=False, + **extra, + ).close() + assert ( + dict(client.call_args.kwargs["http_headers"]).get("x-databricks-org-id") + == expected_org + ) + + +@pytest.mark.realkernel +@pytest.mark.parametrize( + "build", + [ + lambda p: {"server_hostname": "127.0.0.1", "http_path": _P, "_port": p}, + lambda p: { + "server_hostname": "", + "http_path": "", + "_connection_uri": f"https://127.0.0.1:{p}{_P}", + }, + ], + ids=["port", "uri-only"], +) +def test_kernel_dials_over_tls(build): + """Loopback dial on the real kernel wheel: a TCP listener on 127.0.0.1 must receive the + kernel's TLS handshake (first byte 0x16) at the requested port. Nothing completes the + handshake, so connect() is expected to fail; only the dial is asserted.""" + try: + ilm.version("databricks-sql-kernel") + except ilm.PackageNotFoundError: + pytest.skip("databricks-sql-kernel wheel not installed") + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(60) + port = listener.getsockname()[1] + received = {} + + def accept_one(): + try: + sock, _ = listener.accept() + sock.settimeout(10) + received["first_byte"] = sock.recv(1) + sock.close() + except OSError as exc: + received["error"] = repr(exc) + finally: + listener.close() + + thread = threading.Thread(target=accept_one, daemon=True) + thread.start() + try: + databricks.sql.connect( + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + _retry_stop_after_attempts_count=1, + **build(port), + ).close() + except Exception: + pass # expected: nothing behind the listener completes TLS + thread.join(timeout=70) + assert "error" not in received, received + assert received.get("first_byte") == b"\x16", received + + +@pytest.mark.realkernel +def test_kernel_embedded_port_wins_over_explicit_port(): + """Loopback on the real kernel wheel: server_hostname 'h:P1' with an explicit _port P2 + dials P1 (TLS first byte 0x16) and never P2.""" + try: + ilm.version("databricks-sql-kernel") + except ilm.PackageNotFoundError: + pytest.skip("databricks-sql-kernel wheel not installed") + + def listen(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + sock.settimeout(15) + return sock + + embedded, explicit = listen(), listen() + embedded_port = embedded.getsockname()[1] + explicit_port = explicit.getsockname()[1] + received = {} + + def accept_one(name, sock): + try: + conn, _ = sock.accept() + conn.settimeout(10) + received[name] = conn.recv(1) + conn.close() + except OSError as exc: + received[name + "_error"] = repr(exc) + finally: + sock.close() + + threads = [ + threading.Thread(target=accept_one, args=("embedded", embedded), daemon=True), + threading.Thread(target=accept_one, args=("explicit", explicit), daemon=True), + ] + for thread in threads: + thread.start() + try: + databricks.sql.connect( + server_hostname=f"127.0.0.1:{embedded_port}", + http_path=_P, + use_kernel=True, + access_token="dapi-xyz", + enable_telemetry=False, + _retry_stop_after_attempts_count=1, + _port=explicit_port, + ).close() + except Exception: + pass # expected: nothing behind the listener completes TLS + for thread in threads: + thread.join(timeout=70) + assert received.get("embedded") == b"\x16", received + assert "explicit" not in received, received + assert "explicit_error" in received, received