From b47e3b585c9fea1a082d26b7fa9071e2b1c09bc9 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Thu, 4 Jun 2026 05:14:36 +0000 Subject: [PATCH 1/3] feat(kernel): surface kernel logs through Python logging on use_kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the kernel backend loads, auto-initialize the kernel's tracing -> Python logging bridge so `use_kernel=True` users see kernel logs with no extra setup. Kernel logs land under the `databricks.sql.kernel` logger (a child of the connector's `databricks.sql.*` namespace), so an existing `logging.getLogger("databricks.sql").setLevel(...)` cascades to them. - `_errors.py` calls `databricks_sql_kernel.init_logging()` once at extension load (it's the canonical kernel-import site). The call is `getattr`-guarded so an older kernel wheel without the function still works — just without kernel logs. - e2e tests assert kernel records reach the `databricks.sql.kernel` logger (and the pyo3 boundary under `databricks.sql.kernel.pyo3`) and that the level set on the logger is respected. Creds-gated per the existing kernel e2e convention. Requires the companion kernel/pyo3 change (databricks-sql-kernel#120) that exposes `init_logging()`. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- src/databricks/sql/backend/kernel/_errors.py | 11 ++++ tests/e2e/test_kernel_backend.py | 67 ++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index 78b542300..f294ca998 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -57,6 +57,17 @@ "(into the same venv as databricks-sql-connector)." ) from exc +# Route the kernel's Rust-side logs into Python's ``logging`` as soon as +# the extension loads. The kernel emits under the ``databricks.sql.kernel`` +# logger (a child of the connector's ``databricks.sql`` namespace), so a +# customer who configures ``databricks.sql`` logging gets kernel logs for +# free with no extra setup. ``init_logging`` is idempotent on the Rust +# side; ``getattr`` guards against an older kernel wheel that predates the +# function so ``use_kernel=True`` still works (just without kernel logs). +_kernel_init_logging = getattr(_kernel, "init_logging", None) +if _kernel_init_logging is not None: + _kernel_init_logging() + # Map a kernel `code` slug to the PEP 249 exception class that best # captures it. The match isn't a perfect 1:1 — PEP 249 has a diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index 1e61bd7b8..b77c4ac2b 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -127,6 +127,73 @@ def test_fetchall_arrow(conn): assert table.column_names == ["a", "b"] +# ─── Logging (Rust kernel -> Python logging bridge) ────────────────────────── +# +# Layer 3 of the logger-name drift guard (see also the Rust tests +# `klog::tests::klog_emits_contract_target` and +# `logging::tests::kernel_target_matches_contract` in the kernel repo). +# Asserts the *customer-facing* contract end-to-end: kernel logs reach +# Python `logging` under the `databricks.sql.kernel` logger, respect the +# level set on it, and the pyo3 boundary surfaces under +# `databricks.sql.kernel.pyo3`. If the kernel's tracing target or the +# pyo3-log wiring ever drifts, these fail. + +import logging + + +def test_kernel_logs_reach_python_logging(kernel_conn_params, caplog): + """A query at DEBUG produces records on the `databricks.sql.kernel` + logger — proving the tracing -> log -> pyo3-log -> logging chain.""" + with caplog.at_level(logging.DEBUG, logger="databricks.sql.kernel"): + c = sql.connect(**kernel_conn_params) + try: + with c.cursor() as cur: + cur.execute("SELECT 1 AS a") + cur.fetchall() + finally: + c.close() + + kernel_records = [ + r for r in caplog.records if r.name.startswith("databricks.sql.kernel") + ] + assert kernel_records, ( + "expected log records under the 'databricks.sql.kernel' logger; " + "the kernel tracing -> Python logging bridge did not deliver any" + ) + # The core kernel logger (not just the .pyo3 child) must be present: + assert any( + r.name == "databricks.sql.kernel" for r in kernel_records + ), "expected core kernel records on the exact 'databricks.sql.kernel' logger" + # The pyo3 boundary breadcrumb must surface under the .pyo3 child: + assert any( + r.name == "databricks.sql.kernel.pyo3" for r in kernel_records + ), "expected pyo3-boundary records on 'databricks.sql.kernel.pyo3'" + + +def test_kernel_log_level_is_respected(kernel_conn_params, caplog): + """At WARNING, the chatty DEBUG kernel records are filtered out + before reaching Python — proving level control works (and that + filtering happens, not that everything is forwarded unconditionally).""" + with caplog.at_level(logging.WARNING, logger="databricks.sql.kernel"): + c = sql.connect(**kernel_conn_params) + try: + with c.cursor() as cur: + cur.execute("SELECT 1 AS a") + cur.fetchall() + finally: + c.close() + + debug_records = [ + r + for r in caplog.records + if r.name.startswith("databricks.sql.kernel") and r.levelno < logging.WARNING + ] + assert not debug_records, ( + "DEBUG/INFO kernel records leaked through at WARNING level: " + f"{[(r.name, r.levelname, r.getMessage()) for r in debug_records]}" + ) + + # ── Metadata ────────────────────────────────────────────────────── From ca9234bc4cc448d9556ad0112b07b512ef9318b4 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Thu, 4 Jun 2026 05:33:16 +0000 Subject: [PATCH 2/3] fix(kernel): harden log-bridge init + address review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback from PR #824 (gopalldb): - P1: guard the init_logging() call against throwing, not just a missing function. A panic across the PyO3 boundary surfaces as pyo3_runtime.PanicException (a BaseException, not Exception), so a bare call could escape module import and fail every use_kernel=True connection over a non-essential logging feature. Wrap in try/except BaseException, log a debug breadcrumb, continue. This also neutralizes the idempotency concern regardless of the Rust impl. - P2: soften the level-control test docstring to make clear it asserts the effective customer-visible outcome (sub-threshold records don't surface), not source-side suppression — caplog filters after the FFI. - P2: downgrade the databricks.sql.kernel.pyo3 assertion to a soft warning so a benign kernel change to the boundary breadcrumb target doesn't break the connector e2e suite. The core databricks.sql.kernel contract is still hard-asserted. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- src/databricks/sql/backend/kernel/_errors.py | 24 +++++++++++--- tests/e2e/test_kernel_backend.py | 34 +++++++++++++++----- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/databricks/sql/backend/kernel/_errors.py b/src/databricks/sql/backend/kernel/_errors.py index f294ca998..dee287e7d 100644 --- a/src/databricks/sql/backend/kernel/_errors.py +++ b/src/databricks/sql/backend/kernel/_errors.py @@ -38,6 +38,8 @@ from __future__ import annotations +import logging + from databricks.sql.exc import ( DatabaseError, Error, @@ -61,12 +63,26 @@ # the extension loads. The kernel emits under the ``databricks.sql.kernel`` # logger (a child of the connector's ``databricks.sql`` namespace), so a # customer who configures ``databricks.sql`` logging gets kernel logs for -# free with no extra setup. ``init_logging`` is idempotent on the Rust -# side; ``getattr`` guards against an older kernel wheel that predates the -# function so ``use_kernel=True`` still works (just without kernel logs). +# free with no extra setup. +# +# This is a best-effort, non-essential feature: it must never take down +# ``use_kernel=True`` for a process. ``getattr`` guards against an older +# kernel wheel that predates the function. The ``try`` guards against the +# call itself throwing — note ``except BaseException`` is deliberate: a +# panic raised across the PyO3 boundary surfaces as +# ``pyo3_runtime.PanicException``, which derives from ``BaseException`` +# (not ``Exception``), so a narrower clause would let it escape module +# import and fail every kernel-backed connection. The kernel side is +# idempotent and returns rather than panics on a double install, but we +# do not rely on that here — the guard holds regardless of the Rust impl. _kernel_init_logging = getattr(_kernel, "init_logging", None) if _kernel_init_logging is not None: - _kernel_init_logging() + try: + _kernel_init_logging() + except BaseException as exc: # noqa: BLE001 - see comment above re: PanicException + logging.getLogger(__name__).debug( + "kernel log bridge init failed; continuing without it: %r", exc + ) # Map a kernel `code` slug to the PEP 249 exception class that best diff --git a/tests/e2e/test_kernel_backend.py b/tests/e2e/test_kernel_backend.py index b77c4ac2b..69b96f1c5 100644 --- a/tests/e2e/test_kernel_backend.py +++ b/tests/e2e/test_kernel_backend.py @@ -160,20 +160,38 @@ def test_kernel_logs_reach_python_logging(kernel_conn_params, caplog): "expected log records under the 'databricks.sql.kernel' logger; " "the kernel tracing -> Python logging bridge did not deliver any" ) - # The core kernel logger (not just the .pyo3 child) must be present: + # The core kernel logger (not just any child) must be present — this + # is the customer-facing contract. assert any( r.name == "databricks.sql.kernel" for r in kernel_records ), "expected core kernel records on the exact 'databricks.sql.kernel' logger" - # The pyo3 boundary breadcrumb must surface under the .pyo3 child: - assert any( - r.name == "databricks.sql.kernel.pyo3" for r in kernel_records - ), "expected pyo3-boundary records on 'databricks.sql.kernel.pyo3'" + # The pyo3-boundary breadcrumb (`databricks.sql.kernel.pyo3`) is a + # nice-to-have, but the exact sub-target is a kernel-internal naming + # detail — assert softly so a benign kernel change to the boundary + # breadcrumbs doesn't break the connector e2e suite. + if not any(r.name == "databricks.sql.kernel.pyo3" for r in kernel_records): + import warnings + + warnings.warn( + "no 'databricks.sql.kernel.pyo3' boundary records seen; " + "the kernel may have changed its pyo3 breadcrumb target", + stacklevel=2, + ) def test_kernel_log_level_is_respected(kernel_conn_params, caplog): - """At WARNING, the chatty DEBUG kernel records are filtered out - before reaching Python — proving level control works (and that - filtering happens, not that everything is forwarded unconditionally).""" + """At WARNING on the kernel logger, no DEBUG/INFO kernel records reach + `caplog.records` — i.e. level control on `databricks.sql.kernel` + behaves like any other Python logger. + + Scope note: `caplog.at_level` sets the logger's level and attaches a + handler, so this asserts the *effective* outcome a customer sees + (sub-threshold records don't surface), not specifically that the Rust + side suppressed them at source. A DEBUG record that crossed the FFI + would still be dropped by Python's level check before reaching + `caplog`. Source-side suppression (and its per-record FFI cost + avoidance) is covered by the kernel-side filtering, not asserted + here.""" with caplog.at_level(logging.WARNING, logger="databricks.sql.kernel"): c = sql.connect(**kernel_conn_params) try: From 66262bc9231a2d3215232a13e358bf711a55f4ee Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Fri, 5 Jun 2026 06:27:33 +0000 Subject: [PATCH 3/3] chore: bump KERNEL_REV to merged kernel main (f4ee6fe) for the logging bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel tracing -> Python logging bridge (init_logging / pyo3-log, routing under databricks.sql.kernel) landed in kernel #120 — AFTER the previously-pinned 101aa46 (#118). The kernel-e2e built a wheel without the bridge, so test_kernel_logs_reach_python_logging failed with 'no records delivered' (assert []). Bump to f4ee6fe (current kernel main: includes #118, #120, #123, #125). Verified live against a wheel built from f4ee6fe: the bridge delivers records to the databricks.sql.kernel logger, and both test_kernel_logs_reach_python_logging and test_kernel_log_level_is_respected pass. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index af059324d..696572aef 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -101aa465e71991eec98102bba77aad2f7ad8faed +f4ee6fec78aabce8c0ea9c1ff47fc11b8191d013