diff --git a/.github/workflows/kernel-e2e.yml b/.github/workflows/kernel-e2e.yml index 40cbd223..becb84d8 100644 --- a/.github/workflows/kernel-e2e.yml +++ b/.github/workflows/kernel-e2e.yml @@ -160,7 +160,7 @@ jobs: # Run when the kernel driver layer, the napi binding contract, kernel # e2e tests, this workflow, the kernel revision pin, or core deps # move. - if echo "$CHANGED" | grep -qE "^(lib/kernel/|lib/result/|native/kernel/|tests/e2e/kernel/|tests/e2e/utils/rowMaterialization\.ts|tests/unit/kernel/|\.github/workflows/kernel-e2e\.yml|KERNEL_REV|package\.json|package-lock\.json)"; then + if echo "$CHANGED" | grep -qE "^(lib/kernel/|lib/result/|native/kernel/|tests/e2e/kernel/|tests/e2e/utils/rowMaterialization\.ts|tests/e2e/utils/stagingTests\.ts|tests/unit/kernel/|\.github/workflows/kernel-e2e\.yml|KERNEL_REV|package\.json|package-lock\.json)"; then echo "run_tests=true" >> "$GITHUB_OUTPUT" else echo "run_tests=false" >> "$GITHUB_OUTPUT" diff --git a/CHANGELOG.md b/CHANGELOG.md index f8aeb947..cc2016de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) - Honor `cloudFetchConcurrentDownloads` when opening kernel sessions, with a default window of 16 and explicit chunk-setting overrides. Values other than positive safe integers raise `HiveDriverError` at session open. - Reduce result chunking and `fetchAll()` overhead for large query results. +- Support Volume file transfers on the kernel backend when `stagingAllowedLocalPath` is supplied. Symlink targets must remain within configured allowed roots; in-root symlinks are permitted. (#613) - Thrift CloudFetch downloads are now bounded end to end by `retriesTimeout` (default 15 minutes), including every attempt and retry backoff. Previously a stalled download, where the response headers or body never arrived, blocked the fetch forever. A download still running when the deadline expires is aborted and fails with `RetryError` (`TIMEOUT_EXCEEDED`, message `CloudFetch result download failed: Retry timeout exceeded`). (#520) - Thrift connections made through `proxy` now apply `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey`. Previously the proxied path ignored them and verified against Node's process-wide trust store, so a certificate outside the configured trust set could be accepted. **Compatibility:** proxied connections now verify unless `checkServerCertificate: false` is set; as on direct connections, `NODE_TLS_REJECT_UNAUTHORIZED=0` no longer turns verification off. For an `https` proxy these options also apply to the proxy's own certificate. (#485) - Honor the client `useCloudFetch: false` default on the kernel backend so complete results can be fetched without cloud-storage downloads. (#574) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index edba6002..46eab303 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -76,17 +76,17 @@ Per-statement `useCloudFetch` overrides remain unsupported on the kernel backend ## HTTP client, proxy, and retries -| Option | Type | Thrift | Kernel | Default Value | Note | -| --------------------------- | ------------------------ | :----: | :----: | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel forwards only positive values. `0` means no timeout on Thrift but is omitted on Kernel, which then keeps its 120-second default. | -| `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. | -| `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. | -| `customHeaders` | `Record` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. | -| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. | -| `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. | -| `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. | -| `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. | -| `maxConnections` (internal) | `number` | ❌ | ✅ | — / Kernel default | Kernel connection-pool size. Must be a positive integer within the napi `u32` range. | +| Option | Type | Thrift | Kernel | Default Value | Note | +| --------------------------- | ------------------------ | :----: | :----: | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel uses a total SQL request deadline and a storage idle timeout (15 minutes by default). Positive values set both; `0` keeps the SQL default and disables storage inactivity. Storage idle time bounds reads, upload progress and PUT response waits; redirected stream PUT resends have no idle bound. | +| `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. | +| `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. | +| `customHeaders` | `Record` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. | +| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. | +| `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. | +| `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. | +| `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. | +| `maxConnections` (internal) | `number` | ❌ | ✅ | — / Kernel default | Kernel connection-pool size. Must be a positive integer within the napi `u32` range. | ## TLS / SSL @@ -213,7 +213,7 @@ telemetry implementation, so only the options explicitly forwarded by | `maxRows` | `number \| bigint \| Int64 \| null` | ✅ | ❌ | `100000` / — | Thrift uses it for the initial Direct Results row count; `null` disables Direct Results. Kernel does not retain or forward the supplied value. | | `useCloudFetch` | `boolean` | ✅ | ❌ | `true` / — | Kernel logs and ignores this per-statement hint; Kernel result fetching owns its CloudFetch behavior. | | `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift uses it when supported and when the result is not CloudFetch. Kernel owns and auto-detects result compression. | -| `stagingAllowedLocalPath` | `string \| string[]` | ✅ | ❌ | — | Local allowlist for Thrift volume/staging operations. Kernel volume operations are unsupported. | +| `stagingAllowedLocalPath` | `string \| string[]` | ✅ | ✅ | — | Local allowlist for Volume operations. Kernel also requires resolved symlink targets within allowed roots; links within those roots are permitted. | | `namedParameters` | `Record` | ✅ | ✅ | — | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries. | | `ordinalParameters` | `Array` | ✅ | ✅ | — | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`. | | `queryTags` | `Record` | ✅ | ✅ | — | Serialized into the per-statement `query_tags` conf overlay on both. | @@ -244,8 +244,7 @@ session parameter surface. Thrift-only capabilities include custom/token-provider authentication, custom OAuth persistence, SOCKS proxies, authenticated/driver-buffer telemetry knobs, -query timeout, Direct Results `maxRows`, per-statement CloudFetch/LZ4 hints, and -volume operations. +query timeout, Direct Results `maxRows`, and per-statement CloudFetch/LZ4 hints. Kernel-only capabilities include connection-pool sizing, proxy bypass hosts, independent TLS hostname verification, optional U2M disk caching, and diff --git a/KERNEL_REV b/KERNEL_REV index 18337b65..6af4073b 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -dac32e2bf44468bdab06747dad4f289d1c0a18bd +c2ddddc61bd41d9529dd19f36546d603ba21150d diff --git a/lib/DBSQLSession.ts b/lib/DBSQLSession.ts index 5bca61d6..724b4494 100644 --- a/lib/DBSQLSession.ts +++ b/lib/DBSQLSession.ts @@ -108,12 +108,17 @@ export default class DBSQLSession implements IDBSQLSession { // If `stagingAllowedLocalPath` is not provided - don't do anything to the operation. In a case of regular // operation, everything will work as usual. In a case of staging operation, it will be processed like any // other query - it will be possible to get data from it as usual, or use other operation methods. + // Backends with explicit staging completion handle it before metadata is fetched. if (options.stagingAllowedLocalPath !== undefined) { + const allowedLocalPath = Array.isArray(options.stagingAllowedLocalPath) + ? options.stagingAllowedLocalPath + : [options.stagingAllowedLocalPath]; + if (opBackend.completeStagingOperation) { + await this.runBackend(() => opBackend.completeStagingOperation!(allowedLocalPath)); + return operation; + } const metadata = await operation.getResultMetadata(); if (metadata.isStagingOperation) { - const allowedLocalPath = Array.isArray(options.stagingAllowedLocalPath) - ? options.stagingAllowedLocalPath - : [options.stagingAllowedLocalPath]; return this.handleStagingOperation(operation, allowedLocalPath, metadata); } } diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index 12d7d6c2..da5a01b1 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -32,7 +32,8 @@ export type ExecuteStatementOptions = { * `false`/unset (default) runs the blocking direct-results path (faster, * cancellable mid-compute); `true` submits and polls (returns a pending * handle before completion). Default is sync, matching the python - * connector's `cursor.execute()`. + * connector's `cursor.execute()`. With `stagingAllowedLocalPath` configured, + * execution waits for staging completion regardless of this option. */ runAsync?: boolean; maxRows?: number | bigint | Int64 | null; diff --git a/lib/contracts/IOperationBackend.ts b/lib/contracts/IOperationBackend.ts index 109de87e..db8240af 100644 --- a/lib/contracts/IOperationBackend.ts +++ b/lib/contracts/IOperationBackend.ts @@ -113,6 +113,9 @@ export default interface IOperationBackend { */ getCachedResultMetadata?(): ResultMetadata | undefined; + /** Complete configured staging transfers before returning the operation. */ + completeStagingOperation?(allowedPaths: string[]): Promise; + /** Cancel the operation. */ cancel(): Promise; diff --git a/lib/kernel/KernelAuth.ts b/lib/kernel/KernelAuth.ts index 8d900485..218ee8d9 100644 --- a/lib/kernel/KernelAuth.ts +++ b/lib/kernel/KernelAuth.ts @@ -523,14 +523,8 @@ export function buildKernelHttpOptions(options: ConnectionOptions): KernelHttpOp headers.push({ name: 'User-Agent', value: buildUserAgentString(userAgentEntry) }); const http: KernelHttpOptions = { customHeaders: headers }; - // Per-connection socket read timeout (ms). The public `socketTimeout` - // ConnectionOption maps onto the kernel napi `socketTimeoutMs` - // (kernel `HttpConfig::request_timeout` / reqwest `Client::timeout`). - // Only forward a POSITIVE value: `socketTimeout: 0` means "disabled / wait - // indefinitely" on the Thrift path, but forwarding `0` would make reqwest - // time out immediately, so we omit it and let the kernel keep its (large) - // default — preserving the "effectively no idle timeout" semantics. - if (typeof socketTimeout === 'number' && socketTimeout > 0) { + // Preserve zero so native storage inactivity can be disabled like Thrift. + if (typeof socketTimeout === 'number' && socketTimeout >= 0) { http.socketTimeoutMs = socketTimeout; } return http; diff --git a/lib/kernel/KernelErrorMapping.ts b/lib/kernel/KernelErrorMapping.ts index d83d7e59..d54333bb 100644 --- a/lib/kernel/KernelErrorMapping.ts +++ b/lib/kernel/KernelErrorMapping.ts @@ -1,7 +1,19 @@ +import { constants } from 'node:os'; +import { getSystemErrorName } from 'node:util'; import HiveDriverError from '../errors/HiveDriverError'; import AuthenticationError from '../errors/AuthenticationError'; import OperationStateError, { OperationStateErrorCode } from '../errors/OperationStateError'; import ParameterError from '../errors/ParameterError'; +import StagingError from '../errors/StagingError'; + +const kindCodes: Record = { + NotFound: 'ENOENT', + PermissionDenied: 'EACCES', + AlreadyExists: 'EEXIST', + NotADirectory: 'ENOTDIR', + IsADirectory: 'EISDIR', + DirectoryNotEmpty: 'ENOTEMPTY', +}; /** * Sentinel prefix the napi binding's `napi_err_from_kernel` puts on @@ -111,6 +123,20 @@ function defineErrorMetadata(error: Error, key: K, value: V }); } +export function toStagingTransferError(mapped: ErrorWithSqlState): ErrorWithSqlState { + const metadata = mapped.kernelMetadata; + if ( + (mapped instanceof ParameterError && !metadata?.httpStatus) || + metadata?.errorCode === 'VOLUME_STORAGE_HTTP_ERROR' + ) { + const staging = new StagingError(mapped.message); + if (metadata) defineErrorMetadata(staging, 'kernelMetadata', metadata); + if ('cause' in mapped) defineErrorMetadata(staging, 'cause', mapped.cause); + return staging; + } + return mapped; +} + /** * Map a kernel error (as surfaced by the napi-binding) to the appropriate JS * driver error class. @@ -311,7 +337,21 @@ export function decodeNapiKernelError(err: unknown): Error { const msg = envelope.message as string; const sqlState = typeof envelope.sqlState === 'string' ? envelope.sqlState : undefined; - const jsErr = mapKernelErrorToJsError({ code, message: msg, sqlstate: sqlState }); + let jsErr = mapKernelErrorToJsError({ code, message: msg, sqlstate: sqlState }); + if (typeof envelope.localPath === 'string' && typeof envelope.localErrorKind === 'string') { + const raw = + typeof envelope.localOsError === 'number' && Number.isSafeInteger(envelope.localOsError) + ? envelope.localOsError + : undefined; + const systemCode = process.platform !== 'win32' && raw ? getSystemErrorName(-Math.abs(raw)) : undefined; + const localCode = + systemCode && systemCode in constants.errno ? systemCode : kindCodes[envelope.localErrorKind] ?? 'EIO'; + jsErr = Object.assign(new Error(msg), { + code: localCode, + errno: -(constants.errno as Record)[localCode], + path: envelope.localPath, + }); + } const meta = buildKernelMetadata(envelope); // Skip the namespace attachment entirely when no fields validated diff --git a/lib/kernel/KernelOperationBackend.ts b/lib/kernel/KernelOperationBackend.ts index 81647ece..b2695053 100644 --- a/lib/kernel/KernelOperationBackend.ts +++ b/lib/kernel/KernelOperationBackend.ts @@ -46,12 +46,13 @@ import { LogLevel } from '../contracts/IDBSQLLogger'; import { RowMode } from '../contracts/IDBSQLSession'; import Status from '../dto/Status'; import HiveDriverError from '../errors/HiveDriverError'; +import StagingError from '../errors/StagingError'; import OperationStateError, { OperationStateErrorCode } from '../errors/OperationStateError'; import ArrowResultConverter from '../result/ArrowResultConverter'; import ResultSlicer from '../result/ResultSlicer'; import KernelResultsProvider from './KernelResultsProvider'; import { arrowSchemaToThriftSchema, decodeIpcSchema, patchIpcBytes } from './KernelArrowIpc'; -import { decodeNapiKernelError } from './KernelErrorMapping'; +import { decodeNapiKernelError, toStagingTransferError } from './KernelErrorMapping'; import { KernelStatement, KernelNativeAsyncStatement, @@ -84,7 +85,10 @@ export type KernelOperationStatement = KernelStatementHandle & Partial; +type KernelFetchHandle = Pick< + KernelStatement, + 'fetchNextBatch' | 'schema' | 'isVolumeOperation' | 'completeVolumeOperation' +>; /** * The rich operation-status surface the kernel exposes on a terminal sync @@ -220,6 +224,8 @@ export default class KernelOperationBackend implements IOperationBackend { // already-terminal statement. Drives both fetch and result-metadata. private fetchHandlePromise?: Promise; + private volumeCompleted = false; + // Async path only: set once the kernel reported the statement Succeeded (a // status poll, or `awaitResult()` resolving). The kernel exposes no // synchronous terminal-state accessor, so the observed success is retained. @@ -308,6 +314,22 @@ export default class KernelOperationBackend implements IOperationBackend { // Fetch / metadata (owned by the kernel-results pipeline). // --------------------------------------------------------------------------- + public async completeStagingOperation(allowedPaths: string[]): Promise { + failIfNotActive(this.lifecycle); + await this.waitUntilReady(); + const handle = await this.getFetchHandle(); + let outcome; + try { + outcome = await handle.completeVolumeOperation(allowedPaths, true); // Thrift REMOVE treats missing objects as success. + } catch (err) { + throw toStagingTransferError(decodeNapiKernelError(err)); + } + if (outcome === 'needs_input_stream') { + throw new StagingError('Volume PUT requires a local file path'); + } + this.volumeCompleted = outcome === 'completed'; + } + public async fetchChunk({ limit, disableBuffering, @@ -320,6 +342,7 @@ export default class KernelOperationBackend implements IOperationBackend { // Cancel-mid-fetch propagation: if cancel() has flipped the // lifecycle flag, fail locally without a wire round-trip. failIfNotActive(this.lifecycle); + if (this.volumeCompleted) return []; // Cooperative cancel (parity with ThriftOperationBackend): the facade // supplies `isClosed` and re-checks `failIfClosed()` after we return, so // bailing with `[]` at a yield point is the contract-correct way for a @@ -340,6 +363,7 @@ export default class KernelOperationBackend implements IOperationBackend { public async hasMore(): Promise { failIfNotActive(this.lifecycle); + if (this.volumeCompleted) return false; try { // `hasMore()` prefetches the next batch, so it fails like `fetchChunk()`. const slicer = await this.getResultSlicer(); @@ -383,7 +407,7 @@ export default class KernelOperationBackend implements IOperationBackend { // `arrowSchema` doesn't hit apache-arrow@13's `Unrecognized type // "Duration" (18)`. Matches what the per-batch fetch path already does. arrowSchema: patchIpcBytes(arrowSchemaIpc.ipcBytes), - isStagingOperation: false, + isStagingOperation: Boolean(handle.isVolumeOperation), }; this.metadata = meta; return meta; diff --git a/lib/kernel/KernelSessionBackend.ts b/lib/kernel/KernelSessionBackend.ts index dd55dd4c..c6cab33a 100644 --- a/lib/kernel/KernelSessionBackend.ts +++ b/lib/kernel/KernelSessionBackend.ts @@ -148,10 +148,9 @@ export default class KernelSessionBackend implements ISessionBackend { * `query_tags` key (the same wire shape Thrift's `serializeQueryTags` * produces), merged with any explicit `statementConf`. * - * Still rejected (genuinely unsupported on kernel, rather than silently - * dropped): `useCloudFetch` (governed by the kernel `ResultConfig`, not a - * per-statement knob), `useLZ4Compression` (kernel owns result compression), - * and `stagingAllowedLocalPath` (volume operations). `maxRows` is applied by + * `useCloudFetch` and `useLZ4Compression` are logged and ignored because the + * kernel owns result fetching and compression. `stagingAllowedLocalPath` is + * applied by the facade. `maxRows` is applied by * the facade at fetch time, so it is intentionally not handled here. */ public async executeStatement(statement: string, options: ExecuteStatementOptions): Promise { @@ -160,8 +159,7 @@ export default class KernelSessionBackend implements ISessionBackend { // Per-statement options the kernel backend doesn't honour are treated as // NO-OPs (logged at warn), not errors — so call sites written for the Thrift // backend are drop-in on the kernel path. These are perf/format hints the - // kernel governs internally (useCloudFetch / useLZ4Compression) or a feature - // it doesn't expose yet (staging); ignoring them cannot change query results. + // kernel governs internally (useCloudFetch / useLZ4Compression). // NB: parameter binding (compound/BINARY) is deliberately NOT no-op'd — a // dropped param would silently change results, so it still throws. if (options.useCloudFetch !== undefined) { @@ -180,15 +178,6 @@ export default class KernelSessionBackend implements ISessionBackend { 'kernel executeStatement: ignoring per-statement `useLZ4Compression` — result compression is governed by the kernel (no-op on kernel).', ); } - if (options.stagingAllowedLocalPath !== undefined) { - this.context - .getLogger() - .log( - LogLevel.warn, - 'kernel executeStatement: ignoring `stagingAllowedLocalPath` — volume/staging operations are not yet supported on the kernel backend (no-op).', - ); - } - // `runAsync` selects the kernel execution path. NOTE: this is a kernel- // specific use of the option — the Thrift backend hardcodes `runAsync: true` // on the wire and never reads `options.runAsync`, so the field is a no-op diff --git a/native/kernel/index.d.ts b/native/kernel/index.d.ts index 87640ca0..737fb431 100644 --- a/native/kernel/index.d.ts +++ b/native/kernel/index.d.ts @@ -18,6 +18,9 @@ * post-close contract. */ export declare class AsyncResultHandle { + get isVolumeOperation(): boolean + /** Complete a server-marked file transfer while retaining its instruction rows. */ + completeVolumeOperation(allowedPaths: Array, removeNotFoundIsSuccess: boolean): Promise /** * Server-issued statement id. Cached at construction; readable * for log correlation. Matches the parent `AsyncStatement`'s @@ -149,9 +152,8 @@ export declare class AsyncStatement { * re-attempting the wire call. The kernel's own `Drop` * fire-and-forget retry runs once in the background. * - * Also invalidates the handles `awaitResult()` returned: their - * `fetchNextBatch()` then rejects with `InvalidStatementHandle`, - * unless a fetch was in progress when `close()` ran. + * Also invalidates the handles `awaitResult()` returned. An operation + * already in flight may finish before its handle is cleared. */ close(): Promise } @@ -438,6 +440,9 @@ export declare class Connection { * disposed statement can still read them. */ export declare class Statement { + get isVolumeOperation(): boolean + /** Complete a server-marked file transfer while retaining its instruction rows. */ + completeVolumeOperation(allowedPaths: Array, removeNotFoundIsSuccess: boolean): Promise /** * Server-issued statement id. Cached at construction; readable * even after `close()` so JS-side log lines can correlate against @@ -1054,15 +1059,11 @@ export interface ConnectionOptions { */ proxy?: ProxyInput /** - * Per-connection socket read timeout, in milliseconds. Caps how - * long a single HTTP round-trip may block waiting on the server - * before the request errors out. Maps onto the kernel - * [`HttpConfig::request_timeout`] (the internal reqwest - * `Client::timeout`). - * - * Omitted ⇒ kernel default (120 000 ms / 120 s). Napi-rs - * serialises `u32` as JS `number`; the largest representable value - * (~49.7 days) far exceeds any sensible socket timeout. + * SQL request deadline and Volume storage idle timeout, in milliseconds. + * Storage idle time bounds reads, upload progress and the PUT response wait; + * a redirected stream PUT resend has no idle bound. + * Omitted: SQL uses the kernel default; storage uses 15 minutes. + * Zero disables storage inactivity and keeps the SQL default. */ socketTimeoutMs?: number } @@ -1325,3 +1326,9 @@ export interface TypedValueInput { * loaded?" probe for the JS-side loader's structured diagnostics. */ export declare function version(): string + +export declare const enum VolumeCompletion { + NotVolume = 'not_volume', + NeedsInputStream = 'needs_input_stream', + Completed = 'completed' +} diff --git a/native/kernel/index.js b/native/kernel/index.js index 6902ddee..511f20a1 100644 --- a/native/kernel/index.js +++ b/native/kernel/index.js @@ -711,3 +711,4 @@ module.exports.kernelLoggingStats = nativeBinding.kernelLoggingStats module.exports.openSession = nativeBinding.openSession module.exports.setKernelLogLevel = nativeBinding.setKernelLogLevel module.exports.version = nativeBinding.version +module.exports.VolumeCompletion = nativeBinding.VolumeCompletion diff --git a/tests/e2e/kernel/staging-close-e2e.test.ts b/tests/e2e/kernel/staging-close-e2e.test.ts new file mode 100644 index 00000000..fb592797 --- /dev/null +++ b/tests/e2e/kernel/staging-close-e2e.test.ts @@ -0,0 +1,61 @@ +import { expect } from 'chai'; +import { createServer } from 'node:http'; +import { AddressInfo } from 'node:net'; +import { openSession } from '../../../native/kernel'; +import DBSQLClient from '../../../lib/DBSQLClient'; +import DBSQLSession from '../../../lib/DBSQLSession'; +import KernelSessionBackend from '../../../lib/kernel/KernelSessionBackend'; + +describe('native staging close', () => { + it('closes a session during a pending staged statement with default options', async function () { + this.timeout(10000); + let polled!: () => void; + const polling = new Promise((resolve) => { + polled = resolve; + }); + const server = createServer((request, response) => { + response.setHeader('content-type', 'application/json'); + if (request.method === 'POST' && request.url?.endsWith('/sessions')) { + response.end(JSON.stringify({ session_id: 'session', warehouse_id: 'wh' })); + } else if (request.url?.startsWith('/api/2.0/sql/statements')) { + response.end(JSON.stringify({ statement_id: 'pending', status: { state: 'RUNNING' } })); + if (request.method === 'GET') polled(); + } else { + response.end('{}'); + } + }); + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + const context = new DBSQLClient({ logger: { log() {} } }); + const connection = await openSession({ + hostName: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + httpPath: '/sql/1.0/warehouses/wh', + token: 'test-token', + telemetryEnabled: false, + }); + const session = new DBSQLSession({ + backend: new KernelSessionBackend({ connection, context, id: connection.sessionId }), + context, + }); + try { + const executing = session.executeStatement('REMOVE remote', { stagingAllowedLocalPath: [] }).then( + () => { + throw new Error('pending statement completed'); + }, + (error) => error, + ); + await polling; + const started = Date.now(); + await session.close(); + expect(Date.now() - started).to.be.lessThan(1000); + expect(await executing).to.be.instanceOf(Error); + } finally { + await session.close(); + server.closeAllConnections(); + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + }); +}); diff --git a/tests/e2e/kernel/staging-e2e.test.ts b/tests/e2e/kernel/staging-e2e.test.ts new file mode 100644 index 00000000..da885ee7 --- /dev/null +++ b/tests/e2e/kernel/staging-e2e.test.ts @@ -0,0 +1,18 @@ +import stagingTests from '../utils/stagingTests'; +import config from '../utils/config'; + +const kernelDescribe = + process.env.DATABRICKS_PECOTESTING_SERVER_HOSTNAME && + process.env.DATABRICKS_PECOTESTING_HTTP_PATH && + process.env.DATABRICKS_PECOTESTING_TOKEN_PERSONAL + ? describe + : describe.skip; + +kernelDescribe('Kernel Staging Test', () => { + stagingTests(() => ({ + host: config.host, + path: config.path, + token: config.token, + useKernel: true, + })); +}); diff --git a/tests/e2e/staging_ingestion.test.ts b/tests/e2e/staging_ingestion.test.ts index 7721dc75..bc739ef5 100644 --- a/tests/e2e/staging_ingestion.test.ts +++ b/tests/e2e/staging_ingestion.test.ts @@ -1,135 +1,10 @@ -import { expect } from 'chai'; -import fs from 'fs'; -import path from 'path'; -import os from 'os'; -import * as uuid from 'uuid'; -import { DBSQLClient } from '../../lib'; -import StagingError from '../../lib/errors/StagingError'; - +import stagingTests from './utils/stagingTests'; import config from './utils/config'; describe('Staging Test', () => { - const localPath = fs.mkdtempSync(path.join(os.tmpdir(), 'databricks-sql-tests-')); - - after(() => { - fs.rmSync(localPath, { - recursive: true, - force: true, - }); - }); - - it('put staging data and receive it', async () => { - const { catalog, schema, volume } = config; - - const client = new DBSQLClient(); - await client.connect({ - host: config.host, - path: config.path, - token: config.token, - }); - - const session = await client.openSession({ - initialCatalog: catalog, - initialSchema: schema, - }); - - const expectedData = 'Hello World!'; - const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; - const localFile = path.join(localPath, `${uuid.v4()}.csv`); - - fs.writeFileSync(localFile, expectedData); - await session.executeStatement(`PUT '${localFile}' INTO '${stagingFileName}' OVERWRITE`, { - stagingAllowedLocalPath: [localPath], - }); - fs.rmSync(localFile); - - await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, { - stagingAllowedLocalPath: [localPath], - }); - const result = fs.readFileSync(localFile); - fs.rmSync(localFile); - expect(result.toString() === expectedData).to.be.true; - }); - - it('put staging data and remove it', async () => { - const { catalog, schema, volume } = config; - - const client = new DBSQLClient(); - await client.connect({ - host: config.host, - path: config.path, - token: config.token, - }); - - const session = await client.openSession({ - initialCatalog: catalog, - initialSchema: schema, - }); - - const expectedData = 'Hello World!'; - const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; - const localFile = path.join(localPath, `${uuid.v4()}.csv`); - - fs.writeFileSync(localFile, expectedData); - await session.executeStatement(`PUT '${localFile}' INTO '${stagingFileName}' OVERWRITE`, { - stagingAllowedLocalPath: [localPath], - }); - fs.rmSync(localFile); - - await session.executeStatement(`REMOVE '${stagingFileName}'`, { stagingAllowedLocalPath: [localPath] }); - - try { - await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, { - stagingAllowedLocalPath: [localPath], - }); - expect.fail('It should throw HTTP 404 error'); - } catch (error) { - if (error instanceof StagingError) { - // File should not exist after deleting - expect(error.message).to.contain('404'); - } else { - throw error; - } - } finally { - fs.rmSync(localFile, { force: true }); - } - }); - - it('delete non-existent data', async () => { - const { catalog, schema, volume } = config; - - const client = new DBSQLClient(); - await client.connect({ - host: config.host, - path: config.path, - token: config.token, - }); - - const session = await client.openSession({ - initialCatalog: catalog, - initialSchema: schema, - }); - - const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; - const localFile = path.join(localPath, `${uuid.v4()}.csv`); - - // File should not exist before removing - try { - await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, { - stagingAllowedLocalPath: [localPath], - }); - expect.fail('It should throw HTTP 404 error'); - } catch (error) { - if (error instanceof StagingError) { - expect(error.message).to.contain('404'); - } else { - throw error; - } - } finally { - fs.rmSync(localFile, { force: true }); - } - - // Try to remove the file - it should succeed and not throw any errors - await session.executeStatement(`REMOVE '${stagingFileName}'`, { stagingAllowedLocalPath: [localPath] }); - }); + stagingTests(() => ({ + host: config.host, + path: config.path, + token: config.token, + })); }); diff --git a/tests/e2e/utils/stagingTests.ts b/tests/e2e/utils/stagingTests.ts new file mode 100644 index 00000000..ccb503ed --- /dev/null +++ b/tests/e2e/utils/stagingTests.ts @@ -0,0 +1,124 @@ +import { expect } from 'chai'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import * as uuid from 'uuid'; +import { DBSQLClient } from '../../../lib'; +import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; +import StagingError from '../../../lib/errors/StagingError'; + +import config from './config'; + +export default function stagingTests(connectOptions: () => ConnectionOptions) { + const localPath = fs.mkdtempSync(path.join(os.tmpdir(), 'databricks-sql-tests-')); + + after(() => { + fs.rmSync(localPath, { + recursive: true, + force: true, + }); + }); + + it('put staging data and receive it', async () => { + const { catalog, schema, volume } = config; + + const client = new DBSQLClient(); + await client.connect(connectOptions()); + + const session = await client.openSession({ + initialCatalog: catalog, + initialSchema: schema, + }); + + const expectedData = 'Hello World!'; + const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; + const localFile = path.join(localPath, `${uuid.v4()}.csv`); + + fs.writeFileSync(localFile, expectedData); + await session.executeStatement(`PUT '${localFile}' INTO '${stagingFileName}' OVERWRITE`, { + stagingAllowedLocalPath: [localPath], + }); + fs.rmSync(localFile); + + await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, { + stagingAllowedLocalPath: [localPath], + }); + const result = fs.readFileSync(localFile); + fs.rmSync(localFile); + expect(result.toString()).to.equal(expectedData); + }); + + it('put staging data and remove it', async () => { + const { catalog, schema, volume } = config; + + const client = new DBSQLClient(); + await client.connect(connectOptions()); + + const session = await client.openSession({ + initialCatalog: catalog, + initialSchema: schema, + }); + + const expectedData = 'Hello World!'; + const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; + const localFile = path.join(localPath, `${uuid.v4()}.csv`); + + fs.writeFileSync(localFile, expectedData); + await session.executeStatement(`PUT '${localFile}' INTO '${stagingFileName}' OVERWRITE`, { + stagingAllowedLocalPath: [localPath], + }); + fs.rmSync(localFile); + + await session.executeStatement(`REMOVE '${stagingFileName}'`, { stagingAllowedLocalPath: [localPath] }); + + try { + await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, { + stagingAllowedLocalPath: [localPath], + }); + expect.fail('It should throw HTTP 404 error'); + } catch (error) { + if (error instanceof StagingError) { + // File should not exist after deleting + expect(error.message).to.contain('404'); + } else { + throw error; + } + } finally { + fs.rmSync(localFile, { force: true }); + } + }); + + it('delete non-existent data', async () => { + const { catalog, schema, volume } = config; + + const client = new DBSQLClient(); + await client.connect(connectOptions()); + + const session = await client.openSession({ + initialCatalog: catalog, + initialSchema: schema, + }); + + const stagingFileName = `/Volumes/${catalog}/${schema}/${volume}/${uuid.v4()}.csv`; + const localFile = path.join(localPath, `${uuid.v4()}.csv`); + + // File should not exist before removing + try { + await session.executeStatement(`GET '${stagingFileName}' TO '${localFile}'`, { + stagingAllowedLocalPath: [localPath], + }); + expect.fail('It should throw HTTP 404 error'); + } catch (error) { + if (error instanceof StagingError) { + expect(error.message).to.contain('404'); + } else { + throw error; + } + } finally { + fs.rmSync(localFile, { force: true }); + } + + // Try to remove the file - it should succeed and not throw any errors + await session.executeStatement(`REMOVE '${stagingFileName}'`, { stagingAllowedLocalPath: [localPath] }); + }); +} diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 00498c54..d38c1b41 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -7,6 +7,7 @@ import Status from '../../lib/dto/Status'; import DBSQLOperation from '../../lib/DBSQLOperation'; import { PrimaryKeysRequest } from '../../lib/contracts/IDBSQLSession'; import ISessionBackend from '../../lib/contracts/ISessionBackend'; +import IOperationBackend from '../../lib/contracts/IOperationBackend'; import ParameterError from '../../lib/errors/ParameterError'; import { DBSQLParameter } from '../../lib/DBSQLParameter'; import { TSessionHandle, TProtocolVersion, TOperationState, TStatusCode } from '../../thrift/TCLIService_types'; @@ -883,3 +884,19 @@ describe('DBSQLSession', () => { }); }); }); + +describe('DBSQLSession native staging dispatch', () => { + for (const configured of [false, true]) { + it(`calls native completion only when local paths are ${configured ? 'configured' : 'omitted'}`, async () => { + const completeStagingOperation = sinon.stub().resolves(); + const op = { id: 'volume-operation', completeStagingOperation } as unknown as IOperationBackend; + const session = new DBSQLSession({ + backend: { id: 'volume-session', executeStatement: async () => op } as unknown as ISessionBackend, + context: new ClientContextStub(), + }); + await session.executeStatement('REMOVE remote', configured ? { stagingAllowedLocalPath: '/tmp' } : {}); + expect(completeStagingOperation.callCount).to.equal(configured ? 1 : 0); + if (configured) expect(completeStagingOperation.firstCall.args).to.deep.equal([['/tmp']]); + }); + } +}); diff --git a/tests/unit/kernel/KernelOperationBackend.test.ts b/tests/unit/kernel/KernelOperationBackend.test.ts index 972de4fc..0fb650f5 100644 --- a/tests/unit/kernel/KernelOperationBackend.test.ts +++ b/tests/unit/kernel/KernelOperationBackend.test.ts @@ -40,6 +40,8 @@ import { import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; import { TTypeId } from '../../../thrift/TCLIService_types'; import ClientContextStub from '../.stubs/ClientContextStub'; +import ParameterError from '../../../lib/errors/ParameterError'; +import StagingError from '../../../lib/errors/StagingError'; // Minimal stub of the napi `Statement` surface that emits a precomputed // Arrow IPC payload per `fetchNextBatch()` call. Used to feed @@ -389,3 +391,92 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe expect(stub.closed).to.equal(true); }); }); + +describe('KernelOperationBackend Volume completion', () => { + it('preserves SQL execution errors', async () => { + const error = new ParameterError('invalid SQL parameter'); + const backend = new KernelOperationBackend({ + asyncStatement: { + status: async () => 'Succeeded', + awaitResult: async () => { + throw error; + }, + } as any, + context: new ClientContextStub(), + }); + const actual = await backend.completeStagingOperation(['/allowed']).then( + () => expect.fail('execution should fail'), + (failure) => failure, + ); + expect(actual).to.equal(error); + }); + + for (const { error, staging } of [ + { error: new ParameterError('path is not allowed'), staging: true }, + { + error: Object.assign(new Error('storage HTTP 403'), { + kernelMetadata: { httpStatus: 403, errorCode: 'VOLUME_STORAGE_HTTP_ERROR' }, + }), + staging: true, + }, + { + error: Object.assign(new ParameterError('invalid instruction request'), { + sqlState: '22023', + kernelMetadata: { httpStatus: 400 }, + }), + staging: false, + }, + ]) { + it(`maps completion failure: ${error.message}`, async () => { + const schema = new Schema([withTypeName(new Field('x', new Int32(), true), 'INT')]); + const statement = new StatementStub(ipcSchemaOnly(schema), []); + Object.assign(statement, { + completeVolumeOperation: async () => { + throw error; + }, + }); + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub() }); + const actual = await backend.completeStagingOperation(['/allowed']).then( + () => expect.fail('completion should fail'), + (failure) => failure, + ); + if (staging) { + expect(actual).to.be.instanceOf(StagingError).with.property('message', error.message); + expect(actual.kernelMetadata).to.equal((error as any).kernelMetadata); + } else { + expect(actual).to.equal(error); + if ('sqlState' in error) expect(actual.sqlState).to.equal(error.sqlState); + if ('kernelMetadata' in error) expect(actual.kernelMetadata).to.equal(error.kernelMetadata); + } + }); + } + + it('rejects a stream PUT that needs input', async () => { + const schema = new Schema([withTypeName(new Field('x', new Int32(), true), 'INT')]); + const statement = new StatementStub(ipcSchemaOnly(schema), []); + Object.assign(statement, { completeVolumeOperation: async () => 'needs_input_stream' }); + const backend = new KernelOperationBackend({ statement, context: new ClientContextStub() }); + const error = await backend.completeStagingOperation(['/allowed']).catch((failure) => failure); + expect(error).to.be.instanceOf(StagingError).with.property('message', 'Volume PUT requires a local file path'); + }); + + for (const outcome of ['completed', 'not_volume']) { + it(`${ + outcome === 'completed' ? 'consumes completed instructions' : 'preserves ordinary result rows' + }`, async () => { + const schema = new Schema([withTypeName(new Field('x', new Int32(), true), 'INT')]); + const stub = new StatementStub(ipcSchemaOnly(schema), [ipcFromColumns(schema, { x: [1] })]); + const completeVolumeOperation = async (paths: string[], remove404: boolean) => { + expect(paths).to.deep.equal(['/allowed']); + expect(remove404).to.equal(true); + return outcome; + }; + Object.assign(stub, { isVolumeOperation: outcome === 'completed', completeVolumeOperation }); + const backend = new KernelOperationBackend({ statement: stub, context: new ClientContextStub() }); + expect((await backend.getResultMetadata()).isStagingOperation).to.equal(outcome === 'completed'); + await backend.completeStagingOperation(['/allowed']); + expect((await backend.fetchChunk({ limit: 100 })).length).to.equal(outcome === 'completed' ? 0 : 1); + expect(await backend.hasMore()).to.equal(false); + }); + } +}); diff --git a/tests/unit/kernel/connectionOptions.test.ts b/tests/unit/kernel/connectionOptions.test.ts index 153e388f..43c054f0 100644 --- a/tests/unit/kernel/connectionOptions.test.ts +++ b/tests/unit/kernel/connectionOptions.test.ts @@ -331,6 +331,12 @@ describe('KernelAuth HTTP options (buildKernelHttpOptions)', () => { const names = (http: { customHeaders?: Array<{ name: string; value: string }> }) => (http.customHeaders ?? []).map((h) => h.name); + it('passes zero and positive socketTimeout values through unchanged', () => { + for (const socketTimeout of [0, 4500]) { + expect(buildKernelHttpOptions(opts({ socketTimeout })).socketTimeoutMs).to.equal(socketTimeout); + } + }); + it('always emits a User-Agent identifying the connector', () => { const http = buildKernelHttpOptions(opts({})); expect(ua(http)).to.match(/NodejsDatabricksSqlConnector\//); diff --git a/tests/unit/kernel/error-mapping.test.ts b/tests/unit/kernel/error-mapping.test.ts index fefa8f2a..d15f614f 100644 --- a/tests/unit/kernel/error-mapping.test.ts +++ b/tests/unit/kernel/error-mapping.test.ts @@ -372,3 +372,32 @@ describe('KernelErrorMapping.decodeNapiKernelError — cause chain', () => { expect(err.cause).to.equal(undefined); }); }); + +describe('KernelErrorMapping.decodeNapiKernelError — local file errors', () => { + for (const [raw, kind, expected] of [ + [2, 'NotFound', 'ENOENT'], + [undefined, 'PermissionDenied', 'EACCES'], + [undefined, 'Unrecognized', 'EIO'], + [1000000, 'NotFound', 'ENOENT'], + ] as const) { + it(`preserves local file error code and path for ${kind}/${raw}`, () => { + const error = decodeNapiKernelError( + new Error( + `__databricks_error__:${JSON.stringify({ + code: 'NotFound', + message: 'Local file does not exist', + localOsError: raw, + localErrorKind: kind, + localPath: '/allowed/missing', + sqlState: 'HY000', + queryId: 'query-id', + })}`, + ), + ) as NodeJS.ErrnoException; + expect(error.code).to.equal(expected); + expect(error.path).to.equal('/allowed/missing'); + expect(error.errno).to.be.lessThan(0); + expect(error).to.have.property('kernelMetadata').with.property('queryId', 'query-id'); + }); + } +}); diff --git a/tests/unit/kernel/execution.test.ts b/tests/unit/kernel/execution.test.ts index 9aae43f5..211dfff7 100644 --- a/tests/unit/kernel/execution.test.ts +++ b/tests/unit/kernel/execution.test.ts @@ -24,6 +24,7 @@ import DBSQLSession from '../../../lib/DBSQLSession'; import DBSQLOperation from '../../../lib/DBSQLOperation'; import asyncDisposable from '../../../lib/asyncDisposable'; import { KernelNativeBinding, KernelConnection, KernelStatement } from '../../../lib/kernel/KernelNativeLoader'; +import type { VolumeCompletion } from '../../../native/kernel'; import IClientContext, { ClientConfig } from '../../../lib/contracts/IClientContext'; import { CrossReferenceRequest, PrimaryKeysRequest } from '../../../lib/contracts/IDBSQLSession'; import IDBSQLLogger, { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; @@ -41,6 +42,12 @@ import { DBSQLParameter, DBSQLParameterType } from '../../../lib/DBSQLParameter' // ----------------------------------------------------------------------------- class FakeNativeStatement implements KernelStatement { + public readonly isVolumeOperation = false; + + public async completeVolumeOperation(): Promise { + return 'not_volume' as VolumeCompletion; + } + public closed = false; public cancelled = false; @@ -1152,13 +1159,11 @@ describe('KernelSessionBackend', () => { // Unsupported-on-kernel per-statement HINTS are NO-OPs (not errors) so Thrift- // targeted call sites are drop-in on the kernel path. They are kernel-governed - // perf/format hints (useCloudFetch / useLZ4Compression) or an unexposed feature - // (staging); ignoring them can't change results. (Parameter binding is NOT here + // perf/format hints (useCloudFetch / useLZ4Compression). (Parameter binding is NOT here // — a dropped param would change results, so it still throws; asserted above.) for (const { name, options } of [ { name: 'useCloudFetch', options: { useCloudFetch: true } }, { name: 'useLZ4Compression', options: { useLZ4Compression: true } }, - { name: 'stagingAllowedLocalPath', options: { stagingAllowedLocalPath: '/tmp' } }, ] as const) { it(`executeStatement ignores unsupported ${name} (no-op, does not throw)`, async () => { const connection = new FakeNativeConnection();