diff --git a/CHANGELOG.md b/CHANGELOG.md index f8aeb947..04033889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Kernel backend: a custom `port` is honored over HTTPS, as on Thrift. It was dropped unless `https: false` was set. A non-default port folds into `host` (bare IPv6 hosts are bracketed); `port: 443`, a falsy `port` (`0`, `NaN`, `''`, `null`, `false`) and an omitted port leave `host` unchanged; a numeric string is accepted; a port embedded in `host` wins over a different `port` (a warning is logged); an out-of-range port throws `HiveDriverError`; a `host` with a path, query or fragment throws when the `port` is folded into it (`https: false`, or a non-443 `port` over HTTPS). With OAuth and a non-default port, the kernel reaches its OAuth endpoints at the same `host:port`, while Thrift keeps OAuth on the bare host (documented, not guarded). - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index edba6002..9bad42c3 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -38,15 +38,15 @@ backend (the default), the **SEA / Kernel** backend, or both. ## Connection identity and backend selection -| Option | Type | Thrift | Kernel | Default Value | Note | -| ----------------------- | --------- | :----: | :----: | ------------- | -------------------------------------------------------------------------------------- | -| `host` | `string` | ✅ | ✅ | — (required) | Required on both. | -| `path` | `string` | ✅ | ✅ | — (required) | Kernel also derives `x-databricks-org-id` from a `?o=` query parameter. | -| `port` | `number` | ✅ | ⚠️ | `443` / — | Kernel uses it only with `https: false`; otherwise put a non-default port in `host`. | -| `https` | `boolean` | ✅ | ✅ | `true` | `false` selects plain HTTP; see [Plain HTTP transport](#plain-http-transport). | -| `userAgentEntry` | `string` | ✅ | ✅ | — | Folded into the connector `User-Agent` on both. | -| `useKernel` (internal) | `boolean` | — | — | `false` | Selects the backend: `false`/omitted → Thrift, `true` → Kernel. | -| `clientId` (deprecated) | `string` | ✅ | ✅ | — | Runtime-only alias for `userAgentEntry`. `userAgentEntry` wins when both are supplied. | +| Option | Type | Thrift | Kernel | Default Value | Note | +| ----------------------- | --------- | :----: | :----: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | `string` | ✅ | ✅ | — (required) | Required on both. | +| `path` | `string` | ✅ | ✅ | — (required) | Kernel also derives `x-databricks-org-id` from a `?o=` query parameter. | +| `port` | `number` | ✅ | ✅ | `443` / — | Kernel folds a non-default port into a bare `host` over HTTPS and with `https: false` (a bare IPv6 host is bracketed). A port already in `host` wins over `port`, with a warning when they differ; a falsy `port` means the default. With OAuth, the kernel uses the same `host:port` for its OAuth endpoints; Thrift keeps OAuth on the bare `host` (port 443). | +| `https` | `boolean` | ✅ | ✅ | `true` | `false` selects plain HTTP; see [Plain HTTP transport](#plain-http-transport). | +| `userAgentEntry` | `string` | ✅ | ✅ | — | Folded into the connector `User-Agent` on both. | +| `useKernel` (internal) | `boolean` | — | — | `false` | Selects the backend: `false`/omitted → Thrift, `true` → Kernel. | +| `clientId` (deprecated) | `string` | ✅ | ✅ | — | Runtime-only alias for `userAgentEntry`. `userAgentEntry` wins when both are supplied. | Set `client.getConfig().useCloudFetch = false` (default `true`) before opening a session to fetch complete results without cloud-storage downloads. On the kernel backend this diff --git a/lib/kernel/KernelAuth.ts b/lib/kernel/KernelAuth.ts index 8d900485..ade0e495 100644 --- a/lib/kernel/KernelAuth.ts +++ b/lib/kernel/KernelAuth.ts @@ -830,23 +830,74 @@ export function buildKernelProxyOptions(options: ConnectionOptions): KernelProxy return { proxy: out }; } -// The kernel prepends `https://` to a bare host but keeps an explicit scheme. -// It otherwise ignores `port`, so carry it into the URL unless `host` already has one. -function buildKernelHostName(options: ConnectionOptions): string { - if (options.https !== false) { - return options.host; +// `name:port` or `[ipv6]:port`; a bare IPv6 literal like `::1` has no port. +const EMBEDDED_PORT_RE = /^(\[[^\]]+\]|[^:]+):(\d+)$/; + +// An explicit `port` is read as Thrift reads it (`options.port || 443`): a falsy value +// (unset, null, '', 0, NaN, false) means the default. A truthy value must be a whole +// port number, and a decimal string such as `'8443'` is accepted. Anything else throws, +// as Thrift also fails on it. +function explicitPort(port: unknown): number | undefined { + if (!port) return undefined; + let value = Number.NaN; + if (typeof port === 'number') value = port; + else if (typeof port === 'string' && /^[0-9]+$/.test(port)) value = Number(port); + if (value === 0) return undefined; + if (!Number.isInteger(value) || value < 1 || value > 65535) { + throw new HiveDriverError(`kernel backend: \`port\` must be an integer between 1 and 65535; got ${String(port)}.`); + } + return value; +} + +// A bare IPv6 literal (colons, no brackets) needs brackets before a port is appended. +const bracketIpv6 = (authority: string): string => + authority.includes(':') && !authority.startsWith('[') ? `[${authority}]` : authority; + +// The kernel keeps a `host:port` authority but has no port setter, so an explicit port is +// folded into the host. Precedence, as on the pre-existing kernel path: a port embedded in +// `host` always wins. The explicit `port` applies only to a bare host, and when it differs +// from the embedded port the driver warns and connects to the embedded port. A trailing +// "/" is dropped before any port handling, as the kernel does. +function buildKernelHostName(options: ConnectionOptions, logger?: IDBSQLLogger): string { + const port = explicitPort(options.port); + const scheme = /^https?:\/\//i.exec(options.host)?.[0] ?? ''; + const authority = options.host.slice(scheme.length).replace(/\/$/, ''); + // A path, query or fragment in host cannot take a folded port: it is rejected when the port would + // be folded in (https: false, or HTTPS with a non-443 port). With no port, or with 443 over HTTPS, + // the host is passed through unchanged, as on origin/main (including `h.io/path:8443`). + if (port !== undefined && /[/?#]/.test(authority) && (options.https === false || port !== 443)) { + throw new HiveDriverError( + `kernel backend: host ${JSON.stringify(options.host)} has a path, query or fragment, so \`port\` ${port} ` + + 'cannot be applied to it. Use a bare host or host:port.', + ); + } + const match = EMBEDDED_PORT_RE.exec(authority); + const embedded = match ? Number(match[2]) : undefined; + if (embedded !== undefined && (embedded < 1 || embedded > 65535)) { + throw new HiveDriverError(`kernel backend: host carries port ${embedded}, which is outside 1..65535.`); + } + // 443 is the default that getConnectionOptions fills in, so it is not reported as a conflict. + if (port !== undefined && port !== 443 && embedded !== undefined && embedded !== port) { + logger?.log( + LogLevel.warn, + `kernel backend: host ${JSON.stringify(options.host)} carries port ${embedded} but \`port\` is ${port}. ` + + `Connecting to port ${embedded}; \`port\` is ignored.`, + ); + } + // Fold a port into a bare host. `https: false` always folds a supplied port, while HTTPS + // folds only a non-default one (443 is the default and adds nothing). + if (embedded === undefined && port !== undefined && (options.https === false || port !== 443)) { + const hostPort = `${bracketIpv6(authority)}:${port}`; + return options.https === false ? `http://${hostPort}` : `${scheme}${hostPort}`; } - // The kernel accepts a full URL in `host`, so drop an existing `http://` or - // `https://` scheme before applying HTTP (`https: false` selects the scheme). - // Then strip a trailing `/` like the Thrift URL builder does. - const host = options.host.replace(/^https?:\/\//i, '').replace(/\/$/, ''); - // `name:port` or `[ipv6]:port`; a bare IPv6 literal like `::1` has no port. - const hasPort = /^(\[[^\]]+\]|[^:]+):\d+$/.test(host); - const port = options.port && !hasPort ? `:${options.port}` : ''; - return `http://${host}${port}`; + if (options.https === false) return `http://${authority}`; + return `${scheme}${authority}`; } -export function buildKernelConnectionOptions(options: ConnectionOptions): KernelNativeConnectionOptions { +export function buildKernelConnectionOptions( + options: ConnectionOptions, + logger?: IDBSQLLogger, +): KernelNativeConnectionOptions { const { authType } = options as { authType?: string }; const base: { @@ -859,7 +910,7 @@ export function buildKernelConnectionOptions(options: ConnectionOptions): Kernel KernelHttpOptions & KernelProxyOptions & KernelFederationOptions = { - hostName: buildKernelHostName(options), + hostName: buildKernelHostName(options, logger), httpPath: prependSlash(options.path), // Match the NodeJS Thrift driver, which surfaces INTERVAL columns as // strings. The kernel defaults to native Arrow interval/duration types; diff --git a/lib/kernel/KernelBackend.ts b/lib/kernel/KernelBackend.ts index 5c622ea0..e2357acc 100644 --- a/lib/kernel/KernelBackend.ts +++ b/lib/kernel/KernelBackend.ts @@ -96,7 +96,7 @@ export default class KernelBackend implements IBackend { // config (the same `ClientConfig` knobs the Thrift `HttpRetryPolicy` reads), // converted from the connector's milliseconds to the kernel's whole seconds. this.nativeOptions = { - ...buildKernelConnectionOptions(options), + ...buildKernelConnectionOptions(options, this.context.getLogger()), ...buildKernelRetryOptions(this.context.getConfig()), ...buildKernelTelemetryOptions(this.context.getConfig(), options, this.context.getLogger()), }; diff --git a/tests/unit/DBSQLClient.test.ts b/tests/unit/DBSQLClient.test.ts index 497889db..3ed48b4c 100644 --- a/tests/unit/DBSQLClient.test.ts +++ b/tests/unit/DBSQLClient.test.ts @@ -234,6 +234,40 @@ describe('DBSQLClient.connect', () => { expect(connectionOptions.port).to.equal(3015); }); + it('keeps the Thrift URL for a host with an embedded port unchanged (current behavior, not changed by the kernel fix)', async () => { + const client = new DBSQLClient(); + + // Thrift appends options.port (443 by default) after the host, so an embedded port becomes + // `h:8443:443`, which node-fetch rejects as an invalid URL. This pins that current behavior. + const https = client['getConnectionOptions']({ ...connectOptions, host: 'h.databricks.com:8443', port: 443 }); + expect(https.host).to.equal('h.databricks.com:8443'); + expect(https.port).to.equal(443); + const httpsConnection = await new HttpConnection(https, client).getThriftConnection(); + expect(httpsConnection.url).to.equal('https://h.databricks.com:8443:443/'); + + const http = client['getConnectionOptions']({ + ...connectOptions, + host: 'localhost:3015', + port: 80, + https: false, + }); + const httpConnection = await new HttpConnection(http, client).getThriftConnection(); + expect(httpConnection.url).to.equal('http://localhost:3015:80/'); + }); + + it('treats a falsy port as the default (443 over HTTPS, 80 over HTTP), as the kernel path does', async () => { + const client = new DBSQLClient(); + + for (const port of [0, Number.NaN, undefined, null, '', false]) { + const label = String(port); + expect(client['getConnectionOptions']({ ...connectOptions, port: port as any }).port, label).to.equal(443); + expect( + client['getConnectionOptions']({ ...connectOptions, port: port as any, https: false }).port, + label, + ).to.equal(80); + } + }); + it('should only switch to plain HTTP for an explicit https: false', async () => { const client = new DBSQLClient(); diff --git a/tests/unit/kernel/connectionOptions.test.ts b/tests/unit/kernel/connectionOptions.test.ts index 153e388f..42aeb1f1 100644 --- a/tests/unit/kernel/connectionOptions.test.ts +++ b/tests/unit/kernel/connectionOptions.test.ts @@ -21,6 +21,7 @@ import { } from '../../../lib/kernel/KernelAuth'; import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; import HiveDriverError from '../../../lib/errors/HiveDriverError'; +import IDBSQLLogger, { LogLevel } from '../../../lib/contracts/IDBSQLLogger'; const PAT = { host: 'h.databricks.com', path: '/sql/1.0/warehouses/abc', token: 'dapi-x' }; @@ -127,11 +128,126 @@ describe('KernelAuth connection options — https transport', () => { ); // Same URL shape as the Thrift transport, which also requires brackets. expect(buildKernelConnectionOptions(opts({ host: '::1', port: 3015, https: false })).hostName).to.equal( - 'http://::1:3015', + 'http://[::1]:3015', ); }); }); +describe('KernelAuth connection options — port folding', () => { + const hostName = (extra: Record) => buildKernelConnectionOptions(opts(extra)).hostName; + + // [connection options, expected hostName] for HTTPS (default) and https: false. + // An embedded host port always wins; the explicit port applies only to a bare host. + const accepted: Array<[Record, string]> = [ + [{ port: 8443 }, 'h.databricks.com:8443'], + [{ port: 8443, https: true }, 'h.databricks.com:8443'], + [{ port: '8443' }, 'h.databricks.com:8443'], // a decimal string is accepted, as on Thrift + [{ port: 443 }, 'h.databricks.com'], // no :443 suffix + // Falsy means the default, as on Thrift (`options.port || 443`); never a throw. + [{ port: 0 }, 'h.databricks.com'], + [{ port: '' }, 'h.databricks.com'], + [{ port: null }, 'h.databricks.com'], + [{ port: undefined }, 'h.databricks.com'], + [{ port: Number.NaN }, 'h.databricks.com'], + [{ port: false }, 'h.databricks.com'], + [{ port: Number.NaN, https: false }, 'http://h.databricks.com'], + [{}, 'h.databricks.com'], + [{ host: 'h.databricks.com:8443' }, 'h.databricks.com:8443'], + [{ host: 'h.databricks.com:8443', port: 8443 }, 'h.databricks.com:8443'], + // A port embedded in host wins over a different explicit port (warned, not thrown). + [{ host: 'h.databricks.com:8443', port: 443 }, 'h.databricks.com:8443'], + [{ host: 'h.databricks.com:7000', port: 8443 }, 'h.databricks.com:7000'], + [{ host: '[::1]:7000', port: 8443 }, '[::1]:7000'], + [{ host: 'https://h.databricks.com', port: 8443 }, 'https://h.databricks.com:8443'], + [{ host: '[::1]', port: 8443 }, '[::1]:8443'], + [{ host: '[::1]:8443', port: 8443 }, '[::1]:8443'], + [{ host: '::1', port: 8443 }, '[::1]:8443'], // bare IPv6 is bracketed + [{ host: '::1', port: 443 }, '::1'], + // Trailing "/" is dropped before port handling; a host with no port is otherwise unchanged. + [{ host: 'h.databricks.com/', port: 8443 }, 'h.databricks.com:8443'], + [{ host: 'https://h.databricks.com/', port: 8443 }, 'https://h.databricks.com:8443'], + [{ host: 'h.databricks.com:8443/' }, 'h.databricks.com:8443'], + [{ host: 'h.databricks.com/' }, 'h.databricks.com'], + // A path host with no port, or with 443 over HTTPS, is not folded: passed through unchanged, as before. + [{ host: 'h.databricks.com/x' }, 'h.databricks.com/x'], + [{ host: 'h.databricks.com/x', port: 443 }, 'h.databricks.com/x'], + [{ host: 'https://h.databricks.com/x', port: 443 }, 'https://h.databricks.com/x'], + [{ host: 'h.databricks.com/x', port: 443, https: true }, 'h.databricks.com/x'], + // An embedded port after a path, with no folded port, passes through as on origin/main. + [{ host: 'h.io/path:8443' }, 'h.io/path:8443'], + [{ host: 'h.io/path:8443', port: 443 }, 'h.io/path:8443'], + [{ host: 'h.databricks.com', port: 3015, https: false }, 'http://h.databricks.com:3015'], + [{ host: 'localhost', port: 80, https: false }, 'http://localhost:80'], + [{ host: 'localhost', port: '3015', https: false }, 'http://localhost:3015'], + [{ host: '::1', port: 3015, https: false }, 'http://[::1]:3015'], // bare IPv6 is bracketed + [{ host: '::1', https: false }, 'http://::1'], // no port: unchanged + [{ host: 'http://localhost:3015', https: false }, 'http://localhost:3015'], + // Reviewer cases: the embedded port wins over the explicit one, on both transports' shape. + [{ host: 'h.databricks.com:8443', port: 443, https: true }, 'h.databricks.com:8443'], + [{ host: 'localhost:3015', port: 80, https: false }, 'http://localhost:3015'], + [{ host: 'localhost:3015', port: 443, https: false }, 'http://localhost:3015'], + [{ host: 'h.databricks.com:8443/', port: 443, https: false }, 'http://h.databricks.com:8443'], + [{ host: 'h.databricks.com/', port: 8443, https: false }, 'http://h.databricks.com:8443'], + ]; + it('folds the port into the host as the table specifies', () => { + for (const [extra, want] of accepted) { + expect(hostName(extra), JSON.stringify(extra)).to.equal(want); + } + }); + + // [connection options, message] that must throw HiveDriverError: truthy invalid ports + // (Thrift also fails on these), a path/query/fragment that would take a folded port, and + // out-of-range embedded ports. + const badPorts: unknown[] = [65536, -1, 1.5, Number.POSITIVE_INFINITY, true, 'abc', '8443x', '-1', '65536']; + const rejected: Array<[Record, RegExp]> = [ + ...badPorts.flatMap( + (port): Array<[Record, RegExp]> => [ + [{ port }, /must be an integer between 1 and 65535/], + [{ port, https: false }, /must be an integer between 1 and 65535/], + ], + ), + [{ host: 'h.databricks.com/x', port: 8443 }, /has a path, query or fragment/], + [{ host: 'h.databricks.com/x/', port: 8443 }, /has a path, query or fragment/], + [{ host: 'https://h.databricks.com/x', port: 8443 }, /has a path, query or fragment/], + [{ host: 'h.databricks.com?x', port: 8443 }, /has a path, query or fragment/], + [{ host: 'h.databricks.com#x', port: 3015, https: false }, /has a path, query or fragment/], + // A folded port (explicit non-443, or https: false) on a path host is rejected, including + // when the host also carries an embedded port after the path. + [{ host: 'h.io/path:8443', port: 8443 }, /has a path, query or fragment/], + [{ host: 'h.io/path:8443', port: 443, https: false }, /has a path, query or fragment/], + [{ host: 'h.databricks.com:70000' }, /outside 1\.\.65535/], + ]; + it('rejects invalid ports and unfoldable hosts', () => { + for (const [extra, message] of rejected) { + expect(() => hostName(extra), JSON.stringify(extra)).to.throw(HiveDriverError, message); + } + }); + + it('warns and connects to the embedded port when an explicit port differs', () => { + const logs: Array<[LogLevel, string]> = []; + const logger = { + log: (level: LogLevel, message: string) => logs.push([level, message]), + } as unknown as IDBSQLLogger; + const native = buildKernelConnectionOptions(opts({ host: 'h.databricks.com:7000', port: 8443 }), logger); + expect(native.hostName).to.equal('h.databricks.com:7000'); + expect(logs).to.have.length(1); + expect(logs[0][0]).to.equal(LogLevel.warn); + expect(logs[0][1]).to.contain('carries port 7000'); + expect(logs[0][1]).to.contain('`port` is 8443'); + }); + + it('does not warn when the ports agree, the explicit port is the default, or there is no logger', () => { + const logs: Array<[LogLevel, string]> = []; + const logger = { + log: (level: LogLevel, message: string) => logs.push([level, message]), + } as unknown as IDBSQLLogger; + buildKernelConnectionOptions(opts({ host: 'h.databricks.com:8443', port: 8443 }), logger); + buildKernelConnectionOptions(opts({ host: 'h.databricks.com:8443', port: 443 }), logger); + buildKernelConnectionOptions(opts({ host: 'h.databricks.com:7000', port: 8443 })); + expect(logs).to.have.length(0); + }); +}); + describe('KernelAuth connection options — maxConnections', () => { it('forwards a valid positive integer', () => { const native = buildKernelConnectionOptions(opts({ maxConnections: 10 })) as { maxConnections?: number };