From 0a652d2c58f743320525d9b15ae3af72a368c502 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 15:49:59 +0000 Subject: [PATCH 1/4] Release completed CloudFetch body-reader listeners Detach only the listeners installed by node-fetch after each body read settles, preserving other listeners and late-error handling. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + lib/result/CloudFetchResultHandler.ts | 25 +++++++- .../result/CloudFetchResultHandler.test.ts | 60 +++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c06d6f7..6c1a4ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Release completed Thrift CloudFetch body-reader listeners so idle connections do not retain downloaded result buffers. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 71d8c44b..bf28511f 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,4 +1,5 @@ import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; +import { Readable } from 'stream'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; import RetryError, { RetryErrorCode } from '../errors/RetryError'; @@ -149,6 +150,26 @@ export default class CloudFetchResultHandler implements IResultsProvider { + const { body } = response; + if (!(body instanceof Readable)) { + return response.buffer(); + } + const events = ['data', 'end', 'error']; + const existing = events.map((event) => body.listeners(event)); + const result = response.buffer(); + const added = events.map((event, index) => + body.listeners(event).filter((listener) => !existing[index].includes(listener)), + ); + try { + return await result; + } finally { + events.forEach((event, index) => { + added[index].forEach((listener) => body.removeListener(event, listener as () => void)); + }); + } + } + private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: Buffer }> { const connectionProvider = await this.context.getConnectionProvider(); let requestUrl = url; @@ -194,7 +215,7 @@ export default class CloudFetchResultHandler implements IResultsProvider { }, ); + describe('response body retention', () => { + function createHandler(response: Response) { + const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 1 }); + context.invokeWithRetryStub.resolves({ request: new Request('http://localhost'), response }); + return new CloudFetchResultHandler(context, new ResultsProviderStub([sampleRowSet1], undefined), { + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + } + + it('releases consumed body listeners without removing existing or later listeners', async () => { + const body = new PassThrough(); + const response = new Response(body); + const existing = ['data', 'end', 'error'].map((event) => { + const listener = sinon.spy(); + body.on(event, listener); + return listener; + }); + const later = sinon.spy(); + body.once('data', () => body.on('error', later)); + const errorsBefore = body.listeners('error'); + const handler = createHandler(response); + setImmediate(() => body.end(sampleArrowBatch)); + + const batch = await handler.fetchNext({ limit: 10000 }); + + expect(batch.batches).to.deep.eq([sampleArrowBatch]); + expect(response.bodyUsed).to.be.true; + expect(body.listeners('data')).to.deep.eq([existing[0]]); + expect(body.listeners('end')).to.deep.eq([existing[1]]); + expect(body.listeners('error')).to.deep.eq([...errorsBefore, later]); + body.emit('error', new Error('late stream error')); + expect(existing[2].calledOnce).to.be.true; + expect(later.calledOnce).to.be.true; + }); + + it('releases body reader listeners when consumption rejects', async () => { + const body = new PassThrough(); + const response = new Response(body); + const errorsBefore = body.listeners('error'); + const handler = createHandler(response); + setImmediate(() => body.destroy(new Error('body interrupted'))); + + let thrown: unknown; + try { + await handler.fetchNext({ limit: 10000 }); + } catch (error) { + thrown = error; + } + + expect(thrown).to.be.instanceOf(Error); + expect(response.bodyUsed).to.be.true; + expect(body.listenerCount('data')).to.eq(0); + expect(body.listenerCount('end')).to.eq(0); + expect(body.listeners('error')).to.deep.eq(errorsBefore); + }); + }); + describe('download deadline', () => { let server: http.Server | undefined; @@ -840,6 +898,8 @@ describe('CloudFetchResultHandler', () => { expect(buffer.calledOnce).to.be.true; expect(arrayBuffer.called).to.be.false; expect(buffer.firstCall.thisValue.body).to.have.property('pipe').that.is.a('function'); + expect(buffer.firstCall.thisValue.body.listenerCount('data')).to.eq(0); + expect(buffer.firstCall.thisValue.body.listenerCount('end')).to.eq(0); const downloaded = await buffer.firstCall.returnValue; expect(downloaded).to.deep.eq(body); expect(downloaded.length).to.be.lt(Buffer.poolSize >>> 1); From 4b276e0e0750ed9691f9cdee1ca0ccba262ea70f Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 17:14:25 +0000 Subject: [PATCH 2/4] Test late CloudFetch body errors after listener cleanup Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .../result/CloudFetchResultHandler.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index b896228c..23c3c588 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -1,5 +1,6 @@ import { expect, AssertionError } from 'chai'; import sinon, { SinonStub } from 'sinon'; +import { execFileSync } from 'child_process'; import http from 'http'; import { PassThrough } from 'stream'; import { AddressInfo } from 'net'; @@ -808,6 +809,41 @@ describe('CloudFetchResultHandler', () => { expect(body.listenerCount('end')).to.eq(0); expect(body.listeners('error')).to.deep.eq(errorsBefore); }); + + it('handles late body errors without application error handlers or unhandled rejections', function () { + this.timeout(15000); + const script = ` + const assert = require('assert').strict; + const { PassThrough } = require('stream'); + const { Response } = require('node-fetch'); + const Handler = require('./lib/result/CloudFetchResultHandler').default; + (async () => { + const body = new PassThrough(); + const response = new Response(body); + const originalErrors = body.listeners('error'); + assert.equal(originalErrors.length, 1); + const reading = Handler.prototype.readBody.call({}, response); + body.end(Buffer.from('result bytes')); + assert.equal((await reading).toString(), 'result bytes'); + assert.equal(response.bodyUsed, true); + assert.equal(body.listenerCount('data'), 0); + assert.equal(body.listenerCount('end'), 0); + await new Promise((resolve) => setImmediate(() => { + body.emit('error', new Error('late body error')); + resolve(); + })); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(body.listeners('error'), originalErrors); + process.stdout.write('late body error handled\\n'); + })().catch((error) => { console.error(error); process.exitCode = 1; }); + `; + const output = execFileSync( + process.execPath, + ['--unhandled-rejections=strict', '--require', 'ts-node/register', '-e', script], + { cwd: process.cwd(), encoding: 'utf8', timeout: 10000, stdio: 'pipe' }, + ); + expect(output).to.eq('late body error handled\n'); + }); }); describe('download deadline', () => { From cfbf0b2b5bebb11fa752af1e2eb5de120ac8c0a4 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 18:55:21 +0000 Subject: [PATCH 3/4] Strengthen CloudFetch retention regression coverage Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- lib/result/CloudFetchResultHandler.ts | 2 ++ tests/unit/result/CloudFetchResultHandler.test.ts | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index bf28511f..50a92b0b 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -155,6 +155,8 @@ export default class CloudFetchResultHandler implements IResultsProvider body.listeners(event)); const result = response.buffer(); diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 23c3c588..74d31781 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -2,6 +2,7 @@ import { expect, AssertionError } from 'chai'; import sinon, { SinonStub } from 'sinon'; import { execFileSync } from 'child_process'; import http from 'http'; +import path from 'path'; import { PassThrough } from 'stream'; import { AddressInfo } from 'net'; import Int64 from 'node-int64'; @@ -812,11 +813,13 @@ describe('CloudFetchResultHandler', () => { it('handles late body errors without application error handlers or unhandled rejections', function () { this.timeout(15000); + const repoRoot = path.resolve(__dirname, '../../..'); + const handlerPath = path.join(repoRoot, 'lib/result/CloudFetchResultHandler'); const script = ` const assert = require('assert').strict; const { PassThrough } = require('stream'); const { Response } = require('node-fetch'); - const Handler = require('./lib/result/CloudFetchResultHandler').default; + const Handler = require(${JSON.stringify(handlerPath)}).default; (async () => { const body = new PassThrough(); const response = new Response(body); @@ -839,8 +842,8 @@ describe('CloudFetchResultHandler', () => { `; const output = execFileSync( process.execPath, - ['--unhandled-rejections=strict', '--require', 'ts-node/register', '-e', script], - { cwd: process.cwd(), encoding: 'utf8', timeout: 10000, stdio: 'pipe' }, + ['--unhandled-rejections=strict', '--require', require.resolve('ts-node/register'), '-e', script], + { cwd: repoRoot, encoding: 'utf8', timeout: 10000, stdio: 'pipe' }, ); expect(output).to.eq('late body error handled\n'); }); @@ -936,6 +939,7 @@ describe('CloudFetchResultHandler', () => { expect(buffer.firstCall.thisValue.body).to.have.property('pipe').that.is.a('function'); expect(buffer.firstCall.thisValue.body.listenerCount('data')).to.eq(0); expect(buffer.firstCall.thisValue.body.listenerCount('end')).to.eq(0); + expect(buffer.firstCall.thisValue.body.listenerCount('error')).to.eq(1); const downloaded = await buffer.firstCall.returnValue; expect(downloaded).to.deep.eq(body); expect(downloaded.length).to.be.lt(Buffer.poolSize >>> 1); From 0173b05632d52562edb61ee64d7bc996a211c95b Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 07:55:26 +0000 Subject: [PATCH 4/4] Pin CloudFetch body collectability over HTTPS keep-alive Explain the Node core retention the listener cleanup works around: on a pooled TLS socket, https.Agent#createConnection listeners pin the first ClientRequest and, through node-fetch, the body readers and settled body. Add an HTTPS keep-alive test on the production download path that asserts the downloaded buffer is collectable (WeakRef + gc); it fails without the cleanup, while plain HTTP retains nothing either way. Share the per-run localhost certificate fixture with the HttpConnection proxy tests. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- lib/result/CloudFetchResultHandler.ts | 8 +- tests/unit/.stubs/localhostCertificate.ts | 43 ++++++++++ .../connections/HttpConnection.test.ts | 38 +-------- .../result/CloudFetchResultHandler.test.ts | 85 +++++++++++++++++++ 4 files changed, 138 insertions(+), 36 deletions(-) create mode 100644 tests/unit/.stubs/localhostCertificate.ts diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 50a92b0b..94ec2442 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -155,8 +155,12 @@ export default class CloudFetchResultHandler implements IResultsProvider body.listeners(event)); const result = response.buffer(); diff --git a/tests/unit/.stubs/localhostCertificate.ts b/tests/unit/.stubs/localhostCertificate.ts new file mode 100644 index 00000000..ba14df44 --- /dev/null +++ b/tests/unit/.stubs/localhostCertificate.ts @@ -0,0 +1,43 @@ +import { execFileSync } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +// Self-signed `localhost` certificate, generated per run so no private key is committed. +// Returns `undefined` when openssl is not available. +export default function createLocalhostCertificate(): { cert: string; key: string } | undefined { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dbsql-localhost-tls-')); + try { + execFileSync( + 'openssl', + [ + 'req', + '-x509', + '-newkey', + 'ec', + '-pkeyopt', + 'ec_paramgen_curve:prime256v1', + '-nodes', + '-days', + '1', + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=DNS:localhost,IP:127.0.0.1', + '-keyout', + path.join(dir, 'key.pem'), + '-out', + path.join(dir, 'cert.pem'), + ], + { stdio: 'ignore' }, + ); + return { + cert: fs.readFileSync(path.join(dir, 'cert.pem'), 'utf8'), + key: fs.readFileSync(path.join(dir, 'key.pem'), 'utf8'), + }; + } catch { + return undefined; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} diff --git a/tests/unit/connection/connections/HttpConnection.test.ts b/tests/unit/connection/connections/HttpConnection.test.ts index 5dbfa5aa..5dc8b2ed 100644 --- a/tests/unit/connection/connections/HttpConnection.test.ts +++ b/tests/unit/connection/connections/HttpConnection.test.ts @@ -1,10 +1,6 @@ -import { execFileSync } from 'child_process'; -import fs from 'fs'; import http from 'http'; import https from 'https'; import net from 'net'; -import os from 'os'; -import path from 'path'; import tls from 'tls'; import { expect } from 'chai'; import { ProxyAgent } from 'proxy-agent'; @@ -12,6 +8,7 @@ import HttpConnection from '../../../../lib/connection/connections/HttpConnectio import ThriftHttpConnection from '../../../../lib/connection/connections/ThriftHttpConnection'; import IConnectionOptions from '../../../../lib/connection/contracts/IConnectionOptions'; import ClientContextStub from '../../.stubs/ClientContextStub'; +import createLocalhostCertificate from '../../.stubs/localhostCertificate'; describe('HttpConnection.connect', () => { it('should create Thrift connection', async () => { @@ -338,38 +335,11 @@ describe('HttpConnection through an HTTP CONNECT proxy', () => { const agents: Array = []; before(async function () { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dbsql-proxy-tls-')); - try { - execFileSync( - 'openssl', - [ - 'req', - '-x509', - '-newkey', - 'ec', - '-pkeyopt', - 'ec_paramgen_curve:prime256v1', - '-nodes', - '-days', - '1', - '-subj', - '/CN=localhost', - '-addext', - 'subjectAltName=DNS:localhost,IP:127.0.0.1', - '-keyout', - path.join(dir, 'key.pem'), - '-out', - path.join(dir, 'cert.pem'), - ], - { stdio: 'ignore' }, - ); - cert = fs.readFileSync(path.join(dir, 'cert.pem'), 'utf8'); - key = fs.readFileSync(path.join(dir, 'key.pem'), 'utf8'); - } catch { + const certificate = createLocalhostCertificate(); + if (!certificate) { this.skip(); // openssl is not available - } finally { - fs.rmSync(dir, { recursive: true, force: true }); } + ({ cert, key } = certificate); server = https.createServer({ cert, key, requestCert: true, rejectUnauthorized: false }, (req, res) => { serverRequests += 1; diff --git a/tests/unit/result/CloudFetchResultHandler.test.ts b/tests/unit/result/CloudFetchResultHandler.test.ts index 74d31781..e0260957 100644 --- a/tests/unit/result/CloudFetchResultHandler.test.ts +++ b/tests/unit/result/CloudFetchResultHandler.test.ts @@ -2,8 +2,11 @@ import { expect, AssertionError } from 'chai'; import sinon, { SinonStub } from 'sinon'; import { execFileSync } from 'child_process'; import http from 'http'; +import https from 'https'; import path from 'path'; import { PassThrough } from 'stream'; +import v8 from 'v8'; +import vm from 'vm'; import { AddressInfo } from 'net'; import Int64 from 'node-int64'; import { compressFrameSync } from 'lz4-napi'; @@ -22,6 +25,22 @@ import BaseClientContextStub from '../.stubs/ClientContextStub'; import { ClientConfig } from '../../../lib/contracts/IClientContext'; import ConnectionProviderStub from '../.stubs/ConnectionProviderStub'; import HttpRetryPolicy from '../../../lib/connection/connections/HttpRetryPolicy'; +import createLocalhostCertificate from '../.stubs/localhostCertificate'; + +// `WeakRef` is newer than the ES2018 lib this project compiles against. +const { WeakRef } = globalThis as unknown as { + WeakRef: new (target: T) => { deref(): T | undefined }; +}; + +async function collectGarbage(): Promise { + v8.setFlagsFromString('--expose-gc'); + const gc = vm.runInNewContext('gc') as () => void; + for (let i = 0; i < 3; i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } +} const sampleArrowSchema = Buffer.from([ 255, 255, 255, 255, 208, 0, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 14, 0, 6, 0, 13, 0, 8, 0, 10, 0, 0, 0, 0, 0, 4, 0, 16, 0, @@ -847,6 +866,72 @@ describe('CloudFetchResultHandler', () => { ); expect(output).to.eq('late body error handled\n'); }); + + describe('over HTTPS keep-alive', () => { + let server: https.Server | undefined; + let agent: https.Agent | undefined; + + afterEach(async () => { + agent?.destroy(); + agent = undefined; + if (server) { + server.closeAllConnections(); + await new Promise((resolve) => { + server!.close(() => resolve()); + }); + server = undefined; + } + }); + + // Real node-fetch, HttpRetryPolicy and a keep-alive https.Agent: the pooled TLS socket keeps its first request + // alive (see readBody), so only this path shows whether the downloaded body can be collected. + it('lets the downloaded body be garbage-collected while its TLS socket stays pooled', async function () { + this.timeout(15000); + const certificate = createLocalhostCertificate(); + if (!certificate) { + this.skip(); // openssl is not available + } + const payload = Buffer.alloc(1024 * 1024, 7); + server = https.createServer({ cert: certificate.cert, key: certificate.key }, (req, res) => res.end(payload)); + await new Promise((resolve) => { + server!.listen(0, '127.0.0.1', () => resolve()); + }); + agent = new https.Agent({ keepAlive: true, ca: certificate.cert }); + const fileLink = `https://127.0.0.1:${(server.address() as AddressInfo).port}/chunk`; + + const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, retriesTimeout: 10000 }); + sinon.stub(context.connectionProvider, 'getAgent').resolves(agent); + sinon.stub(context.connectionProvider, 'getRetryPolicy').callsFake(async () => new HttpRetryPolicy(context)); + + const downloaded = await (async () => { + const rowSet: TRowSet = { + startRowOffset: new Int64(0), + rows: [], + resultLinks: [ + { + fileLink, + expiryTime: new Int64(defaultLinkExpiryTime), + rowCount: new Int64(1), + startRowOffset: new Int64(0), + bytesNum: new Int64(0), + }, + ], + }; + const handler = new CloudFetchResultHandler(context, new ResultsProviderStub([rowSet], undefined), { + lz4Compressed: false, + status: { statusCode: TStatusCode.SUCCESS_STATUS }, + }); + const batch = await handler.fetchNext({ limit: 10000 }); + expect(batch.batches).to.have.length(1); + expect(batch.batches[0].equals(payload)).to.be.true; + return new WeakRef(batch.batches[0]); + })(); + + expect(Object.values(agent.freeSockets).flat()).to.have.length(1); + await collectGarbage(); + expect(downloaded.deref()).to.be.undefined; + }); + }); }); describe('download deadline', () => {