Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
- `customHeaders` are now sent on every Thrift request (session, statement and metadata calls), as the kernel backend already did, instead of only on telemetry and feature-flag requests. On these requests, caller values for driver-managed headers (`Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`) are ignored on both backends, and the Thrift backend rejects header names or values that cannot be sent at connect time. (#491)
- Error telemetry now reports only the error class name and the server SQLSTATE. It no longer includes the error message or stack trace, which could contain table and column names from the query. On the Thrift backend, `statement.complete` telemetry now also reports the result delivery format (inline Arrow, inline columnar, or CloudFetch) and whether the server LZ4-compressed the result. (#508)
- Avoid an extra full-buffer copy when downloading Thrift CloudFetch results.
- Release completed Thrift CloudFetch body-reader listeners so idle connections do not retain downloaded result buffers.

## 3.0.0

Expand Down
31 changes: 29 additions & 2 deletions lib/result/CloudFetchResultHandler.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch';
import { Readable } from 'stream';
import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types';
import HiveDriverError from '../errors/HiveDriverError';
import RetryError, { RetryErrorCode } from '../errors/RetryError';
Expand Down Expand Up @@ -149,6 +150,32 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
};
}

private async readBody(response: Response): Promise<Buffer> {
const { body } = response;
if (!(body instanceof Readable)) {
return response.buffer();
}
// Works around a Node core retention: https.Agent#createConnection's 'session'/'close' socket listeners share a
// closure context with its `cb` (captured by the proxy-tunnel cleanupAndPropagate), so a pooled TLS socket pins
// its first ClientRequest and, through node-fetch's request 'error' closure and the Response, these body readers
// and the settled body. Plain HTTP and later requests on the same socket aren't pinned. node-fetch installs the
// readers synchronously: remove only those, keeping the constructor's error handler for late stream errors.
// This can go once Node no longer captures `cb` in those listeners.
const events = ['data', 'end', 'error'];
const existing = events.map((event) => body.listeners(event));
const result = response.buffer();
const added = events.map((event, index) =>
body.listeners(event).filter((listener) => !existing[index].includes(listener)),
);
try {
return await result;
} finally {
events.forEach((event, index) => {
added[index].forEach((listener) => body.removeListener(event, listener as () => void));
});
}
}

private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: Buffer }> {
const connectionProvider = await this.context.getConnectionProvider();
let requestUrl = url;
Expand Down Expand Up @@ -194,7 +221,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
try {
const response = await fetch(request);
if (response.ok) {
downloaded = await response.buffer();
downloaded = await this.readBody(response);
} else {
lastFailure = { request, response };
}
Expand Down Expand Up @@ -232,7 +259,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
// callback). In production the body is always read inside the retried
// block above, so this path is a no-op.
if (downloaded === undefined && result.response.ok) {
downloaded = await result.response.buffer();
downloaded = await this.readBody(result.response);
}
return { response: result.response, body: downloaded };
}
Expand Down
43 changes: 43 additions & 0 deletions tests/unit/.stubs/localhostCertificate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { execFileSync } from 'child_process';
import fs from 'fs';
import os from 'os';
import path from 'path';

// Self-signed `localhost` certificate, generated per run so no private key is committed.
// Returns `undefined` when openssl is not available.
export default function createLocalhostCertificate(): { cert: string; key: string } | undefined {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dbsql-localhost-tls-'));
try {
execFileSync(
'openssl',
[
'req',
'-x509',
'-newkey',
'ec',
'-pkeyopt',
'ec_paramgen_curve:prime256v1',
'-nodes',
'-days',
'1',
'-subj',
'/CN=localhost',
'-addext',
'subjectAltName=DNS:localhost,IP:127.0.0.1',
'-keyout',
path.join(dir, 'key.pem'),
'-out',
path.join(dir, 'cert.pem'),
],
{ stdio: 'ignore' },
);
return {
cert: fs.readFileSync(path.join(dir, 'cert.pem'), 'utf8'),
key: fs.readFileSync(path.join(dir, 'key.pem'), 'utf8'),
};
} catch {
return undefined;
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}
38 changes: 4 additions & 34 deletions tests/unit/connection/connections/HttpConnection.test.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
import { execFileSync } from 'child_process';
import fs from 'fs';
import http from 'http';
import https from 'https';
import net from 'net';
import os from 'os';
import path from 'path';
import tls from 'tls';
import { expect } from 'chai';
import { ProxyAgent } from 'proxy-agent';
import HttpConnection from '../../../../lib/connection/connections/HttpConnection';
import ThriftHttpConnection from '../../../../lib/connection/connections/ThriftHttpConnection';
import IConnectionOptions from '../../../../lib/connection/contracts/IConnectionOptions';
import ClientContextStub from '../../.stubs/ClientContextStub';
import createLocalhostCertificate from '../../.stubs/localhostCertificate';

describe('HttpConnection.connect', () => {
it('should create Thrift connection', async () => {
Expand Down Expand Up @@ -338,38 +335,11 @@ describe('HttpConnection through an HTTP CONNECT proxy', () => {
const agents: Array<http.Agent | undefined> = [];

before(async function () {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dbsql-proxy-tls-'));
try {
execFileSync(
'openssl',
[
'req',
'-x509',
'-newkey',
'ec',
'-pkeyopt',
'ec_paramgen_curve:prime256v1',
'-nodes',
'-days',
'1',
'-subj',
'/CN=localhost',
'-addext',
'subjectAltName=DNS:localhost,IP:127.0.0.1',
'-keyout',
path.join(dir, 'key.pem'),
'-out',
path.join(dir, 'cert.pem'),
],
{ stdio: 'ignore' },
);
cert = fs.readFileSync(path.join(dir, 'cert.pem'), 'utf8');
key = fs.readFileSync(path.join(dir, 'key.pem'), 'utf8');
} catch {
const certificate = createLocalhostCertificate();
if (!certificate) {
this.skip(); // openssl is not available
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
({ cert, key } = certificate);

server = https.createServer({ cert, key, requestCert: true, rejectUnauthorized: false }, (req, res) => {
serverRequests += 1;
Expand Down
185 changes: 185 additions & 0 deletions tests/unit/result/CloudFetchResultHandler.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import { expect, AssertionError } from 'chai';
import sinon, { SinonStub } from 'sinon';
import { execFileSync } from 'child_process';
import http from 'http';
import https from 'https';
import path from 'path';
import { PassThrough } from 'stream';
import v8 from 'v8';
import vm from 'vm';
import { AddressInfo } from 'net';
import Int64 from 'node-int64';
import { compressFrameSync } from 'lz4-napi';
Expand All @@ -19,6 +25,22 @@ import BaseClientContextStub from '../.stubs/ClientContextStub';
import { ClientConfig } from '../../../lib/contracts/IClientContext';
import ConnectionProviderStub from '../.stubs/ConnectionProviderStub';
import HttpRetryPolicy from '../../../lib/connection/connections/HttpRetryPolicy';
import createLocalhostCertificate from '../.stubs/localhostCertificate';

// `WeakRef` is newer than the ES2018 lib this project compiles against.
const { WeakRef } = globalThis as unknown as {
WeakRef: new <T extends object>(target: T) => { deref(): T | undefined };
};

async function collectGarbage(): Promise<void> {
v8.setFlagsFromString('--expose-gc');
const gc = vm.runInNewContext('gc') as () => void;
for (let i = 0; i < 3; i += 1) {
// eslint-disable-next-line no-await-in-loop
await new Promise((resolve) => setImmediate(resolve));
gc();
}
}

const sampleArrowSchema = Buffer.from([
255, 255, 255, 255, 208, 0, 0, 0, 16, 0, 0, 0, 0, 0, 10, 0, 14, 0, 6, 0, 13, 0, 8, 0, 10, 0, 0, 0, 0, 0, 4, 0, 16, 0,
Expand Down Expand Up @@ -752,6 +774,166 @@ describe('CloudFetchResultHandler', () => {
},
);

describe('response body retention', () => {
function createHandler(response: Response) {
const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 1 });
context.invokeWithRetryStub.resolves({ request: new Request('http://localhost'), response });
return new CloudFetchResultHandler(context, new ResultsProviderStub([sampleRowSet1], undefined), {
status: { statusCode: TStatusCode.SUCCESS_STATUS },
});
}

it('releases consumed body listeners without removing existing or later listeners', async () => {
const body = new PassThrough();
const response = new Response(body);
const existing = ['data', 'end', 'error'].map((event) => {
const listener = sinon.spy();
body.on(event, listener);
return listener;
});
const later = sinon.spy();
body.once('data', () => body.on('error', later));
const errorsBefore = body.listeners('error');
const handler = createHandler(response);
setImmediate(() => body.end(sampleArrowBatch));

const batch = await handler.fetchNext({ limit: 10000 });

expect(batch.batches).to.deep.eq([sampleArrowBatch]);
expect(response.bodyUsed).to.be.true;
expect(body.listeners('data')).to.deep.eq([existing[0]]);
expect(body.listeners('end')).to.deep.eq([existing[1]]);
expect(body.listeners('error')).to.deep.eq([...errorsBefore, later]);
body.emit('error', new Error('late stream error'));
expect(existing[2].calledOnce).to.be.true;
expect(later.calledOnce).to.be.true;
});

it('releases body reader listeners when consumption rejects', async () => {
const body = new PassThrough();
const response = new Response(body);
const errorsBefore = body.listeners('error');
const handler = createHandler(response);
setImmediate(() => body.destroy(new Error('body interrupted')));

let thrown: unknown;
try {
await handler.fetchNext({ limit: 10000 });
} catch (error) {
thrown = error;
}

expect(thrown).to.be.instanceOf(Error);
expect(response.bodyUsed).to.be.true;
expect(body.listenerCount('data')).to.eq(0);
expect(body.listenerCount('end')).to.eq(0);
expect(body.listeners('error')).to.deep.eq(errorsBefore);
});

it('handles late body errors without application error handlers or unhandled rejections', function () {
this.timeout(15000);
const repoRoot = path.resolve(__dirname, '../../..');
const handlerPath = path.join(repoRoot, 'lib/result/CloudFetchResultHandler');
const script = `
const assert = require('assert').strict;
const { PassThrough } = require('stream');
const { Response } = require('node-fetch');
const Handler = require(${JSON.stringify(handlerPath)}).default;
(async () => {
const body = new PassThrough();
const response = new Response(body);
const originalErrors = body.listeners('error');
assert.equal(originalErrors.length, 1);
const reading = Handler.prototype.readBody.call({}, response);
body.end(Buffer.from('result bytes'));
assert.equal((await reading).toString(), 'result bytes');
assert.equal(response.bodyUsed, true);
assert.equal(body.listenerCount('data'), 0);
assert.equal(body.listenerCount('end'), 0);
await new Promise((resolve) => setImmediate(() => {
body.emit('error', new Error('late body error'));
resolve();
}));
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(body.listeners('error'), originalErrors);
process.stdout.write('late body error handled\\n');
})().catch((error) => { console.error(error); process.exitCode = 1; });
`;
const output = execFileSync(
process.execPath,
['--unhandled-rejections=strict', '--require', require.resolve('ts-node/register'), '-e', script],
{ cwd: repoRoot, encoding: 'utf8', timeout: 10000, stdio: 'pipe' },
);
expect(output).to.eq('late body error handled\n');
});

describe('over HTTPS keep-alive', () => {
let server: https.Server | undefined;
let agent: https.Agent | undefined;

afterEach(async () => {
agent?.destroy();
agent = undefined;
if (server) {
server.closeAllConnections();
await new Promise<void>((resolve) => {
server!.close(() => resolve());
});
server = undefined;
}
});

// Real node-fetch, HttpRetryPolicy and a keep-alive https.Agent: the pooled TLS socket keeps its first request
// alive (see readBody), so only this path shows whether the downloaded body can be collected.
it('lets the downloaded body be garbage-collected while its TLS socket stays pooled', async function () {
this.timeout(15000);
const certificate = createLocalhostCertificate();
if (!certificate) {
this.skip(); // openssl is not available
}
const payload = Buffer.alloc(1024 * 1024, 7);
server = https.createServer({ cert: certificate.cert, key: certificate.key }, (req, res) => res.end(payload));
await new Promise<void>((resolve) => {
server!.listen(0, '127.0.0.1', () => resolve());
});
agent = new https.Agent({ keepAlive: true, ca: certificate.cert });
const fileLink = `https://127.0.0.1:${(server.address() as AddressInfo).port}/chunk`;

const context = new BaseClientContextStub({ cloudFetchConcurrentDownloads: 1, retriesTimeout: 10000 });
sinon.stub(context.connectionProvider, 'getAgent').resolves(agent);
sinon.stub(context.connectionProvider, 'getRetryPolicy').callsFake(async () => new HttpRetryPolicy(context));

const downloaded = await (async () => {
const rowSet: TRowSet = {
startRowOffset: new Int64(0),
rows: [],
resultLinks: [
{
fileLink,
expiryTime: new Int64(defaultLinkExpiryTime),
rowCount: new Int64(1),
startRowOffset: new Int64(0),
bytesNum: new Int64(0),
},
],
};
const handler = new CloudFetchResultHandler(context, new ResultsProviderStub([rowSet], undefined), {
lz4Compressed: false,
status: { statusCode: TStatusCode.SUCCESS_STATUS },
});
const batch = await handler.fetchNext({ limit: 10000 });
expect(batch.batches).to.have.length(1);
expect(batch.batches[0].equals(payload)).to.be.true;
return new WeakRef(batch.batches[0]);
})();

expect(Object.values(agent.freeSockets).flat()).to.have.length(1);
await collectGarbage();
expect(downloaded.deref()).to.be.undefined;
});
});
});

describe('download deadline', () => {
let server: http.Server | undefined;

Expand Down Expand Up @@ -840,6 +1022,9 @@ describe('CloudFetchResultHandler', () => {
expect(buffer.calledOnce).to.be.true;
expect(arrayBuffer.called).to.be.false;
expect(buffer.firstCall.thisValue.body).to.have.property('pipe').that.is.a('function');
expect(buffer.firstCall.thisValue.body.listenerCount('data')).to.eq(0);
expect(buffer.firstCall.thisValue.body.listenerCount('end')).to.eq(0);
expect(buffer.firstCall.thisValue.body.listenerCount('error')).to.eq(1);
const downloaded = await buffer.firstCall.returnValue;
expect(downloaded).to.deep.eq(body);
expect(downloaded.length).to.be.lt(Buffer.poolSize >>> 1);
Expand Down
Loading