From 18dcabdb924e45cbd59e72563e827b0bee977eaa Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 08:16:53 +0000 Subject: [PATCH 1/9] Decode all LZ4 frames and enable Thrift CloudFetch compression Bound every frame before decoding, preserve CloudFetch frame buffers, and validate decoded row counts before exposing results. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + lib/result/CloudFetchResultHandler.ts | 10 ++- lib/thrift-backend/ThriftSessionBackend.ts | 2 +- lib/utils/lz4.ts | 48 +++++++++++- tests/e2e/cloudfetch.test.ts | 7 +- tests/e2e/lz4.test.ts | 61 +++++++++++++++ tests/unit/DBSQLSession.test.ts | 29 +++++--- tests/unit/result/CompressedArrow.test.ts | 86 ++++++++++++++++++++++ tests/unit/utils/fixtures/lz4-frames.json | 66 +++++++++++++++++ tests/unit/utils/lz4.test.ts | 60 ++++++++++++++- 10 files changed, 351 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/lz4.test.ts create mode 100644 tests/unit/result/CompressedArrow.test.ts create mode 100644 tests/unit/utils/fixtures/lz4-frames.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 48139e6b..beff6d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Decode every concatenated LZ4 frame in Thrift Arrow results and enable LZ4 for CloudFetch when the native codec is available. Set `useLZ4Compression: false` on the client or statement to disable compression. - Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 71d8c44b..11f3d93c 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,4 +1,5 @@ import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; +import { RecordBatchReader } from 'apache-arrow'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; import RetryError, { RetryErrorCode } from '../errors/RetryError'; @@ -96,7 +97,14 @@ export default class CloudFetchResultHandler implements IResultsProvider LZ4()!.decode(buffer)); + batch.batches = batch.batches.flatMap((buffer) => LZ4()!.decodeFrames(buffer)); + let decodedRowCount = 0; + for (const recordBatch of RecordBatchReader.from(batch.batches)) { + decodedRowCount += recordBatch.numRows; + } + if (decodedRowCount !== batch.rowCount) { + throw new HiveDriverError(`CloudFetch row count mismatch: expected ${batch.rowCount}, got ${decodedRowCount}`); + } } return batch; } diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index 49ec539a..6695ac0c 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -220,7 +220,7 @@ export default class ThriftSessionBackend implements ISessionBackend { request.canDownloadResult = options.useCloudFetch ?? clientConfig.useCloudFetch; } - if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion) && request.canDownloadResult !== true) { + if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion)) { request.canDecompressLZ4Result = (options.useLZ4Compression ?? clientConfig.useLZ4Compression) && Boolean(LZ4()); } diff --git a/lib/utils/lz4.ts b/lib/utils/lz4.ts index 5a94d10f..d46b249a 100644 --- a/lib/utils/lz4.ts +++ b/lib/utils/lz4.ts @@ -14,6 +14,8 @@ // and cross-decoding both directions. We expose the same `{ encode, decode }` // surface the rest of the driver already calls, so call sites are unchanged. +import HiveDriverError from '../errors/HiveDriverError'; + type Lz4Napi = { compressFrameSync(data: Buffer): Buffer; decompressFrameSync(data: Buffer): Buffer; @@ -23,6 +25,46 @@ type Lz4Napi = { export interface LZ4Codec { encode(data: Buffer): Buffer; decode(data: Buffer): Buffer; + decodeFrames(data: Buffer): Array; +} + +function decodeFrames(data: Buffer, napi: Lz4Napi): Array { + const frames: Array = []; + let offset = 0; + const advance = (size: number) => { + offset += size; + if (offset > data.length) { + throw new HiveDriverError('Truncated LZ4 frame'); + } + }; + + while (offset < data.length) { + const start = offset; + advance(7); + if (data.readUInt32LE(start) !== 0x184d2204) { + throw new HiveDriverError('Invalid LZ4 frame magic'); + } + const flags = data[start + 4]; + if (flags & 0x08) advance(8); + if (flags & 0x01) advance(4); + + let blockSize: number; + do { + advance(4); + blockSize = data.readUInt32LE(offset - 4); + if (blockSize !== 0) { + advance((blockSize & 0x7fffffff) + (flags & 0x10 ? 4 : 0)); + } + } while (blockSize !== 0); + if (flags & 0x04) advance(4); + + try { + frames.push(napi.decompressFrameSync(data.subarray(start, offset))); + } catch { + throw new HiveDriverError('Invalid LZ4 frame'); + } + } + return frames; } function tryLoadLZ4Module(): LZ4Codec | undefined { @@ -47,7 +89,11 @@ function tryLoadLZ4Module(): LZ4Codec | undefined { // hot path (once per Arrow batch). return { encode: (data: Buffer): Buffer => napi.compressFrameSync(data), - decode: (data: Buffer): Buffer => napi.decompressFrameSync(data), + decode: (data: Buffer): Buffer => { + const frames = decodeFrames(data, napi); + return frames.length === 1 ? frames[0] : Buffer.concat(frames); + }, + decodeFrames: (data: Buffer): Array => decodeFrames(data, napi), }; } diff --git a/tests/e2e/cloudfetch.test.ts b/tests/e2e/cloudfetch.test.ts index 68a7a58c..2f379022 100644 --- a/tests/e2e/cloudfetch.test.ts +++ b/tests/e2e/cloudfetch.test.ts @@ -100,11 +100,11 @@ describe('CloudFetch', () => { expect(fetchedRowCount).to.be.equal(queriedRowsCount); }); - it('should not use LZ4 compression with cloud fetch', async () => { + it('should use LZ4 compression with cloud fetch', async () => { const cloudFetchConcurrentDownloads = 5; const session = await openSession({ cloudFetchConcurrentDownloads, - useLZ4Compression: true, // This is ignored when cloud fetch is enabled + useLZ4Compression: true, }); const queriedRowsCount = 10000000; // result has to be quite big to enable CloudFetch @@ -129,8 +129,7 @@ describe('CloudFetch', () => { expect(resultHandler).to.be.instanceof(ResultSlicer); expect(resultHandler.source).to.be.instanceof(ArrowResultConverter); expect(resultHandler.source.source).to.be.instanceOf(CloudFetchResultHandler); - // LZ4 compression should not be enabled with cloud fetch - expect(resultHandler.source.source.isLZ4Compressed).to.be.false; + expect(resultHandler.source.source.isLZ4Compressed).to.be.true; const chunk = await operation.fetchChunk({ maxRows: 100000, disableBuffering: true }); expect(chunk.length).to.be.gt(0); diff --git a/tests/e2e/lz4.test.ts b/tests/e2e/lz4.test.ts new file mode 100644 index 00000000..d16f3a32 --- /dev/null +++ b/tests/e2e/lz4.test.ts @@ -0,0 +1,61 @@ +import { createHash } from 'crypto'; +import { expect } from 'chai'; +import { DBSQLClient } from '../../lib'; +import CloudFetchResultHandler from '../../lib/result/CloudFetchResultHandler'; +import IOperation from '../../lib/contracts/IOperation'; +import config from './utils/config'; + +describe('LZ4 result parity', () => { + for (const useCloudFetch of [false, true]) { + it(`preserves every value with CloudFetch=${useCloudFetch}`, async () => { + const client = new DBSQLClient(); + await client.connect({ host: config.host, path: config.path, token: config.token }); + let operation: IOperation | undefined; + const session = await client.openSession(); + try { + const read = async (useLZ4Compression: boolean) => { + operation = await session.executeStatement( + `SELECT id, IF(id % 7 = 0, NULL, id * 0.125) AS value, + IF(id % 5 = 0, NULL, sha2(CAST(id AS STRING), 512)) AS text, + IF(id % 3 = 0, NULL, id % 2 = 0) AS flag + FROM range(500000) ORDER BY id`, + { useCloudFetch, useLZ4Compression, maxRows: null }, + ); + await operation.finished(); + const handler = await (operation as any).getResultHandler(); + expect(handler.source.source.isLZ4Compressed).to.equal(useLZ4Compression); + expect(handler.source.source instanceof CloudFetchResultHandler).to.equal(useCloudFetch); + const hash = createHash('sha256'); + let count = 0; + for await (const chunk of operation.iterateChunks({ maxRows: 11111 })) { + for (const row of chunk as Array<{ + id: number; + value: number | null; + text: string | null; + flag: boolean | null; + }>) { + expect(row.id).to.equal(count); + expect(row.value).to.equal(count % 7 === 0 ? null : count * 0.125); + expect(row.text).to.equal( + count % 5 === 0 ? null : createHash('sha512').update(String(count)).digest('hex'), + ); + expect(row.flag).to.equal(count % 3 === 0 ? null : count % 2 === 0); + hash.update(JSON.stringify(row)); + count += 1; + } + } + expect(count).to.equal(500000); + await operation.close(); + operation = undefined; + return hash.digest('hex'); + }; + const uncompressed = await read(false); + expect(await read(true)).to.equal(uncompressed); + } finally { + await operation?.close(); + await session.close(); + await client.close(); + } + }); + } +}); diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 00498c54..9381a226 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -165,12 +165,7 @@ describe('DBSQLSession', () => { } if (version >= TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6) { - // Since cloud fetch is enabled, canDecompressLZ4Result should not be set - if (req.canDownloadResult === true) { - expect(req.canDecompressLZ4Result).to.not.be.true; - } else { - expect(req.canDecompressLZ4Result).to.be.true; - } + expect(req.canDecompressLZ4Result).to.be.true; } else { expect(req.canDecompressLZ4Result).to.not.be.true; } @@ -194,7 +189,24 @@ describe('DBSQLSession', () => { }); describe('LZ4 compression with cloud fetch', () => { - it('should not set canDecompressLZ4Result when cloud fetch is enabled (canDownloadResult=true)', async () => { + for (const clientSetting of [false, true]) { + for (const statementSetting of [undefined, false, true]) { + it(`honors client=${clientSetting}, statement=${statementSetting}`, async () => { + const context = new ClientContextStub({ useLZ4Compression: clientSetting }); + const driver = sinon.spy(context.driver); + const session = createSessionForTest({ + handle: sessionHandleStub, + context, + serverProtocolVersion: TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6, + }); + await session.executeStatement('SELECT 1', { useCloudFetch: true, useLZ4Compression: statementSetting }); + expect(driver.executeStatement.firstCall.args[0].canDecompressLZ4Result).to.equal( + statementSetting ?? clientSetting, + ); + }); + } + } + it('should set canDecompressLZ4Result when cloud fetch is enabled (canDownloadResult=true)', async () => { const context = new ClientContextStub({ useLZ4Compression: true }); const driver = sinon.spy(context.driver); const statement = 'SELECT * FROM table'; @@ -212,9 +224,8 @@ describe('DBSQLSession', () => { expect(driver.executeStatement.callCount).to.eq(1); const req = driver.executeStatement.firstCall.args[0]; - // canDownloadResult should be true and canDecompressLZ4Result should NOT be set expect(req.canDownloadResult).to.be.true; - expect(req.canDecompressLZ4Result).to.not.be.true; + expect(req.canDecompressLZ4Result).to.be.true; }); it('should set canDecompressLZ4Result when cloud fetch is disabled (canDownloadResult=false)', async () => { diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts new file mode 100644 index 00000000..12b6f184 --- /dev/null +++ b/tests/unit/result/CompressedArrow.test.ts @@ -0,0 +1,86 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import Int64 from 'node-int64'; +import { Binary, Bool, Float64, Table, Utf8, tableFromIPC, tableToIPC, vectorFromArray } from 'apache-arrow'; +import { compressFrameSync } from 'lz4-napi'; +import ArrowResultHandler from '../../../lib/result/ArrowResultHandler'; +import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; +import { TStatusCode } from '../../../thrift/TCLIService_types'; +import ResultsProviderStub from '../.stubs/ResultsProviderStub'; +import ClientContextStub from '../.stubs/ClientContextStub'; + +describe('concatenated LZ4 Arrow results', () => { + const table = new Table({ + number: vectorFromArray([1, null, 3.5], new Float64()), + text: vectorFromArray(['first', 'second', null], new Utf8()), + boolean: vectorFromArray([true, null, false], new Bool()), + binary: vectorFromArray([Buffer.from([0, 255]), null, Buffer.alloc(0)], new Binary()), + }); + const ipc = Buffer.from(tableToIPC(table)); + const schemaEnd = 8 + ipc.readInt32LE(4); + const schema = ipc.subarray(0, schemaEnd); + const payload = ipc.subarray(schemaEnd); + const compress = (bytes: Buffer) => + Buffer.concat([ + compressFrameSync(bytes.subarray(0, 16)), + compressFrameSync(Buffer.alloc(0)), + compressFrameSync(bytes.subarray(16)), + ]); + + it('preserves inline values, types, nulls and bytes across frame boundaries', async () => { + const handler = new ArrowResultHandler( + new ClientContextStub(), + new ResultsProviderStub( + [ + { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [{ batch: compress(payload), rowCount: new Int64(3) }], + }, + ], + undefined, + ), + { arrowSchema: schema, lz4Compressed: true, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, + ); + const result = await handler.fetchNext({ limit: 1 }); + const bytes = Buffer.concat(result.batches); + expect(bytes).to.deep.equal(ipc); + expect(result.rowCount).to.equal(3); + expect(tableFromIPC(bytes).toArray()).to.deep.equal(table.toArray()); + expect(tableFromIPC(bytes).schema).to.deep.equal(table.schema); + }); + + for (const expectedRows of [0, 2, 3, 4]) { + it(`validates CloudFetch row count ${expectedRows} against all decoded frames`, async () => { + const link = { + fileLink: 'http://localhost/result', + rowCount: new Int64(expectedRows), + startRowOffset: new Int64(0), + expiryTime: new Int64(Date.now() + 60000), + bytesNum: new Int64(0), + }; + const handler = new CloudFetchResultHandler( + new ClientContextStub(), + new ResultsProviderStub([{ startRowOffset: new Int64(0), rows: [], resultLinks: [link] }], undefined), + { lz4Compressed: true, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, + ); + sinon.stub(handler as any, 'downloadLink').resolves({ batches: [compress(ipc)], rowCount: expectedRows }); + if (expectedRows === 3) { + const result = await handler.fetchNext({ limit: 1 }); + expect(Buffer.concat(result.batches)).to.deep.equal(ipc); + expect(result.batches).to.have.length(3); + expect(result.rowCount).to.equal(3); + } else { + let caught: unknown; + try { + await handler.fetchNext({ limit: 1 }); + } catch (error) { + caught = error; + } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal(`CloudFetch row count mismatch: expected ${expectedRows}, got 3`); + } + }); + } +}); diff --git a/tests/unit/utils/fixtures/lz4-frames.json b/tests/unit/utils/fixtures/lz4-frames.json new file mode 100644 index 00000000..02f6b979 --- /dev/null +++ b/tests/unit/utils/fixtures/lz4-frames.json @@ -0,0 +1,66 @@ +[ + { + "linked": false, + "contentChecksum": false, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1868408884020000000000862c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637382c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536b0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a00000000" + }, + { + "linked": false, + "contentChecksum": false, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1878408884020000000000e62c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab2c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536d1e5fa6fb0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a0d4c81a400000000" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d186c408884020000000000482c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637382c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536b0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a00000000b5dab6f5" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d187c408884020000000000312c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab2c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536d1e5fa6fb0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a0d4c81a400000000b5dab6f5" + }, + { + "linked": true, + "contentChecksum": false, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1848408884020000000000102c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637380a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe85032333435368e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390a00000000" + }, + { + "linked": true, + "contentChecksum": false, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1858408884020000000000942c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab0a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe850323334353698d1b0d08e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390ae007cfb600000000" + }, + { + "linked": true, + "contentChecksum": true, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d184c408884020000000000fe2c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637380a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe85032333435368e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390a00000000b5dab6f5" + }, + { + "linked": true, + "contentChecksum": true, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d185c4088840200000000001e2c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab0a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe850323334353698d1b0d08e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390ae007cfb600000000b5dab6f5" + } +] diff --git a/tests/unit/utils/lz4.test.ts b/tests/unit/utils/lz4.test.ts index 95713e47..598adbfc 100644 --- a/tests/unit/utils/lz4.test.ts +++ b/tests/unit/utils/lz4.test.ts @@ -1,5 +1,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; +import fs from 'fs'; +import path from 'path'; import getLZ4 from '../../../lib/utils/lz4'; // Exercises the lz4 loader (lib/utils/lz4.ts), which now wraps `lz4-napi` @@ -69,9 +71,11 @@ describe('lz4 module loader', function () { it('exposes an { encode, decode } codec backed by lz4-napi frame APIs', () => { // Fake lz4-napi: frame APIs the adapter calls. + const testData = Buffer.from('Hello, World!'); + const encoded = Buffer.from([4, 34, 77, 24, 96, 64, 130, 0, 0, 0, 0]); const fakeNapi = { - compressFrameSync: (buf: Buffer) => Buffer.from(`frame:${buf.toString()}`), - decompressFrameSync: (buf: Buffer) => Buffer.from(buf.toString().replace('frame:', '')), + compressFrameSync: sinon.stub().returns(encoded), + decompressFrameSync: sinon.stub().returns(testData), }; mockNapiLoad(fakeNapi); @@ -81,10 +85,10 @@ describe('lz4 module loader', function () { expect(lz4Module.encode).to.be.a('function'); expect(lz4Module.decode).to.be.a('function'); - const testData = Buffer.from('Hello, World!'); const roundTripped = lz4Module.decode(lz4Module.encode(testData)); expect(roundTripped.toString()).to.equal('Hello, World!'); expect(consoleWarnStub.called).to.be.false; + expect(fakeNapi.decompressFrameSync.calledOnceWithExactly(encoded)).to.be.true; }); it('returns undefined (no warning) when lz4-napi is not installed (MODULE_NOT_FOUND)', () => { @@ -194,4 +198,54 @@ describe('lz4 frame codec (real lz4-napi)', function () { expect(codec!.decode(encoded).equals(payload)).to.be.true; } }); + + it('decodes every concatenated frame, including empty frames', () => { + const payloads = [Buffer.from('first'), Buffer.alloc(0), Buffer.from('second'), Buffer.from('third')]; + const input = Buffer.concat(payloads.map((payload) => codec!.encode(payload))); + expect(codec!.decode(input)).to.deep.equal(Buffer.concat(payloads)); + }); + + it('rejects truncated frames and trailing bytes instead of returning partial results', () => { + const frame = codec!.encode(Buffer.from('first')); + for (let length = 1; length < frame.length; length += 1) { + expect(() => codec!.decode(frame.subarray(0, length)), `truncated at ${length}`).to.throw(); + expect(() => codec!.decode(Buffer.concat([frame, frame.subarray(0, length)]))).to.throw(); + } + expect(() => codec!.decode(Buffer.concat([frame, Buffer.from('trailing bytes')]))).to.throw(); + }); + + const fixtures: Array<{ + linked: boolean; + contentChecksum: boolean; + blockChecksum: boolean; + text: string; + repeat: number; + hex: string; + }> = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures/lz4-frames.json'), 'utf8')); + + for (const fixture of fixtures) { + const name = `linked=${fixture.linked}, content checksum=${fixture.contentChecksum}, block checksum=${fixture.blockChecksum}`; + it(`decodes reference liblz4 frames (${name})`, () => { + const frame = Buffer.from(fixture.hex, 'hex'); + const payload = Buffer.from(fixture.text.repeat(fixture.repeat)); + expect(codec!.decode(Buffer.concat([frame, frame]))).to.deep.equal(Buffer.concat([payload, payload])); + expect(codec!.decodeFrames(frame)).to.deep.equal([payload]); + }); + + it(`rejects corrupt reference liblz4 frames (${name})`, () => { + const frame = Buffer.from(fixture.hex, 'hex'); + const offsets = [0, 14]; + if (fixture.contentChecksum) offsets.push(frame.length - 1); + if (fixture.blockChecksum) offsets.push(19 + (frame.readUInt32LE(15) & 0x7fffffff)); + for (const offset of offsets) { + const corrupt = Buffer.from(frame); + corrupt[offset] ^= 1; + expect(() => codec!.decode(corrupt)).to.throw(); + expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw(); + } + for (let missing = 1; missing <= 8; missing += 1) { + expect(() => codec!.decode(frame.subarray(0, -missing))).to.throw(); + } + }); + } }); From dcb45210cb89df2dad8ecf9d43437db949a0d37f Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 08:27:52 +0000 Subject: [PATCH 2/9] Cover empty and raw checksummed frames with reference fixtures Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/e2e/lz4.test.ts | 11 +++++++---- tests/unit/utils/fixtures/lz4-frames.json | 16 ++++++++++++++++ tests/unit/utils/lz4.test.ts | 11 ++++++++--- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/e2e/lz4.test.ts b/tests/e2e/lz4.test.ts index d16f3a32..ac29231c 100644 --- a/tests/e2e/lz4.test.ts +++ b/tests/e2e/lz4.test.ts @@ -3,18 +3,21 @@ import { expect } from 'chai'; import { DBSQLClient } from '../../lib'; import CloudFetchResultHandler from '../../lib/result/CloudFetchResultHandler'; import IOperation from '../../lib/contracts/IOperation'; +import IDBSQLSession from '../../lib/contracts/IDBSQLSession'; import config from './utils/config'; describe('LZ4 result parity', () => { for (const useCloudFetch of [false, true]) { it(`preserves every value with CloudFetch=${useCloudFetch}`, async () => { const client = new DBSQLClient(); - await client.connect({ host: config.host, path: config.path, token: config.token }); let operation: IOperation | undefined; - const session = await client.openSession(); + let session: IDBSQLSession | undefined; try { + await client.connect({ host: config.host, path: config.path, token: config.token }); + const activeSession = await client.openSession(); + session = activeSession; const read = async (useLZ4Compression: boolean) => { - operation = await session.executeStatement( + operation = await activeSession.executeStatement( `SELECT id, IF(id % 7 = 0, NULL, id * 0.125) AS value, IF(id % 5 = 0, NULL, sha2(CAST(id AS STRING), 512)) AS text, IF(id % 3 = 0, NULL, id % 2 = 0) AS flag @@ -53,7 +56,7 @@ describe('LZ4 result parity', () => { expect(await read(true)).to.equal(uncompressed); } finally { await operation?.close(); - await session.close(); + await session?.close(); await client.close(); } }); diff --git a/tests/unit/utils/fixtures/lz4-frames.json b/tests/unit/utils/fixtures/lz4-frames.json index 02f6b979..25ca25f4 100644 --- a/tests/unit/utils/fixtures/lz4-frames.json +++ b/tests/unit/utils/fixtures/lz4-frames.json @@ -62,5 +62,21 @@ "text": "Databricks LZ4 frame: 0123456789\n", "repeat": 5000, "hex": "04224d185c4088840200000000001e2c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab0a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe850323334353698d1b0d08e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390ae007cfb600000000b5dab6f5" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": true, + "text": "", + "repeat": 1, + "hex": "04224d187440bd00000000055dcc02" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": true, + "text": "].4U85YnN0C8 zu7!6-n#OXNZSRgRx)+wS\"ENL>=D22>&8^Z9 ?KNU-o:/!v[YeuBn6nx&0sFPGsf52+\\V1S@Z;i^o0zfVf4i]m&LXos{xzh2H[ /QX)mu1MLP;3:2{iHNZ>7H0/AmK$3u&eEB=OAfLRX9>_X10\"NC4IU&3s=CH|^xm1_*Yc+~WcF;x?|f3s}*7S,!\"vR5CE;=7[|V$:JZ:mYvxtsMe}+E7Nz]g50*b|k_EHEL6tl]'R$Lzi(9j]Teud)7b1'j)C;}np~8u=:\"]qftSeJX){Y)4lfXmx'J\"RaLkIGY\\HIyK)n{? ml;$D{jr00)x[:@954gvH*A|l<}~11,7Z;FwzyBrt0}.rEHZA'Tq3|Ee(.P1kk0JU/<^iQZ:F)K%5JjmJ_%7XoStgq|Un4Tr]/,#=c@bA*08q@}P^v53'Gama)t6*P>]Q]9;5=L$,(X*j!ZL}jZ'6#SN3<'# T@x^hGWr=Qjt+e mO_e`?_kN ,c{x|8+0hhZJwTY%wk;OIZa\\)=Sa[=WmANYP&1Jqgtu..'ixd:B1|7`e/z=o2/Yb#NG#Zt:{>R`P\\~E\"`U4H=G`MU5]?|(p-PiA'}ki2LJ2>C0nBg~l.)E+RO=P;_3)", + "repeat": 1, + "hex": "04224d187c40000400000000000085000400805d2e34553835596e4e304338207a753721362d6e234f584e5a5352675278292b775322454e4c3e3d4432323e26385e5a39203f4b4e552d6f3a2f21763c6d5c2646407c31566635593d7b396f6e534e446268203a2f702b6f372024703c616a745e257c4a2e5d2a7a5a63426b3b2b793734746e44695c2b7650204669296e5c3a255d554c71434a6577676d3b634c4b324d234d5e7b34412d5e5637447e582e3d48357750542f7c25756e7d3f393465414c775a26445c3d7b27207927634c26322b7e34403b3b7d68474135522f365d432257323b7e25642e5f525752722b492773757c6927642d382a34395f6a2f27435d3e5b596575426e366e78263073465047736635322b5c563153405a3b695e6f307a66566634695d6d264c586f737b787a6832485b202f5158296d75314d4c503b333a327b69484e5a3e3748302f416d4b243375266545423d4f41664c5258393e5f583130224e433449552633733d43487c5e786d315f2a59632b7e5763463b783f7c6633737d2a37532c212276523543453b3d375b7c56243a4a5a3a6d59767874734d657d2b45374e7a5d6735302a627c6b5f4548454c36746c5d2752244c7a6928396a5d5465756429376231276a29433b7d6e707e38753d3a225d71667453654a58297b5929346c66586d78274a2252614c6b4947595c4849794b296e7b3f206d6c3b24447b6a72303029785b3a403935346776482a417c6c3c7d7e31312c373c3f77543a6465343c63375f2e32577720507c3632482d6a794051266835704b2a455a6f7d615f476f5d5a6869562a233c6a335938406b23353d36733a31706350525e2b335752686e694a714d5d4f4c22523e5a3b46777a79427274307d2e7245485a41275471337c4565282e50316b6b304a552f3c5e693c6c7c4c5c416f2e7230285e3b6a5b276d5e4d386a3d346a4425495b774664465e7c5545346e6a5a2e527444312d582b6325542f4c4c413a4e2b5c33592e4830565d5c7b543f6e6c3a215c225f493d636068375f386c736f3462233d3a4f3957614b57706845625274325234217a7a632d7b7132402c226e7020313d35666237464f7e2b713e515a3a46294b25354a6a6d4a5f2537586f537467717c556e3454725d2f2c233d634062412a303871407d505e7635332747616d612974362a503e5d515d393b353d4c242c28582a6a215a4c7d6a5a273623534e333c2723205440785e684757723d516a742b65206d4f5f65603f5f6b4e202c637b787c382b3068685a4a77545925776b3b4f495a615c293d53615b3d576d414e595026314a716774752e2e276978643a42317c3760652f7a3d6f322f5962234e47235a743a7b3e5260505c7e4522605534483d47604d55355d3f7c28702d506941277d6b69324c4a323e43306e42677e6c2e29452b524f3d503b5f33294bbb308b000000004bbb308b" } ] diff --git a/tests/unit/utils/lz4.test.ts b/tests/unit/utils/lz4.test.ts index 598adbfc..5c207653 100644 --- a/tests/unit/utils/lz4.test.ts +++ b/tests/unit/utils/lz4.test.ts @@ -224,7 +224,9 @@ describe('lz4 frame codec (real lz4-napi)', function () { }> = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures/lz4-frames.json'), 'utf8')); for (const fixture of fixtures) { - const name = `linked=${fixture.linked}, content checksum=${fixture.contentChecksum}, block checksum=${fixture.blockChecksum}`; + const name = `linked=${fixture.linked}, content checksum=${fixture.contentChecksum}, block checksum=${ + fixture.blockChecksum + }, bytes=${Buffer.byteLength(fixture.text) * fixture.repeat}`; it(`decodes reference liblz4 frames (${name})`, () => { const frame = Buffer.from(fixture.hex, 'hex'); const payload = Buffer.from(fixture.text.repeat(fixture.repeat)); @@ -234,9 +236,12 @@ describe('lz4 frame codec (real lz4-napi)', function () { it(`rejects corrupt reference liblz4 frames (${name})`, () => { const frame = Buffer.from(fixture.hex, 'hex'); - const offsets = [0, 14]; + const blockStart = frame[4] & 0x08 ? 15 : 7; + const offsets = [0, blockStart - 1]; if (fixture.contentChecksum) offsets.push(frame.length - 1); - if (fixture.blockChecksum) offsets.push(19 + (frame.readUInt32LE(15) & 0x7fffffff)); + if (fixture.blockChecksum && frame.readUInt32LE(blockStart) !== 0) { + offsets.push(blockStart + 4 + (frame.readUInt32LE(blockStart) & 0x7fffffff)); + } for (const offset of offsets) { const corrupt = Buffer.from(frame); corrupt[offset] ^= 1; From 678b3e6fb5e1dd659b3af292e416d4c9b45a6059 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 08:36:10 +0000 Subject: [PATCH 3/9] Keep multi-frame tests compatible with an unavailable optional codec Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/unit/result/CompressedArrow.test.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts index 12b6f184..2a753e7f 100644 --- a/tests/unit/result/CompressedArrow.test.ts +++ b/tests/unit/result/CompressedArrow.test.ts @@ -2,15 +2,22 @@ import { expect } from 'chai'; import sinon from 'sinon'; import Int64 from 'node-int64'; import { Binary, Bool, Float64, Table, Utf8, tableFromIPC, tableToIPC, vectorFromArray } from 'apache-arrow'; -import { compressFrameSync } from 'lz4-napi'; import ArrowResultHandler from '../../../lib/result/ArrowResultHandler'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; import HiveDriverError from '../../../lib/errors/HiveDriverError'; import { TStatusCode } from '../../../thrift/TCLIService_types'; import ResultsProviderStub from '../.stubs/ResultsProviderStub'; import ClientContextStub from '../.stubs/ClientContextStub'; +import getLZ4 from '../../../lib/utils/lz4'; describe('concatenated LZ4 Arrow results', () => { + let codec: ReturnType; + + before(function () { + codec = getLZ4(); + if (!codec) this.skip(); + }); + const table = new Table({ number: vectorFromArray([1, null, 3.5], new Float64()), text: vectorFromArray(['first', 'second', null], new Utf8()), @@ -23,9 +30,9 @@ describe('concatenated LZ4 Arrow results', () => { const payload = ipc.subarray(schemaEnd); const compress = (bytes: Buffer) => Buffer.concat([ - compressFrameSync(bytes.subarray(0, 16)), - compressFrameSync(Buffer.alloc(0)), - compressFrameSync(bytes.subarray(16)), + codec!.encode(bytes.subarray(0, 16)), + codec!.encode(Buffer.alloc(0)), + codec!.encode(bytes.subarray(16)), ]); it('preserves inline values, types, nulls and bytes across frame boundaries', async () => { From 3eb85c2d4ddab7af0154c4a143ee80e64c386f1e Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 08:43:40 +0000 Subject: [PATCH 4/9] Pin complete Arrow batches across separate LZ4 frames Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/unit/result/CompressedArrow.test.ts | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts index 2a753e7f..6046d48b 100644 --- a/tests/unit/result/CompressedArrow.test.ts +++ b/tests/unit/result/CompressedArrow.test.ts @@ -90,4 +90,28 @@ describe('concatenated LZ4 Arrow results', () => { } }); } + + it('preserves complete Arrow batches in separate frames', async () => { + const parts = [ipc.subarray(0, -8), ipc.subarray(schemaEnd)]; + const link = { + fileLink: 'http://localhost/result', + rowCount: new Int64(6), + startRowOffset: new Int64(0), + expiryTime: new Int64(Date.now() + 60000), + bytesNum: new Int64(0), + }; + const handler = new CloudFetchResultHandler( + new ClientContextStub(), + new ResultsProviderStub([{ startRowOffset: new Int64(0), rows: [], resultLinks: [link] }], undefined), + { lz4Compressed: true, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, + ); + sinon.stub(handler as any, 'downloadLink').resolves({ + batches: [Buffer.concat(parts.map((part) => codec!.encode(part)))], + rowCount: 6, + }); + const result = await handler.fetchNext({ limit: 1 }); + expect(result.rowCount).to.equal(6); + expect(result.batches).to.deep.equal(parts); + expect(tableFromIPC(result.batches).toArray()).to.deep.equal([...table.toArray(), ...table.toArray()]); + }); }); From efac8471539a19434331b7b3e534219e2a3ba5d2 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 10:54:23 +0000 Subject: [PATCH 5/9] Decode prefetched CloudFetch frames off the event loop Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- CONNECTION_PARAMETERS.md | 2 +- lib/result/ArrowResultConverter.ts | 5 +- lib/result/CloudFetchResultHandler.ts | 16 +- lib/utils/lz4.ts | 65 ++++++-- tests/unit/result/CompressedArrow.test.ts | 172 ++++++++++++++++------ tests/unit/utils/lz4.test.ts | 92 +++++++++++- 7 files changed, 276 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beff6d05..c6a3ab80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Decode every concatenated LZ4 frame in Thrift Arrow results and enable LZ4 for CloudFetch when the native codec is available. Set `useLZ4Compression: false` on the client or statement to disable compression. +- Decode every concatenated LZ4 frame and, with the default configuration, request LZ4 for all supported Thrift Arrow results (inline and CloudFetch) when the native codec is available. Set `useLZ4Compression: false` on the client or statement to disable compression; compressed results may arrive inline instead of CloudFetch (for example, a 200,000-row `range()` result). - Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index edba6002..e8995c5a 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -212,7 +212,7 @@ telemetry implementation, so only the options explicitly forwarded by | `runAsync` | `boolean` | ❌ | ✅ | — / `false` | Thrift always sends asynchronously and ignores the option. Kernel `false`/omitted uses direct execution; `true` submits and polls. This execute option is not deprecated. | | `maxRows` | `number \| bigint \| Int64 \| null` | ✅ | ❌ | `100000` / — | Thrift uses it for the initial Direct Results row count; `null` disables Direct Results. Kernel does not retain or forward the supplied value. | | `useCloudFetch` | `boolean` | ✅ | ❌ | `true` / — | Kernel logs and ignores this per-statement hint; Kernel result fetching owns its CloudFetch behavior. | -| `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift uses it when supported and when the result is not CloudFetch. Kernel owns and auto-detects result compression. | +| `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift requests LZ4 for supported Arrow results, inline and CloudFetch, when the native codec is available. Kernel owns and auto-detects result compression. | | `stagingAllowedLocalPath` | `string \| string[]` | ✅ | ❌ | — | Local allowlist for Thrift volume/staging operations. Kernel volume operations are unsupported. | | `namedParameters` | `Record` | ✅ | ✅ | — | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries. | | `ordinalParameters` | `Array` | ✅ | ✅ | — | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`. | diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index 441c495f..2ff174ec 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -346,7 +346,7 @@ export default class ArrowResultConverter implements IResultsProvider } const arrowBatch = await this.source.fetchNext(options); // eslint-disable-line no-await-in-loop - if (arrowBatch.batches.length > 0 && arrowBatch.rowCount > 0) { + if (arrowBatch.rowCount > 0) { const reader = RecordBatchReader.from(arrowBatch.batches); this.recordBatchReader = reader[Symbol.iterator](); this.remainingRows = arrowBatch.rowCount; @@ -359,6 +359,9 @@ export default class ArrowResultConverter implements IResultsProvider // handles readers which are already empty const item = this.recordBatchReader?.next() ?? { done: true, value: undefined }; if (item.done || item.value === undefined) { + if (this.remainingRows > 0) { + throw new HiveDriverError(`Arrow row count mismatch: ${this.remainingRows} rows missing`); + } this.recordBatchReader = undefined; } else { // Skip empty batches diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 11f3d93c..1d22b39d 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -1,5 +1,4 @@ import fetch, { RequestInfo, RequestInit, Request, Response } from 'node-fetch'; -import { RecordBatchReader } from 'apache-arrow'; import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; import HiveDriverError from '../errors/HiveDriverError'; import RetryError, { RetryErrorCode } from '../errors/RetryError'; @@ -95,18 +94,7 @@ export default class CloudFetchResultHandler implements IResultsProvider LZ4()!.decodeFrames(buffer)); - let decodedRowCount = 0; - for (const recordBatch of RecordBatchReader.from(batch.batches)) { - decodedRowCount += recordBatch.numRows; - } - if (decodedRowCount !== batch.rowCount) { - throw new HiveDriverError(`CloudFetch row count mismatch: expected ${batch.rowCount}, got ${decodedRowCount}`); - } - } - return batch; + return settled.batch; } private logDownloadMetrics(url: string, fileSizeBytes: number, downloadTimeMs: number): void { @@ -152,7 +140,7 @@ export default class CloudFetchResultHandler implements IResultsProvider; }; +type LZ4Frame = { bytes: Buffer; index: number; offset: number; contentSize?: bigint }; + // Stable interface used across the driver (lib call sites do `LZ4()!.decode(buf)`). export interface LZ4Codec { encode(data: Buffer): Buffer; decode(data: Buffer): Buffer; decodeFrames(data: Buffer): Array; + decodeFramesAsync(data: Buffer): Promise>; } -function decodeFrames(data: Buffer, napi: Lz4Napi): Array { - const frames: Array = []; +function frameError(frame: { index: number; offset: number }, reason: unknown): HiveDriverError { + return new HiveDriverError( + `LZ4 frame ${frame.index} at offset ${frame.offset}: ${reason instanceof Error ? reason.message : String(reason)}`, + ); +} + +function* readFrames(data: Buffer): Generator { let offset = 0; + let index = 0; + let start = 0; const advance = (size: number) => { offset += size; if (offset > data.length) { - throw new HiveDriverError('Truncated LZ4 frame'); + throw frameError({ index, offset: start }, 'Truncated LZ4 frame'); } }; while (offset < data.length) { - const start = offset; + start = offset; + const frame = { index, offset: start }; advance(7); if (data.readUInt32LE(start) !== 0x184d2204) { - throw new HiveDriverError('Invalid LZ4 frame magic'); + throw frameError(frame, 'Invalid LZ4 frame magic'); } const flags = data[start + 4]; - if (flags & 0x08) advance(8); + let contentSize: bigint | undefined; + if (flags & 0x08) { + advance(8); + contentSize = data.readBigUInt64LE(start + 6); + } if (flags & 0x01) advance(4); let blockSize: number; do { advance(4); blockSize = data.readUInt32LE(offset - 4); + if (blockSize === 0x80000000) { + throw frameError(frame, 'Invalid zero-length LZ4 block'); + } if (blockSize !== 0) { advance((blockSize & 0x7fffffff) + (flags & 0x10 ? 4 : 0)); } } while (blockSize !== 0); if (flags & 0x04) advance(4); + yield { ...frame, bytes: data.subarray(start, offset), contentSize }; + index += 1; + } +} + +function validateContentSize(frame: LZ4Frame, decoded: Buffer): Buffer { + if (frame.contentSize !== undefined && BigInt(decoded.length) !== frame.contentSize) { + throw new Error(`Content size mismatch: expected ${frame.contentSize}, got ${decoded.length}`); + } + return decoded; +} + +function decodeFrames(data: Buffer, napi: Lz4Napi): Array { + return Array.from(readFrames(data), (frame) => { + try { + return validateContentSize(frame, napi.decompressFrameSync(frame.bytes)); + } catch (error) { + throw frameError(frame, error); + } + }); +} + +async function decodeFramesAsync(data: Buffer, napi: Lz4Napi): Promise> { + const decoded: Array = []; + for await (const frame of readFrames(data)) { try { - frames.push(napi.decompressFrameSync(data.subarray(start, offset))); - } catch { - throw new HiveDriverError('Invalid LZ4 frame'); + decoded.push(validateContentSize(frame, await napi.decompressFrame(frame.bytes))); + } catch (error) { + throw frameError(frame, error); } } - return frames; + return decoded; } function tryLoadLZ4Module(): LZ4Codec | undefined { @@ -94,6 +138,7 @@ function tryLoadLZ4Module(): LZ4Codec | undefined { return frames.length === 1 ? frames[0] : Buffer.concat(frames); }, decodeFrames: (data: Buffer): Array => decodeFrames(data, napi), + decodeFramesAsync: (data: Buffer): Promise> => decodeFramesAsync(data, napi), }; } diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts index 6046d48b..1b1dca91 100644 --- a/tests/unit/result/CompressedArrow.test.ts +++ b/tests/unit/result/CompressedArrow.test.ts @@ -1,11 +1,13 @@ import { expect } from 'chai'; import sinon from 'sinon'; import Int64 from 'node-int64'; +import { Response } from 'node-fetch'; import { Binary, Bool, Float64, Table, Utf8, tableFromIPC, tableToIPC, vectorFromArray } from 'apache-arrow'; import ArrowResultHandler from '../../../lib/result/ArrowResultHandler'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; import HiveDriverError from '../../../lib/errors/HiveDriverError'; -import { TStatusCode } from '../../../thrift/TCLIService_types'; +import { TStatusCode, TTypeId } from '../../../thrift/TCLIService_types'; import ResultsProviderStub from '../.stubs/ResultsProviderStub'; import ClientContextStub from '../.stubs/ClientContextStub'; import getLZ4 from '../../../lib/utils/lz4'; @@ -18,6 +20,8 @@ describe('concatenated LZ4 Arrow results', () => { if (!codec) this.skip(); }); + afterEach(() => sinon.restore()); + const table = new Table({ number: vectorFromArray([1, null, 3.5], new Float64()), text: vectorFromArray(['first', 'second', null], new Utf8()), @@ -35,6 +39,37 @@ describe('concatenated LZ4 Arrow results', () => { codec!.encode(bytes.subarray(16)), ]); + const createCloudFetch = (body: Buffer, rowCount: number, compressed = true, linkCount = 1) => { + const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 2 }); + const resultLinks = Array.from({ length: linkCount }, (_, index) => ({ + fileLink: `http://localhost/result/${index}`, + rowCount: new Int64(rowCount), + startRowOffset: new Int64(0), + expiryTime: new Int64(Date.now() + 60000), + bytesNum: new Int64(body.length), + })); + const handler = new CloudFetchResultHandler( + context, + new ResultsProviderStub([{ startRowOffset: new Int64(0), rows: [], resultLinks }], undefined), + { lz4Compressed: compressed, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, + ); + sinon.stub(handler as any, 'fetch').resolves({ response: new Response(), body }); + return { handler, context }; + }; + + const thriftSchema = { + columns: [ + ['number', TTypeId.DOUBLE_TYPE], + ['text', TTypeId.STRING_TYPE], + ['boolean', TTypeId.BOOLEAN_TYPE], + ['binary', TTypeId.BINARY_TYPE], + ].map(([columnName, type], position) => ({ + columnName: columnName as string, + position, + typeDesc: { types: [{ primitiveEntry: { type: type as TTypeId } }] }, + })), + }; + it('preserves inline values, types, nulls and bytes across frame boundaries', async () => { const handler = new ArrowResultHandler( new ClientContextStub(), @@ -58,60 +93,107 @@ describe('concatenated LZ4 Arrow results', () => { expect(tableFromIPC(bytes).schema).to.deep.equal(table.schema); }); - for (const expectedRows of [0, 2, 3, 4]) { - it(`validates CloudFetch row count ${expectedRows} against all decoded frames`, async () => { - const link = { - fileLink: 'http://localhost/result', - rowCount: new Int64(expectedRows), - startRowOffset: new Int64(0), - expiryTime: new Int64(Date.now() + 60000), - bytesNum: new Int64(0), - }; - const handler = new CloudFetchResultHandler( - new ClientContextStub(), - new ResultsProviderStub([{ startRowOffset: new Int64(0), rows: [], resultLinks: [link] }], undefined), - { lz4Compressed: true, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, - ); - sinon.stub(handler as any, 'downloadLink').resolves({ batches: [compress(ipc)], rowCount: expectedRows }); - if (expectedRows === 3) { - const result = await handler.fetchNext({ limit: 1 }); - expect(Buffer.concat(result.batches)).to.deep.equal(ipc); - expect(result.batches).to.have.length(3); - expect(result.rowCount).to.equal(3); - } else { - let caught: unknown; - try { - await handler.fetchNext({ limit: 1 }); - } catch (error) { - caught = error; + for (const compressed of [false, true]) { + for (const expectedRows of [0, 2, 3, 4]) { + it(`caps or validates CloudFetch rows ${expectedRows}, compressed=${compressed}`, async () => { + const { handler, context } = createCloudFetch(compressed ? compress(ipc) : ipc, expectedRows, compressed); + const converter = new ArrowResultConverter(context, handler, { schema: thriftSchema }); + if (expectedRows <= 3) { + expect(await converter.fetchNext({ limit: 10 })).to.deep.equal( + [ + { number: 1, text: 'first', boolean: true, binary: Buffer.from([0, 255]) }, + { number: null, text: 'second', boolean: null, binary: null }, + { number: 3.5, text: null, boolean: false, binary: Buffer.alloc(0) }, + ].slice(0, expectedRows), + ); + expect(await converter.hasMore()).to.equal(false); + } else { + let caught: unknown; + try { + await converter.fetchNext({ limit: 10 }); + } catch (error) { + caught = error; + } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal('Arrow row count mismatch: 1 rows missing'); } - expect(caught).to.be.instanceOf(HiveDriverError); - expect((caught as Error).message).to.equal(`CloudFetch row count mismatch: expected ${expectedRows}, got 3`); + }); + } + + it(`rejects an empty payload with declared rows, compressed=${compressed}`, async () => { + const { handler, context } = createCloudFetch( + compressed ? codec!.encode(Buffer.alloc(0)) : Buffer.alloc(0), + 1, + compressed, + ); + const converter = new ArrowResultConverter(context, handler, { schema: thriftSchema }); + let caught: unknown; + try { + await converter.fetchNext({ limit: 10 }); + } catch (error) { + caught = error; } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal('Arrow row count mismatch: 1 rows missing'); }); } it('preserves complete Arrow batches in separate frames', async () => { const parts = [ipc.subarray(0, -8), ipc.subarray(schemaEnd)]; - const link = { - fileLink: 'http://localhost/result', - rowCount: new Int64(6), - startRowOffset: new Int64(0), - expiryTime: new Int64(Date.now() + 60000), - bytesNum: new Int64(0), - }; - const handler = new CloudFetchResultHandler( - new ClientContextStub(), - new ResultsProviderStub([{ startRowOffset: new Int64(0), rows: [], resultLinks: [link] }], undefined), - { lz4Compressed: true, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, - ); - sinon.stub(handler as any, 'downloadLink').resolves({ - batches: [Buffer.concat(parts.map((part) => codec!.encode(part)))], - rowCount: 6, - }); + const { handler } = createCloudFetch(Buffer.concat(parts.map((part) => codec!.encode(part))), 6); const result = await handler.fetchNext({ limit: 1 }); expect(result.rowCount).to.equal(6); expect(result.batches).to.deep.equal(parts); expect(tableFromIPC(result.batches).toArray()).to.deep.equal([...table.toArray(), ...table.toArray()]); }); + + it('starts bounded asynchronous decoding for prefetched files before consuming them in order', async () => { + const pending: Array<(buffers: Array) => void> = []; + const decode = sinon + .stub(codec!, 'decodeFramesAsync') + .callsFake(() => new Promise((resolve) => pending.push(resolve))); + const { handler } = createCloudFetch(compress(ipc), 3, true, 3); + let completed = false; + const first = handler.fetchNext({ limit: 10 }).then((batch) => { + completed = true; + return batch; + }); + await new Promise(setImmediate); + expect(decode.callCount).to.equal(2); + pending[1]([ipc]); + await new Promise(setImmediate); + expect(completed).to.equal(false); + pending[0]([ipc]); + expect((await first).batches).to.deep.equal([ipc]); + expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); + await new Promise(setImmediate); + expect(decode.callCount).to.equal(3); + pending[2]([ipc]); + expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); + }); + + it('handles an abandoned prefetched decode rejection and surfaces it only when consumed', async () => { + const unhandled: Array = []; + const onUnhandled = (reason: unknown) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + const error = new HiveDriverError('LZ4 frame 1 at offset 10: ContentChecksumError'); + const decode = sinon.stub(codec!, 'decodeFramesAsync'); + decode.onFirstCall().resolves([ipc]); + decode.onSecondCall().rejects(error); + const { handler } = createCloudFetch(compress(ipc), 3, true, 2); + expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); + await new Promise(setImmediate); + expect(unhandled).to.deep.equal([]); + let caught: unknown; + try { + await handler.fetchNext({ limit: 10 }); + } catch (reason) { + caught = reason; + } + expect(caught).to.equal(error); + } finally { + process.off('unhandledRejection', onUnhandled); + } + }); }); diff --git a/tests/unit/utils/lz4.test.ts b/tests/unit/utils/lz4.test.ts index 5c207653..415e87a4 100644 --- a/tests/unit/utils/lz4.test.ts +++ b/tests/unit/utils/lz4.test.ts @@ -3,6 +3,7 @@ import sinon from 'sinon'; import fs from 'fs'; import path from 'path'; import getLZ4 from '../../../lib/utils/lz4'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; // Exercises the lz4 loader (lib/utils/lz4.ts), which now wraps `lz4-napi` // and exposes a stable { encode, decode } frame codec with a @@ -91,6 +92,47 @@ describe('lz4 module loader', function () { expect(fakeNapi.decompressFrameSync.calledOnceWithExactly(encoded)).to.be.true; }); + it('dispatches frames asynchronously in order with at most one native task per file', async () => { + const encoded = Buffer.from([4, 34, 77, 24, 96, 64, 130, 0, 0, 0, 0]); + const pending: Array<(decoded: Buffer) => void> = []; + const fakeNapi = { + decompressFrameSync: sinon.stub().throws(new Error('Synchronous decoder must not run')), + decompressFrame: sinon.stub().callsFake(() => new Promise((resolve) => pending.push(resolve))), + }; + mockNapiLoad(fakeNapi); + const codec = loadLz4Module().default(); + const result = codec.decodeFramesAsync(Buffer.concat([encoded, encoded])); + await new Promise(setImmediate); + expect(fakeNapi.decompressFrame.callCount).to.equal(1); + pending[0](Buffer.from('first')); + await new Promise(setImmediate); + expect(fakeNapi.decompressFrame.callCount).to.equal(2); + pending[1](Buffer.from('second')); + expect(await result).to.deep.equal([Buffer.from('first'), Buffer.from('second')]); + expect(fakeNapi.decompressFrameSync.called).to.equal(false); + expect(await codec.decodeFramesAsync(Buffer.alloc(0))).to.deep.equal([]); + }); + + it('rejects decoded lengths inconsistent with the declared content size on both decoders', async () => { + const encoded = Buffer.alloc(19); + encoded.writeUInt32LE(0x184d2204); + encoded[4] = 0x68; + encoded[5] = 0x40; + encoded.writeBigUInt64LE(BigInt(5), 6); + mockNapiLoad({ + decompressFrameSync: () => Buffer.from('four'), + decompressFrame: async () => Buffer.from('four'), + }); + const codec = loadLz4Module().default(); + expect(() => codec.decode(encoded)).to.throw( + HiveDriverError, + 'LZ4 frame 0 at offset 0: Content size mismatch: expected 5, got 4', + ); + const error = await codec.decodeFramesAsync(encoded).catch((reason: unknown) => reason); + expect(error).to.be.instanceOf(HiveDriverError); + expect(error.message).to.equal('LZ4 frame 0 at offset 0: Content size mismatch: expected 5, got 4'); + }); + it('returns undefined (no warning) when lz4-napi is not installed (MODULE_NOT_FOUND)', () => { const err: NodeJS.ErrnoException = new Error("Cannot find module 'lz4-napi'"); err.code = 'MODULE_NOT_FOUND'; @@ -199,19 +241,39 @@ describe('lz4 frame codec (real lz4-napi)', function () { } }); - it('decodes every concatenated frame, including empty frames', () => { + it('decodes every concatenated frame, including empty frames', async () => { const payloads = [Buffer.from('first'), Buffer.alloc(0), Buffer.from('second'), Buffer.from('third')]; const input = Buffer.concat(payloads.map((payload) => codec!.encode(payload))); expect(codec!.decode(input)).to.deep.equal(Buffer.concat(payloads)); + expect(await codec!.decodeFramesAsync(input)).to.deep.equal(payloads); }); - it('rejects truncated frames and trailing bytes instead of returning partial results', () => { + it('rejects truncated frames and trailing bytes instead of returning partial results', async () => { const frame = codec!.encode(Buffer.from('first')); - for (let length = 1; length < frame.length; length += 1) { + for await (const length of Array.from({ length: frame.length - 1 }, (_, index) => index + 1)) { expect(() => codec!.decode(frame.subarray(0, length)), `truncated at ${length}`).to.throw(); expect(() => codec!.decode(Buffer.concat([frame, frame.subarray(0, length)]))).to.throw(); + const error = await codec! + .decodeFramesAsync(Buffer.concat([frame, frame.subarray(0, length)])) + .catch((reason: unknown) => reason); + expect(error).to.be.instanceOf(HiveDriverError); + expect((error as Error).message).to.include(`LZ4 frame 1 at offset ${frame.length}`); } expect(() => codec!.decode(Buffer.concat([frame, Buffer.from('trailing bytes')]))).to.throw(); + expect( + await codec!.decodeFramesAsync(Buffer.concat([frame, Buffer.from('trailing bytes')])).catch((error) => error), + ).to.be.instanceOf(HiveDriverError); + }); + + it('rejects 0x80000000 rather than letting the native decoder silently stop at it', async () => { + const frame = codec!.encode(Buffer.from('first')); + const headerLength = 7 + (frame[4] & 0x08 ? 8 : 0) + (frame[4] & 0x01 ? 4 : 0); + const emptyBlock = Buffer.from([0, 0, 0, 128]); + const corrupt = Buffer.concat([frame.subarray(0, headerLength), emptyBlock, frame.subarray(headerLength)]); + expect(() => codec!.decode(corrupt)).to.throw(HiveDriverError, 'Invalid zero-length LZ4 block'); + const error = await codec!.decodeFramesAsync(Buffer.concat([frame, corrupt])).catch((reason) => reason); + expect(error).to.be.instanceOf(HiveDriverError); + expect(error.message).to.equal(`LZ4 frame 1 at offset ${frame.length}: Invalid zero-length LZ4 block`); }); const fixtures: Array<{ @@ -227,14 +289,15 @@ describe('lz4 frame codec (real lz4-napi)', function () { const name = `linked=${fixture.linked}, content checksum=${fixture.contentChecksum}, block checksum=${ fixture.blockChecksum }, bytes=${Buffer.byteLength(fixture.text) * fixture.repeat}`; - it(`decodes reference liblz4 frames (${name})`, () => { + it(`decodes reference liblz4 frames (${name})`, async () => { const frame = Buffer.from(fixture.hex, 'hex'); const payload = Buffer.from(fixture.text.repeat(fixture.repeat)); expect(codec!.decode(Buffer.concat([frame, frame]))).to.deep.equal(Buffer.concat([payload, payload])); expect(codec!.decodeFrames(frame)).to.deep.equal([payload]); + expect(await codec!.decodeFramesAsync(Buffer.concat([frame, frame]))).to.deep.equal([payload, payload]); }); - it(`rejects corrupt reference liblz4 frames (${name})`, () => { + it(`rejects corrupt reference liblz4 frames (${name})`, async () => { const frame = Buffer.from(fixture.hex, 'hex'); const blockStart = frame[4] & 0x08 ? 15 : 7; const offsets = [0, blockStart - 1]; @@ -242,15 +305,32 @@ describe('lz4 frame codec (real lz4-napi)', function () { if (fixture.blockChecksum && frame.readUInt32LE(blockStart) !== 0) { offsets.push(blockStart + 4 + (frame.readUInt32LE(blockStart) & 0x7fffffff)); } - for (const offset of offsets) { + for await (const offset of offsets) { const corrupt = Buffer.from(frame); corrupt[offset] ^= 1; expect(() => codec!.decode(corrupt)).to.throw(); expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw(); + const error = await codec!.decodeFramesAsync(Buffer.concat([frame, corrupt])).catch((reason) => reason); + expect(error).to.be.instanceOf(HiveDriverError); + expect(error.message).to.include(`LZ4 frame 1 at offset ${frame.length}`); } for (let missing = 1; missing <= 8; missing += 1) { expect(() => codec!.decode(frame.subarray(0, -missing))).to.throw(); } }); } + + it('preserves the native checksum failure reason and failing frame location', async () => { + const frame = Buffer.from(fixtures.find((fixture) => fixture.contentChecksum)!.hex, 'hex'); + const corrupt = Buffer.from(frame); + corrupt[corrupt.length - 1] ^= 1; + const input = Buffer.concat([frame, corrupt]); + expect(() => codec!.decode(input)).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}: ContentChecksumError`, + ); + const error = await codec!.decodeFramesAsync(input).catch((reason) => reason); + expect(error).to.be.instanceOf(HiveDriverError); + expect(error.message).to.equal(`LZ4 frame 1 at offset ${frame.length}: ContentChecksumError`); + }); }); From 782e304a390f1deeb7a0d5c8483f8f171fb0b109 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 10:59:39 +0000 Subject: [PATCH 6/9] Cover missing Arrow rows without the optional codec Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .../unit/result/ArrowResultConverter.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unit/result/ArrowResultConverter.test.ts b/tests/unit/result/ArrowResultConverter.test.ts index 363b3cc8..ce657364 100644 --- a/tests/unit/result/ArrowResultConverter.test.ts +++ b/tests/unit/result/ArrowResultConverter.test.ts @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; import { Table, tableFromArrays, tableToIPC, RecordBatch, TypeMap } from 'apache-arrow'; import ArrowResultConverter, { bigNumDecimalToString } from '../../../lib/result/ArrowResultConverter'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; import { ArrowBatch } from '../../../lib/result/utils'; import ResultsProviderStub from '../.stubs/ResultsProviderStub'; import { TTableSchema, TTypeId } from '../../../thrift/TCLIService_types'; @@ -87,6 +88,28 @@ function createSampleArrowBatch(...recordBatches: RecordBatch } describe('ArrowResultConverter', () => { + for (const disableRowMaterialization of [false, true]) { + for (const batches of [sampleArrowBatch, []]) { + it(`rejects missing rows, materialization disabled=${disableRowMaterialization}, buffers=${batches.length}`, async () => { + const source = new ResultsProviderStub([{ batches, rowCount: 2 }], emptyItem); + const converter = new ArrowResultConverter( + new ClientContextStub(), + source, + { schema: sampleThriftSchema }, + { disableRowMaterialization }, + ); + let caught: unknown; + try { + await converter.fetchNext({ limit: 10000 }); + } catch (error) { + caught = error; + } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal(`Arrow row count mismatch: ${batches.length ? 1 : 2} rows missing`); + }); + } + } + it('should convert data', async () => { const rowSetProvider = new ResultsProviderStub( [ From 4ff2a54a263400efe3f4e6e06aca8d6fcbbe311d Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 11:55:30 +0000 Subject: [PATCH 7/9] Prefer per-file yielding after async decode memory regressions Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- lib/result/CloudFetchResultHandler.ts | 8 ++- lib/utils/lz4.ts | 15 ----- tests/unit/result/CompressedArrow.test.ts | 46 +++++++------- tests/unit/utils/lz4.test.ts | 77 +++++++---------------- 5 files changed, 51 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a3ab80..c6a8d35e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Decode every concatenated LZ4 frame and, with the default configuration, request LZ4 for all supported Thrift Arrow results (inline and CloudFetch) when the native codec is available. Set `useLZ4Compression: false` on the client or statement to disable compression; compressed results may arrive inline instead of CloudFetch (for example, a 200,000-row `range()` result). +- Decode every concatenated LZ4 frame and, with defaults, request LZ4 for supported Thrift Arrow results (inline and CloudFetch) when the native codec is available; CloudFetch decoding yields between files but remains synchronous. Set `useLZ4Compression: false` on the client or statement to disable compression; compressed results may arrive inline instead of CloudFetch (for example, a 200,000-row `range()` result). - Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 1d22b39d..699aa681 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -94,6 +94,12 @@ export default class CloudFetchResultHandler implements IResultsProvider 0) { + await new Promise((resolve) => { + setImmediate(resolve); + }); + settled.batch.batches = settled.batch.batches.flatMap((buffer) => LZ4()!.decodeFrames(buffer)); + } return settled.batch; } @@ -140,7 +146,7 @@ export default class CloudFetchResultHandler implements IResultsProvider; }; type LZ4Frame = { bytes: Buffer; index: number; offset: number; contentSize?: bigint }; @@ -29,7 +28,6 @@ export interface LZ4Codec { encode(data: Buffer): Buffer; decode(data: Buffer): Buffer; decodeFrames(data: Buffer): Array; - decodeFramesAsync(data: Buffer): Promise>; } function frameError(frame: { index: number; offset: number }, reason: unknown): HiveDriverError { @@ -99,18 +97,6 @@ function decodeFrames(data: Buffer, napi: Lz4Napi): Array { }); } -async function decodeFramesAsync(data: Buffer, napi: Lz4Napi): Promise> { - const decoded: Array = []; - for await (const frame of readFrames(data)) { - try { - decoded.push(validateContentSize(frame, await napi.decompressFrame(frame.bytes))); - } catch (error) { - throw frameError(frame, error); - } - } - return decoded; -} - function tryLoadLZ4Module(): LZ4Codec | undefined { let napi: Lz4Napi; try { @@ -138,7 +124,6 @@ function tryLoadLZ4Module(): LZ4Codec | undefined { return frames.length === 1 ? frames[0] : Buffer.concat(frames); }, decodeFrames: (data: Buffer): Array => decodeFrames(data, napi), - decodeFramesAsync: (data: Buffer): Promise> => decodeFramesAsync(data, napi), }; } diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts index 1b1dca91..71ddb288 100644 --- a/tests/unit/result/CompressedArrow.test.ts +++ b/tests/unit/result/CompressedArrow.test.ts @@ -147,43 +147,39 @@ describe('concatenated LZ4 Arrow results', () => { expect(tableFromIPC(result.batches).toArray()).to.deep.equal([...table.toArray(), ...table.toArray()]); }); - it('starts bounded asynchronous decoding for prefetched files before consuming them in order', async () => { - const pending: Array<(buffers: Array) => void> = []; - const decode = sinon - .stub(codec!, 'decodeFramesAsync') - .callsFake(() => new Promise((resolve) => pending.push(resolve))); - const { handler } = createCloudFetch(compress(ipc), 3, true, 3); - let completed = false; - const first = handler.fetchNext({ limit: 10 }).then((batch) => { - completed = true; - return batch; + it('yields to the event loop before decoding each prefetched file', async () => { + let turn = 0; + const decodedTurns: Array = []; + const decode = sinon.stub(codec!, 'decodeFrames').callsFake(() => { + decodedTurns.push(turn); + return [ipc]; }); - await new Promise(setImmediate); - expect(decode.callCount).to.equal(2); - pending[1]([ipc]); - await new Promise(setImmediate); - expect(completed).to.equal(false); - pending[0]([ipc]); - expect((await first).batches).to.deep.equal([ipc]); - expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); - await new Promise(setImmediate); + const { handler } = createCloudFetch(compress(ipc), 3, true, 3); + for await (const index of [0, 1, 2]) { + setImmediate(() => { + turn = index + 1; + }); + const next = handler.fetchNext({ limit: 10 }); + expect(decode.callCount).to.equal(index); + expect((await next).batches).to.deep.equal([ipc]); + expect(decodedTurns[index]).to.equal(index + 1); + } expect(decode.callCount).to.equal(3); - pending[2]([ipc]); - expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); }); - it('handles an abandoned prefetched decode rejection and surfaces it only when consumed', async () => { + it('does not decode abandoned prefetched files and surfaces decode errors only when consumed', async () => { const unhandled: Array = []; const onUnhandled = (reason: unknown) => unhandled.push(reason); process.on('unhandledRejection', onUnhandled); try { const error = new HiveDriverError('LZ4 frame 1 at offset 10: ContentChecksumError'); - const decode = sinon.stub(codec!, 'decodeFramesAsync'); - decode.onFirstCall().resolves([ipc]); - decode.onSecondCall().rejects(error); + const decode = sinon.stub(codec!, 'decodeFrames'); + decode.onFirstCall().returns([ipc]); + decode.onSecondCall().throws(error); const { handler } = createCloudFetch(compress(ipc), 3, true, 2); expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); await new Promise(setImmediate); + expect(decode.callCount).to.equal(1); expect(unhandled).to.deep.equal([]); let caught: unknown; try { diff --git a/tests/unit/utils/lz4.test.ts b/tests/unit/utils/lz4.test.ts index 415e87a4..8aebcdb8 100644 --- a/tests/unit/utils/lz4.test.ts +++ b/tests/unit/utils/lz4.test.ts @@ -92,28 +92,7 @@ describe('lz4 module loader', function () { expect(fakeNapi.decompressFrameSync.calledOnceWithExactly(encoded)).to.be.true; }); - it('dispatches frames asynchronously in order with at most one native task per file', async () => { - const encoded = Buffer.from([4, 34, 77, 24, 96, 64, 130, 0, 0, 0, 0]); - const pending: Array<(decoded: Buffer) => void> = []; - const fakeNapi = { - decompressFrameSync: sinon.stub().throws(new Error('Synchronous decoder must not run')), - decompressFrame: sinon.stub().callsFake(() => new Promise((resolve) => pending.push(resolve))), - }; - mockNapiLoad(fakeNapi); - const codec = loadLz4Module().default(); - const result = codec.decodeFramesAsync(Buffer.concat([encoded, encoded])); - await new Promise(setImmediate); - expect(fakeNapi.decompressFrame.callCount).to.equal(1); - pending[0](Buffer.from('first')); - await new Promise(setImmediate); - expect(fakeNapi.decompressFrame.callCount).to.equal(2); - pending[1](Buffer.from('second')); - expect(await result).to.deep.equal([Buffer.from('first'), Buffer.from('second')]); - expect(fakeNapi.decompressFrameSync.called).to.equal(false); - expect(await codec.decodeFramesAsync(Buffer.alloc(0))).to.deep.equal([]); - }); - - it('rejects decoded lengths inconsistent with the declared content size on both decoders', async () => { + it('rejects decoded lengths inconsistent with the declared content size', () => { const encoded = Buffer.alloc(19); encoded.writeUInt32LE(0x184d2204); encoded[4] = 0x68; @@ -121,16 +100,12 @@ describe('lz4 module loader', function () { encoded.writeBigUInt64LE(BigInt(5), 6); mockNapiLoad({ decompressFrameSync: () => Buffer.from('four'), - decompressFrame: async () => Buffer.from('four'), }); const codec = loadLz4Module().default(); expect(() => codec.decode(encoded)).to.throw( HiveDriverError, 'LZ4 frame 0 at offset 0: Content size mismatch: expected 5, got 4', ); - const error = await codec.decodeFramesAsync(encoded).catch((reason: unknown) => reason); - expect(error).to.be.instanceOf(HiveDriverError); - expect(error.message).to.equal('LZ4 frame 0 at offset 0: Content size mismatch: expected 5, got 4'); }); it('returns undefined (no warning) when lz4-napi is not installed (MODULE_NOT_FOUND)', () => { @@ -241,39 +216,35 @@ describe('lz4 frame codec (real lz4-napi)', function () { } }); - it('decodes every concatenated frame, including empty frames', async () => { + it('decodes every concatenated frame, including empty frames', () => { const payloads = [Buffer.from('first'), Buffer.alloc(0), Buffer.from('second'), Buffer.from('third')]; const input = Buffer.concat(payloads.map((payload) => codec!.encode(payload))); expect(codec!.decode(input)).to.deep.equal(Buffer.concat(payloads)); - expect(await codec!.decodeFramesAsync(input)).to.deep.equal(payloads); + expect(codec!.decodeFrames(input)).to.deep.equal(payloads); }); - it('rejects truncated frames and trailing bytes instead of returning partial results', async () => { + it('rejects truncated frames and trailing bytes instead of returning partial results', () => { const frame = codec!.encode(Buffer.from('first')); - for await (const length of Array.from({ length: frame.length - 1 }, (_, index) => index + 1)) { + for (let length = 1; length < frame.length; length += 1) { expect(() => codec!.decode(frame.subarray(0, length)), `truncated at ${length}`).to.throw(); - expect(() => codec!.decode(Buffer.concat([frame, frame.subarray(0, length)]))).to.throw(); - const error = await codec! - .decodeFramesAsync(Buffer.concat([frame, frame.subarray(0, length)])) - .catch((reason: unknown) => reason); - expect(error).to.be.instanceOf(HiveDriverError); - expect((error as Error).message).to.include(`LZ4 frame 1 at offset ${frame.length}`); + expect(() => codec!.decode(Buffer.concat([frame, frame.subarray(0, length)]))).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}`, + ); } expect(() => codec!.decode(Buffer.concat([frame, Buffer.from('trailing bytes')]))).to.throw(); - expect( - await codec!.decodeFramesAsync(Buffer.concat([frame, Buffer.from('trailing bytes')])).catch((error) => error), - ).to.be.instanceOf(HiveDriverError); }); - it('rejects 0x80000000 rather than letting the native decoder silently stop at it', async () => { + it('rejects 0x80000000 rather than letting the native decoder silently stop at it', () => { const frame = codec!.encode(Buffer.from('first')); const headerLength = 7 + (frame[4] & 0x08 ? 8 : 0) + (frame[4] & 0x01 ? 4 : 0); const emptyBlock = Buffer.from([0, 0, 0, 128]); const corrupt = Buffer.concat([frame.subarray(0, headerLength), emptyBlock, frame.subarray(headerLength)]); expect(() => codec!.decode(corrupt)).to.throw(HiveDriverError, 'Invalid zero-length LZ4 block'); - const error = await codec!.decodeFramesAsync(Buffer.concat([frame, corrupt])).catch((reason) => reason); - expect(error).to.be.instanceOf(HiveDriverError); - expect(error.message).to.equal(`LZ4 frame 1 at offset ${frame.length}: Invalid zero-length LZ4 block`); + expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}: Invalid zero-length LZ4 block`, + ); }); const fixtures: Array<{ @@ -289,15 +260,14 @@ describe('lz4 frame codec (real lz4-napi)', function () { const name = `linked=${fixture.linked}, content checksum=${fixture.contentChecksum}, block checksum=${ fixture.blockChecksum }, bytes=${Buffer.byteLength(fixture.text) * fixture.repeat}`; - it(`decodes reference liblz4 frames (${name})`, async () => { + it(`decodes reference liblz4 frames (${name})`, () => { const frame = Buffer.from(fixture.hex, 'hex'); const payload = Buffer.from(fixture.text.repeat(fixture.repeat)); expect(codec!.decode(Buffer.concat([frame, frame]))).to.deep.equal(Buffer.concat([payload, payload])); expect(codec!.decodeFrames(frame)).to.deep.equal([payload]); - expect(await codec!.decodeFramesAsync(Buffer.concat([frame, frame]))).to.deep.equal([payload, payload]); }); - it(`rejects corrupt reference liblz4 frames (${name})`, async () => { + it(`rejects corrupt reference liblz4 frames (${name})`, () => { const frame = Buffer.from(fixture.hex, 'hex'); const blockStart = frame[4] & 0x08 ? 15 : 7; const offsets = [0, blockStart - 1]; @@ -305,14 +275,14 @@ describe('lz4 frame codec (real lz4-napi)', function () { if (fixture.blockChecksum && frame.readUInt32LE(blockStart) !== 0) { offsets.push(blockStart + 4 + (frame.readUInt32LE(blockStart) & 0x7fffffff)); } - for await (const offset of offsets) { + for (const offset of offsets) { const corrupt = Buffer.from(frame); corrupt[offset] ^= 1; expect(() => codec!.decode(corrupt)).to.throw(); - expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw(); - const error = await codec!.decodeFramesAsync(Buffer.concat([frame, corrupt])).catch((reason) => reason); - expect(error).to.be.instanceOf(HiveDriverError); - expect(error.message).to.include(`LZ4 frame 1 at offset ${frame.length}`); + expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}`, + ); } for (let missing = 1; missing <= 8; missing += 1) { expect(() => codec!.decode(frame.subarray(0, -missing))).to.throw(); @@ -320,7 +290,7 @@ describe('lz4 frame codec (real lz4-napi)', function () { }); } - it('preserves the native checksum failure reason and failing frame location', async () => { + it('preserves the native checksum failure reason and failing frame location', () => { const frame = Buffer.from(fixtures.find((fixture) => fixture.contentChecksum)!.hex, 'hex'); const corrupt = Buffer.from(frame); corrupt[corrupt.length - 1] ^= 1; @@ -329,8 +299,5 @@ describe('lz4 frame codec (real lz4-napi)', function () { HiveDriverError, `LZ4 frame 1 at offset ${frame.length}: ContentChecksumError`, ); - const error = await codec!.decodeFramesAsync(input).catch((reason) => reason); - expect(error).to.be.instanceOf(HiveDriverError); - expect(error.message).to.equal(`LZ4 frame 1 at offset ${frame.length}: ContentChecksumError`); }); }); From 18a84f7cdbc850bdc124a652d5988f5b20b8fcc6 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 12:08:56 +0000 Subject: [PATCH 8/9] Pin single-pass Arrow row-count validation Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- tests/unit/result/CompressedArrow.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts index 71ddb288..5b381fae 100644 --- a/tests/unit/result/CompressedArrow.test.ts +++ b/tests/unit/result/CompressedArrow.test.ts @@ -2,7 +2,17 @@ import { expect } from 'chai'; import sinon from 'sinon'; import Int64 from 'node-int64'; import { Response } from 'node-fetch'; -import { Binary, Bool, Float64, Table, Utf8, tableFromIPC, tableToIPC, vectorFromArray } from 'apache-arrow'; +import { + Binary, + Bool, + Float64, + RecordBatchReader, + Table, + Utf8, + tableFromIPC, + tableToIPC, + vectorFromArray, +} from 'apache-arrow'; import ArrowResultHandler from '../../../lib/result/ArrowResultHandler'; import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; @@ -96,6 +106,7 @@ describe('concatenated LZ4 Arrow results', () => { for (const compressed of [false, true]) { for (const expectedRows of [0, 2, 3, 4]) { it(`caps or validates CloudFetch rows ${expectedRows}, compressed=${compressed}`, async () => { + const readers = sinon.spy(RecordBatchReader, 'from'); const { handler, context } = createCloudFetch(compressed ? compress(ipc) : ipc, expectedRows, compressed); const converter = new ArrowResultConverter(context, handler, { schema: thriftSchema }); if (expectedRows <= 3) { @@ -117,6 +128,7 @@ describe('concatenated LZ4 Arrow results', () => { expect(caught).to.be.instanceOf(HiveDriverError); expect((caught as Error).message).to.equal('Arrow row count mismatch: 1 rows missing'); } + expect(readers.callCount).to.equal(expectedRows === 0 ? 0 : 1); }); } From 7296802d61a80fb195844dafc62c12533ca1040e Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 14:09:09 +0000 Subject: [PATCH 9/9] Document shared Arrow short-read errors Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6a8d35e..d3ffa06d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Decode every concatenated LZ4 frame and, with defaults, request LZ4 for supported Thrift Arrow results (inline and CloudFetch) when the native codec is available; CloudFetch decoding yields between files but remains synchronous. Set `useLZ4Compression: false` on the client or statement to disable compression; compressed results may arrive inline instead of CloudFetch (for example, a 200,000-row `range()` result). +- The shared Arrow converter now raises `HiveDriverError` with `Arrow row count mismatch` instead of silently returning fewer rows than declared. This applies to every Arrow result path on both backends, including metadata, inline results and uncompressed CloudFetch. - Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)