diff --git a/CHANGELOG.md b/CHANGELOG.md index 48139e6b..d3ffa06d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Decode every concatenated LZ4 frame and, with defaults, request LZ4 for supported Thrift Arrow results (inline and CloudFetch) when the native codec is available; CloudFetch decoding yields between files but remains synchronous. Set `useLZ4Compression: false` on the client or statement to disable compression; compressed results may arrive inline instead of CloudFetch (for example, a 200,000-row `range()` result). +- The shared Arrow converter now raises `HiveDriverError` with `Arrow row count mismatch` instead of silently returning fewer rows than declared. This applies to every Arrow result path on both backends, including metadata, inline results and uncompressed CloudFetch. - Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index edba6002..e8995c5a 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -212,7 +212,7 @@ telemetry implementation, so only the options explicitly forwarded by | `runAsync` | `boolean` | ❌ | ✅ | — / `false` | Thrift always sends asynchronously and ignores the option. Kernel `false`/omitted uses direct execution; `true` submits and polls. This execute option is not deprecated. | | `maxRows` | `number \| bigint \| Int64 \| null` | ✅ | ❌ | `100000` / — | Thrift uses it for the initial Direct Results row count; `null` disables Direct Results. Kernel does not retain or forward the supplied value. | | `useCloudFetch` | `boolean` | ✅ | ❌ | `true` / — | Kernel logs and ignores this per-statement hint; Kernel result fetching owns its CloudFetch behavior. | -| `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift uses it when supported and when the result is not CloudFetch. Kernel owns and auto-detects result compression. | +| `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift requests LZ4 for supported Arrow results, inline and CloudFetch, when the native codec is available. Kernel owns and auto-detects result compression. | | `stagingAllowedLocalPath` | `string \| string[]` | ✅ | ❌ | — | Local allowlist for Thrift volume/staging operations. Kernel volume operations are unsupported. | | `namedParameters` | `Record` | ✅ | ✅ | — | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries. | | `ordinalParameters` | `Array` | ✅ | ✅ | — | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`. | diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index 441c495f..2ff174ec 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -346,7 +346,7 @@ export default class ArrowResultConverter implements IResultsProvider } const arrowBatch = await this.source.fetchNext(options); // eslint-disable-line no-await-in-loop - if (arrowBatch.batches.length > 0 && arrowBatch.rowCount > 0) { + if (arrowBatch.rowCount > 0) { const reader = RecordBatchReader.from(arrowBatch.batches); this.recordBatchReader = reader[Symbol.iterator](); this.remainingRows = arrowBatch.rowCount; @@ -359,6 +359,9 @@ export default class ArrowResultConverter implements IResultsProvider // handles readers which are already empty const item = this.recordBatchReader?.next() ?? { done: true, value: undefined }; if (item.done || item.value === undefined) { + if (this.remainingRows > 0) { + throw new HiveDriverError(`Arrow row count mismatch: ${this.remainingRows} rows missing`); + } this.recordBatchReader = undefined; } else { // Skip empty batches diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts index 71d8c44b..699aa681 100644 --- a/lib/result/CloudFetchResultHandler.ts +++ b/lib/result/CloudFetchResultHandler.ts @@ -94,11 +94,13 @@ export default class CloudFetchResultHandler implements IResultsProvider LZ4()!.decode(buffer)); + if (this.isLZ4Compressed && settled.batch.batches.length > 0) { + await new Promise((resolve) => { + setImmediate(resolve); + }); + settled.batch.batches = settled.batch.batches.flatMap((buffer) => LZ4()!.decodeFrames(buffer)); } - return batch; + return settled.batch; } private logDownloadMetrics(url: string, fileSizeBytes: number, downloadTimeMs: number): void { diff --git a/lib/thrift-backend/ThriftSessionBackend.ts b/lib/thrift-backend/ThriftSessionBackend.ts index 49ec539a..6695ac0c 100644 --- a/lib/thrift-backend/ThriftSessionBackend.ts +++ b/lib/thrift-backend/ThriftSessionBackend.ts @@ -220,7 +220,7 @@ export default class ThriftSessionBackend implements ISessionBackend { request.canDownloadResult = options.useCloudFetch ?? clientConfig.useCloudFetch; } - if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion) && request.canDownloadResult !== true) { + if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion)) { request.canDecompressLZ4Result = (options.useLZ4Compression ?? clientConfig.useLZ4Compression) && Boolean(LZ4()); } diff --git a/lib/utils/lz4.ts b/lib/utils/lz4.ts index 5a94d10f..8d32da17 100644 --- a/lib/utils/lz4.ts +++ b/lib/utils/lz4.ts @@ -14,15 +14,87 @@ // and cross-decoding both directions. We expose the same `{ encode, decode }` // surface the rest of the driver already calls, so call sites are unchanged. +import HiveDriverError from '../errors/HiveDriverError'; + type Lz4Napi = { compressFrameSync(data: Buffer): Buffer; decompressFrameSync(data: Buffer): Buffer; }; +type LZ4Frame = { bytes: Buffer; index: number; offset: number; contentSize?: bigint }; + // Stable interface used across the driver (lib call sites do `LZ4()!.decode(buf)`). export interface LZ4Codec { encode(data: Buffer): Buffer; decode(data: Buffer): Buffer; + decodeFrames(data: Buffer): Array; +} + +function frameError(frame: { index: number; offset: number }, reason: unknown): HiveDriverError { + return new HiveDriverError( + `LZ4 frame ${frame.index} at offset ${frame.offset}: ${reason instanceof Error ? reason.message : String(reason)}`, + ); +} + +function* readFrames(data: Buffer): Generator { + let offset = 0; + let index = 0; + let start = 0; + const advance = (size: number) => { + offset += size; + if (offset > data.length) { + throw frameError({ index, offset: start }, 'Truncated LZ4 frame'); + } + }; + + while (offset < data.length) { + start = offset; + const frame = { index, offset: start }; + advance(7); + if (data.readUInt32LE(start) !== 0x184d2204) { + throw frameError(frame, 'Invalid LZ4 frame magic'); + } + const flags = data[start + 4]; + let contentSize: bigint | undefined; + if (flags & 0x08) { + advance(8); + contentSize = data.readBigUInt64LE(start + 6); + } + if (flags & 0x01) advance(4); + + let blockSize: number; + do { + advance(4); + blockSize = data.readUInt32LE(offset - 4); + if (blockSize === 0x80000000) { + throw frameError(frame, 'Invalid zero-length LZ4 block'); + } + if (blockSize !== 0) { + advance((blockSize & 0x7fffffff) + (flags & 0x10 ? 4 : 0)); + } + } while (blockSize !== 0); + if (flags & 0x04) advance(4); + + yield { ...frame, bytes: data.subarray(start, offset), contentSize }; + index += 1; + } +} + +function validateContentSize(frame: LZ4Frame, decoded: Buffer): Buffer { + if (frame.contentSize !== undefined && BigInt(decoded.length) !== frame.contentSize) { + throw new Error(`Content size mismatch: expected ${frame.contentSize}, got ${decoded.length}`); + } + return decoded; +} + +function decodeFrames(data: Buffer, napi: Lz4Napi): Array { + return Array.from(readFrames(data), (frame) => { + try { + return validateContentSize(frame, napi.decompressFrameSync(frame.bytes)); + } catch (error) { + throw frameError(frame, error); + } + }); } function tryLoadLZ4Module(): LZ4Codec | undefined { @@ -47,7 +119,11 @@ function tryLoadLZ4Module(): LZ4Codec | undefined { // hot path (once per Arrow batch). return { encode: (data: Buffer): Buffer => napi.compressFrameSync(data), - decode: (data: Buffer): Buffer => napi.decompressFrameSync(data), + decode: (data: Buffer): Buffer => { + const frames = decodeFrames(data, napi); + return frames.length === 1 ? frames[0] : Buffer.concat(frames); + }, + decodeFrames: (data: Buffer): Array => decodeFrames(data, napi), }; } diff --git a/tests/e2e/cloudfetch.test.ts b/tests/e2e/cloudfetch.test.ts index 68a7a58c..2f379022 100644 --- a/tests/e2e/cloudfetch.test.ts +++ b/tests/e2e/cloudfetch.test.ts @@ -100,11 +100,11 @@ describe('CloudFetch', () => { expect(fetchedRowCount).to.be.equal(queriedRowsCount); }); - it('should not use LZ4 compression with cloud fetch', async () => { + it('should use LZ4 compression with cloud fetch', async () => { const cloudFetchConcurrentDownloads = 5; const session = await openSession({ cloudFetchConcurrentDownloads, - useLZ4Compression: true, // This is ignored when cloud fetch is enabled + useLZ4Compression: true, }); const queriedRowsCount = 10000000; // result has to be quite big to enable CloudFetch @@ -129,8 +129,7 @@ describe('CloudFetch', () => { expect(resultHandler).to.be.instanceof(ResultSlicer); expect(resultHandler.source).to.be.instanceof(ArrowResultConverter); expect(resultHandler.source.source).to.be.instanceOf(CloudFetchResultHandler); - // LZ4 compression should not be enabled with cloud fetch - expect(resultHandler.source.source.isLZ4Compressed).to.be.false; + expect(resultHandler.source.source.isLZ4Compressed).to.be.true; const chunk = await operation.fetchChunk({ maxRows: 100000, disableBuffering: true }); expect(chunk.length).to.be.gt(0); diff --git a/tests/e2e/lz4.test.ts b/tests/e2e/lz4.test.ts new file mode 100644 index 00000000..ac29231c --- /dev/null +++ b/tests/e2e/lz4.test.ts @@ -0,0 +1,64 @@ +import { createHash } from 'crypto'; +import { expect } from 'chai'; +import { DBSQLClient } from '../../lib'; +import CloudFetchResultHandler from '../../lib/result/CloudFetchResultHandler'; +import IOperation from '../../lib/contracts/IOperation'; +import IDBSQLSession from '../../lib/contracts/IDBSQLSession'; +import config from './utils/config'; + +describe('LZ4 result parity', () => { + for (const useCloudFetch of [false, true]) { + it(`preserves every value with CloudFetch=${useCloudFetch}`, async () => { + const client = new DBSQLClient(); + let operation: IOperation | undefined; + let session: IDBSQLSession | undefined; + try { + await client.connect({ host: config.host, path: config.path, token: config.token }); + const activeSession = await client.openSession(); + session = activeSession; + const read = async (useLZ4Compression: boolean) => { + operation = await activeSession.executeStatement( + `SELECT id, IF(id % 7 = 0, NULL, id * 0.125) AS value, + IF(id % 5 = 0, NULL, sha2(CAST(id AS STRING), 512)) AS text, + IF(id % 3 = 0, NULL, id % 2 = 0) AS flag + FROM range(500000) ORDER BY id`, + { useCloudFetch, useLZ4Compression, maxRows: null }, + ); + await operation.finished(); + const handler = await (operation as any).getResultHandler(); + expect(handler.source.source.isLZ4Compressed).to.equal(useLZ4Compression); + expect(handler.source.source instanceof CloudFetchResultHandler).to.equal(useCloudFetch); + const hash = createHash('sha256'); + let count = 0; + for await (const chunk of operation.iterateChunks({ maxRows: 11111 })) { + for (const row of chunk as Array<{ + id: number; + value: number | null; + text: string | null; + flag: boolean | null; + }>) { + expect(row.id).to.equal(count); + expect(row.value).to.equal(count % 7 === 0 ? null : count * 0.125); + expect(row.text).to.equal( + count % 5 === 0 ? null : createHash('sha512').update(String(count)).digest('hex'), + ); + expect(row.flag).to.equal(count % 3 === 0 ? null : count % 2 === 0); + hash.update(JSON.stringify(row)); + count += 1; + } + } + expect(count).to.equal(500000); + await operation.close(); + operation = undefined; + return hash.digest('hex'); + }; + const uncompressed = await read(false); + expect(await read(true)).to.equal(uncompressed); + } finally { + await operation?.close(); + await session?.close(); + await client.close(); + } + }); + } +}); diff --git a/tests/unit/DBSQLSession.test.ts b/tests/unit/DBSQLSession.test.ts index 00498c54..9381a226 100644 --- a/tests/unit/DBSQLSession.test.ts +++ b/tests/unit/DBSQLSession.test.ts @@ -165,12 +165,7 @@ describe('DBSQLSession', () => { } if (version >= TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6) { - // Since cloud fetch is enabled, canDecompressLZ4Result should not be set - if (req.canDownloadResult === true) { - expect(req.canDecompressLZ4Result).to.not.be.true; - } else { - expect(req.canDecompressLZ4Result).to.be.true; - } + expect(req.canDecompressLZ4Result).to.be.true; } else { expect(req.canDecompressLZ4Result).to.not.be.true; } @@ -194,7 +189,24 @@ describe('DBSQLSession', () => { }); describe('LZ4 compression with cloud fetch', () => { - it('should not set canDecompressLZ4Result when cloud fetch is enabled (canDownloadResult=true)', async () => { + for (const clientSetting of [false, true]) { + for (const statementSetting of [undefined, false, true]) { + it(`honors client=${clientSetting}, statement=${statementSetting}`, async () => { + const context = new ClientContextStub({ useLZ4Compression: clientSetting }); + const driver = sinon.spy(context.driver); + const session = createSessionForTest({ + handle: sessionHandleStub, + context, + serverProtocolVersion: TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6, + }); + await session.executeStatement('SELECT 1', { useCloudFetch: true, useLZ4Compression: statementSetting }); + expect(driver.executeStatement.firstCall.args[0].canDecompressLZ4Result).to.equal( + statementSetting ?? clientSetting, + ); + }); + } + } + it('should set canDecompressLZ4Result when cloud fetch is enabled (canDownloadResult=true)', async () => { const context = new ClientContextStub({ useLZ4Compression: true }); const driver = sinon.spy(context.driver); const statement = 'SELECT * FROM table'; @@ -212,9 +224,8 @@ describe('DBSQLSession', () => { expect(driver.executeStatement.callCount).to.eq(1); const req = driver.executeStatement.firstCall.args[0]; - // canDownloadResult should be true and canDecompressLZ4Result should NOT be set expect(req.canDownloadResult).to.be.true; - expect(req.canDecompressLZ4Result).to.not.be.true; + expect(req.canDecompressLZ4Result).to.be.true; }); it('should set canDecompressLZ4Result when cloud fetch is disabled (canDownloadResult=false)', async () => { diff --git a/tests/unit/result/ArrowResultConverter.test.ts b/tests/unit/result/ArrowResultConverter.test.ts index 363b3cc8..ce657364 100644 --- a/tests/unit/result/ArrowResultConverter.test.ts +++ b/tests/unit/result/ArrowResultConverter.test.ts @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; import { Table, tableFromArrays, tableToIPC, RecordBatch, TypeMap } from 'apache-arrow'; import ArrowResultConverter, { bigNumDecimalToString } from '../../../lib/result/ArrowResultConverter'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; import { ArrowBatch } from '../../../lib/result/utils'; import ResultsProviderStub from '../.stubs/ResultsProviderStub'; import { TTableSchema, TTypeId } from '../../../thrift/TCLIService_types'; @@ -87,6 +88,28 @@ function createSampleArrowBatch(...recordBatches: RecordBatch } describe('ArrowResultConverter', () => { + for (const disableRowMaterialization of [false, true]) { + for (const batches of [sampleArrowBatch, []]) { + it(`rejects missing rows, materialization disabled=${disableRowMaterialization}, buffers=${batches.length}`, async () => { + const source = new ResultsProviderStub([{ batches, rowCount: 2 }], emptyItem); + const converter = new ArrowResultConverter( + new ClientContextStub(), + source, + { schema: sampleThriftSchema }, + { disableRowMaterialization }, + ); + let caught: unknown; + try { + await converter.fetchNext({ limit: 10000 }); + } catch (error) { + caught = error; + } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal(`Arrow row count mismatch: ${batches.length ? 1 : 2} rows missing`); + }); + } + } + it('should convert data', async () => { const rowSetProvider = new ResultsProviderStub( [ diff --git a/tests/unit/result/CompressedArrow.test.ts b/tests/unit/result/CompressedArrow.test.ts new file mode 100644 index 00000000..5b381fae --- /dev/null +++ b/tests/unit/result/CompressedArrow.test.ts @@ -0,0 +1,207 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import Int64 from 'node-int64'; +import { Response } from 'node-fetch'; +import { + Binary, + Bool, + Float64, + RecordBatchReader, + Table, + Utf8, + tableFromIPC, + tableToIPC, + vectorFromArray, +} from 'apache-arrow'; +import ArrowResultHandler from '../../../lib/result/ArrowResultHandler'; +import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler'; +import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; +import { TStatusCode, TTypeId } from '../../../thrift/TCLIService_types'; +import ResultsProviderStub from '../.stubs/ResultsProviderStub'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import getLZ4 from '../../../lib/utils/lz4'; + +describe('concatenated LZ4 Arrow results', () => { + let codec: ReturnType; + + before(function () { + codec = getLZ4(); + if (!codec) this.skip(); + }); + + afterEach(() => sinon.restore()); + + const table = new Table({ + number: vectorFromArray([1, null, 3.5], new Float64()), + text: vectorFromArray(['first', 'second', null], new Utf8()), + boolean: vectorFromArray([true, null, false], new Bool()), + binary: vectorFromArray([Buffer.from([0, 255]), null, Buffer.alloc(0)], new Binary()), + }); + const ipc = Buffer.from(tableToIPC(table)); + const schemaEnd = 8 + ipc.readInt32LE(4); + const schema = ipc.subarray(0, schemaEnd); + const payload = ipc.subarray(schemaEnd); + const compress = (bytes: Buffer) => + Buffer.concat([ + codec!.encode(bytes.subarray(0, 16)), + codec!.encode(Buffer.alloc(0)), + codec!.encode(bytes.subarray(16)), + ]); + + const createCloudFetch = (body: Buffer, rowCount: number, compressed = true, linkCount = 1) => { + const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 2 }); + const resultLinks = Array.from({ length: linkCount }, (_, index) => ({ + fileLink: `http://localhost/result/${index}`, + rowCount: new Int64(rowCount), + startRowOffset: new Int64(0), + expiryTime: new Int64(Date.now() + 60000), + bytesNum: new Int64(body.length), + })); + const handler = new CloudFetchResultHandler( + context, + new ResultsProviderStub([{ startRowOffset: new Int64(0), rows: [], resultLinks }], undefined), + { lz4Compressed: compressed, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, + ); + sinon.stub(handler as any, 'fetch').resolves({ response: new Response(), body }); + return { handler, context }; + }; + + const thriftSchema = { + columns: [ + ['number', TTypeId.DOUBLE_TYPE], + ['text', TTypeId.STRING_TYPE], + ['boolean', TTypeId.BOOLEAN_TYPE], + ['binary', TTypeId.BINARY_TYPE], + ].map(([columnName, type], position) => ({ + columnName: columnName as string, + position, + typeDesc: { types: [{ primitiveEntry: { type: type as TTypeId } }] }, + })), + }; + + it('preserves inline values, types, nulls and bytes across frame boundaries', async () => { + const handler = new ArrowResultHandler( + new ClientContextStub(), + new ResultsProviderStub( + [ + { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [{ batch: compress(payload), rowCount: new Int64(3) }], + }, + ], + undefined, + ), + { arrowSchema: schema, lz4Compressed: true, status: { statusCode: TStatusCode.SUCCESS_STATUS } }, + ); + const result = await handler.fetchNext({ limit: 1 }); + const bytes = Buffer.concat(result.batches); + expect(bytes).to.deep.equal(ipc); + expect(result.rowCount).to.equal(3); + expect(tableFromIPC(bytes).toArray()).to.deep.equal(table.toArray()); + expect(tableFromIPC(bytes).schema).to.deep.equal(table.schema); + }); + + for (const compressed of [false, true]) { + for (const expectedRows of [0, 2, 3, 4]) { + it(`caps or validates CloudFetch rows ${expectedRows}, compressed=${compressed}`, async () => { + const readers = sinon.spy(RecordBatchReader, 'from'); + const { handler, context } = createCloudFetch(compressed ? compress(ipc) : ipc, expectedRows, compressed); + const converter = new ArrowResultConverter(context, handler, { schema: thriftSchema }); + if (expectedRows <= 3) { + expect(await converter.fetchNext({ limit: 10 })).to.deep.equal( + [ + { number: 1, text: 'first', boolean: true, binary: Buffer.from([0, 255]) }, + { number: null, text: 'second', boolean: null, binary: null }, + { number: 3.5, text: null, boolean: false, binary: Buffer.alloc(0) }, + ].slice(0, expectedRows), + ); + expect(await converter.hasMore()).to.equal(false); + } else { + let caught: unknown; + try { + await converter.fetchNext({ limit: 10 }); + } catch (error) { + caught = error; + } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal('Arrow row count mismatch: 1 rows missing'); + } + expect(readers.callCount).to.equal(expectedRows === 0 ? 0 : 1); + }); + } + + it(`rejects an empty payload with declared rows, compressed=${compressed}`, async () => { + const { handler, context } = createCloudFetch( + compressed ? codec!.encode(Buffer.alloc(0)) : Buffer.alloc(0), + 1, + compressed, + ); + const converter = new ArrowResultConverter(context, handler, { schema: thriftSchema }); + let caught: unknown; + try { + await converter.fetchNext({ limit: 10 }); + } catch (error) { + caught = error; + } + expect(caught).to.be.instanceOf(HiveDriverError); + expect((caught as Error).message).to.equal('Arrow row count mismatch: 1 rows missing'); + }); + } + + it('preserves complete Arrow batches in separate frames', async () => { + const parts = [ipc.subarray(0, -8), ipc.subarray(schemaEnd)]; + const { handler } = createCloudFetch(Buffer.concat(parts.map((part) => codec!.encode(part))), 6); + const result = await handler.fetchNext({ limit: 1 }); + expect(result.rowCount).to.equal(6); + expect(result.batches).to.deep.equal(parts); + expect(tableFromIPC(result.batches).toArray()).to.deep.equal([...table.toArray(), ...table.toArray()]); + }); + + it('yields to the event loop before decoding each prefetched file', async () => { + let turn = 0; + const decodedTurns: Array = []; + const decode = sinon.stub(codec!, 'decodeFrames').callsFake(() => { + decodedTurns.push(turn); + return [ipc]; + }); + const { handler } = createCloudFetch(compress(ipc), 3, true, 3); + for await (const index of [0, 1, 2]) { + setImmediate(() => { + turn = index + 1; + }); + const next = handler.fetchNext({ limit: 10 }); + expect(decode.callCount).to.equal(index); + expect((await next).batches).to.deep.equal([ipc]); + expect(decodedTurns[index]).to.equal(index + 1); + } + expect(decode.callCount).to.equal(3); + }); + + it('does not decode abandoned prefetched files and surfaces decode errors only when consumed', async () => { + const unhandled: Array = []; + const onUnhandled = (reason: unknown) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + const error = new HiveDriverError('LZ4 frame 1 at offset 10: ContentChecksumError'); + const decode = sinon.stub(codec!, 'decodeFrames'); + decode.onFirstCall().returns([ipc]); + decode.onSecondCall().throws(error); + const { handler } = createCloudFetch(compress(ipc), 3, true, 2); + expect((await handler.fetchNext({ limit: 10 })).batches).to.deep.equal([ipc]); + await new Promise(setImmediate); + expect(decode.callCount).to.equal(1); + expect(unhandled).to.deep.equal([]); + let caught: unknown; + try { + await handler.fetchNext({ limit: 10 }); + } catch (reason) { + caught = reason; + } + expect(caught).to.equal(error); + } finally { + process.off('unhandledRejection', onUnhandled); + } + }); +}); diff --git a/tests/unit/utils/fixtures/lz4-frames.json b/tests/unit/utils/fixtures/lz4-frames.json new file mode 100644 index 00000000..25ca25f4 --- /dev/null +++ b/tests/unit/utils/fixtures/lz4-frames.json @@ -0,0 +1,82 @@ +[ + { + "linked": false, + "contentChecksum": false, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1868408884020000000000862c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637382c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536b0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a00000000" + }, + { + "linked": false, + "contentChecksum": false, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1878408884020000000000e62c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab2c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536d1e5fa6fb0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a0d4c81a400000000" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d186c408884020000000000482c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637382c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536b0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a00000000b5dab6f5" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d187c408884020000000000312c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab2c010000ff12390a44617461627269636b73204c5a34206672616d653a203031323334353637382100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc7503233343536d1e5fa6fb0000000ff123738390a44617461627269636b73204c5a34206672616d653a20303132333435362100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd350363738390a0d4c81a400000000b5dab6f5" + }, + { + "linked": true, + "contentChecksum": false, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1848408884020000000000102c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637380a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe85032333435368e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390a00000000" + }, + { + "linked": true, + "contentChecksum": false, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d1858408884020000000000942c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab0a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe850323334353698d1b0d08e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390ae007cfb600000000" + }, + { + "linked": true, + "contentChecksum": true, + "blockChecksum": false, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d184c408884020000000000fe2c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637380a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe85032333435368e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390a00000000b5dab6f5" + }, + { + "linked": true, + "contentChecksum": true, + "blockChecksum": true, + "text": "Databricks LZ4 frame: 0123456789\n", + "repeat": 5000, + "hex": "04224d185c4088840200000000001e2c010000ff1244617461627269636b73204c5a34206672616d653a20303132333435363738390a2100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc75034353637384561d7ab0a0100000fe1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe850323334353698d1b0d08e0000000f2100fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff450363738390ae007cfb600000000b5dab6f5" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": true, + "text": "", + "repeat": 1, + "hex": "04224d187440bd00000000055dcc02" + }, + { + "linked": false, + "contentChecksum": true, + "blockChecksum": true, + "text": "].4U85YnN0C8 zu7!6-n#OXNZSRgRx)+wS\"ENL>=D22>&8^Z9 ?KNU-o:/!v[YeuBn6nx&0sFPGsf52+\\V1S@Z;i^o0zfVf4i]m&LXos{xzh2H[ /QX)mu1MLP;3:2{iHNZ>7H0/AmK$3u&eEB=OAfLRX9>_X10\"NC4IU&3s=CH|^xm1_*Yc+~WcF;x?|f3s}*7S,!\"vR5CE;=7[|V$:JZ:mYvxtsMe}+E7Nz]g50*b|k_EHEL6tl]'R$Lzi(9j]Teud)7b1'j)C;}np~8u=:\"]qftSeJX){Y)4lfXmx'J\"RaLkIGY\\HIyK)n{? ml;$D{jr00)x[:@954gvH*A|l<}~11,7Z;FwzyBrt0}.rEHZA'Tq3|Ee(.P1kk0JU/<^iQZ:F)K%5JjmJ_%7XoStgq|Un4Tr]/,#=c@bA*08q@}P^v53'Gama)t6*P>]Q]9;5=L$,(X*j!ZL}jZ'6#SN3<'# T@x^hGWr=Qjt+e mO_e`?_kN ,c{x|8+0hhZJwTY%wk;OIZa\\)=Sa[=WmANYP&1Jqgtu..'ixd:B1|7`e/z=o2/Yb#NG#Zt:{>R`P\\~E\"`U4H=G`MU5]?|(p-PiA'}ki2LJ2>C0nBg~l.)E+RO=P;_3)", + "repeat": 1, + "hex": "04224d187c40000400000000000085000400805d2e34553835596e4e304338207a753721362d6e234f584e5a5352675278292b775322454e4c3e3d4432323e26385e5a39203f4b4e552d6f3a2f21763c6d5c2646407c31566635593d7b396f6e534e446268203a2f702b6f372024703c616a745e257c4a2e5d2a7a5a63426b3b2b793734746e44695c2b7650204669296e5c3a255d554c71434a6577676d3b634c4b324d234d5e7b34412d5e5637447e582e3d48357750542f7c25756e7d3f393465414c775a26445c3d7b27207927634c26322b7e34403b3b7d68474135522f365d432257323b7e25642e5f525752722b492773757c6927642d382a34395f6a2f27435d3e5b596575426e366e78263073465047736635322b5c563153405a3b695e6f307a66566634695d6d264c586f737b787a6832485b202f5158296d75314d4c503b333a327b69484e5a3e3748302f416d4b243375266545423d4f41664c5258393e5f583130224e433449552633733d43487c5e786d315f2a59632b7e5763463b783f7c6633737d2a37532c212276523543453b3d375b7c56243a4a5a3a6d59767874734d657d2b45374e7a5d6735302a627c6b5f4548454c36746c5d2752244c7a6928396a5d5465756429376231276a29433b7d6e707e38753d3a225d71667453654a58297b5929346c66586d78274a2252614c6b4947595c4849794b296e7b3f206d6c3b24447b6a72303029785b3a403935346776482a417c6c3c7d7e31312c373c3f77543a6465343c63375f2e32577720507c3632482d6a794051266835704b2a455a6f7d615f476f5d5a6869562a233c6a335938406b23353d36733a31706350525e2b335752686e694a714d5d4f4c22523e5a3b46777a79427274307d2e7245485a41275471337c4565282e50316b6b304a552f3c5e693c6c7c4c5c416f2e7230285e3b6a5b276d5e4d386a3d346a4425495b774664465e7c5545346e6a5a2e527444312d582b6325542f4c4c413a4e2b5c33592e4830565d5c7b543f6e6c3a215c225f493d636068375f386c736f3462233d3a4f3957614b57706845625274325234217a7a632d7b7132402c226e7020313d35666237464f7e2b713e515a3a46294b25354a6a6d4a5f2537586f537467717c556e3454725d2f2c233d634062412a303871407d505e7635332747616d612974362a503e5d515d393b353d4c242c28582a6a215a4c7d6a5a273623534e333c2723205440785e684757723d516a742b65206d4f5f65603f5f6b4e202c637b787c382b3068685a4a77545925776b3b4f495a615c293d53615b3d576d414e595026314a716774752e2e276978643a42317c3760652f7a3d6f322f5962234e47235a743a7b3e5260505c7e4522605534483d47604d55355d3f7c28702d506941277d6b69324c4a323e43306e42677e6c2e29452b524f3d503b5f33294bbb308b000000004bbb308b" + } +] diff --git a/tests/unit/utils/lz4.test.ts b/tests/unit/utils/lz4.test.ts index 95713e47..8aebcdb8 100644 --- a/tests/unit/utils/lz4.test.ts +++ b/tests/unit/utils/lz4.test.ts @@ -1,6 +1,9 @@ import { expect } from 'chai'; import sinon from 'sinon'; +import fs from 'fs'; +import path from 'path'; import getLZ4 from '../../../lib/utils/lz4'; +import HiveDriverError from '../../../lib/errors/HiveDriverError'; // Exercises the lz4 loader (lib/utils/lz4.ts), which now wraps `lz4-napi` // and exposes a stable { encode, decode } frame codec with a @@ -69,9 +72,11 @@ describe('lz4 module loader', function () { it('exposes an { encode, decode } codec backed by lz4-napi frame APIs', () => { // Fake lz4-napi: frame APIs the adapter calls. + const testData = Buffer.from('Hello, World!'); + const encoded = Buffer.from([4, 34, 77, 24, 96, 64, 130, 0, 0, 0, 0]); const fakeNapi = { - compressFrameSync: (buf: Buffer) => Buffer.from(`frame:${buf.toString()}`), - decompressFrameSync: (buf: Buffer) => Buffer.from(buf.toString().replace('frame:', '')), + compressFrameSync: sinon.stub().returns(encoded), + decompressFrameSync: sinon.stub().returns(testData), }; mockNapiLoad(fakeNapi); @@ -81,10 +86,26 @@ describe('lz4 module loader', function () { expect(lz4Module.encode).to.be.a('function'); expect(lz4Module.decode).to.be.a('function'); - const testData = Buffer.from('Hello, World!'); const roundTripped = lz4Module.decode(lz4Module.encode(testData)); expect(roundTripped.toString()).to.equal('Hello, World!'); expect(consoleWarnStub.called).to.be.false; + expect(fakeNapi.decompressFrameSync.calledOnceWithExactly(encoded)).to.be.true; + }); + + it('rejects decoded lengths inconsistent with the declared content size', () => { + const encoded = Buffer.alloc(19); + encoded.writeUInt32LE(0x184d2204); + encoded[4] = 0x68; + encoded[5] = 0x40; + encoded.writeBigUInt64LE(BigInt(5), 6); + mockNapiLoad({ + decompressFrameSync: () => Buffer.from('four'), + }); + const codec = loadLz4Module().default(); + expect(() => codec.decode(encoded)).to.throw( + HiveDriverError, + 'LZ4 frame 0 at offset 0: Content size mismatch: expected 5, got 4', + ); }); it('returns undefined (no warning) when lz4-napi is not installed (MODULE_NOT_FOUND)', () => { @@ -194,4 +215,89 @@ describe('lz4 frame codec (real lz4-napi)', function () { expect(codec!.decode(encoded).equals(payload)).to.be.true; } }); + + it('decodes every concatenated frame, including empty frames', () => { + const payloads = [Buffer.from('first'), Buffer.alloc(0), Buffer.from('second'), Buffer.from('third')]; + const input = Buffer.concat(payloads.map((payload) => codec!.encode(payload))); + expect(codec!.decode(input)).to.deep.equal(Buffer.concat(payloads)); + expect(codec!.decodeFrames(input)).to.deep.equal(payloads); + }); + + it('rejects truncated frames and trailing bytes instead of returning partial results', () => { + const frame = codec!.encode(Buffer.from('first')); + for (let length = 1; length < frame.length; length += 1) { + expect(() => codec!.decode(frame.subarray(0, length)), `truncated at ${length}`).to.throw(); + expect(() => codec!.decode(Buffer.concat([frame, frame.subarray(0, length)]))).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}`, + ); + } + expect(() => codec!.decode(Buffer.concat([frame, Buffer.from('trailing bytes')]))).to.throw(); + }); + + it('rejects 0x80000000 rather than letting the native decoder silently stop at it', () => { + const frame = codec!.encode(Buffer.from('first')); + const headerLength = 7 + (frame[4] & 0x08 ? 8 : 0) + (frame[4] & 0x01 ? 4 : 0); + const emptyBlock = Buffer.from([0, 0, 0, 128]); + const corrupt = Buffer.concat([frame.subarray(0, headerLength), emptyBlock, frame.subarray(headerLength)]); + expect(() => codec!.decode(corrupt)).to.throw(HiveDriverError, 'Invalid zero-length LZ4 block'); + expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}: Invalid zero-length LZ4 block`, + ); + }); + + const fixtures: Array<{ + linked: boolean; + contentChecksum: boolean; + blockChecksum: boolean; + text: string; + repeat: number; + hex: string; + }> = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures/lz4-frames.json'), 'utf8')); + + for (const fixture of fixtures) { + const name = `linked=${fixture.linked}, content checksum=${fixture.contentChecksum}, block checksum=${ + fixture.blockChecksum + }, bytes=${Buffer.byteLength(fixture.text) * fixture.repeat}`; + it(`decodes reference liblz4 frames (${name})`, () => { + const frame = Buffer.from(fixture.hex, 'hex'); + const payload = Buffer.from(fixture.text.repeat(fixture.repeat)); + expect(codec!.decode(Buffer.concat([frame, frame]))).to.deep.equal(Buffer.concat([payload, payload])); + expect(codec!.decodeFrames(frame)).to.deep.equal([payload]); + }); + + it(`rejects corrupt reference liblz4 frames (${name})`, () => { + const frame = Buffer.from(fixture.hex, 'hex'); + const blockStart = frame[4] & 0x08 ? 15 : 7; + const offsets = [0, blockStart - 1]; + if (fixture.contentChecksum) offsets.push(frame.length - 1); + if (fixture.blockChecksum && frame.readUInt32LE(blockStart) !== 0) { + offsets.push(blockStart + 4 + (frame.readUInt32LE(blockStart) & 0x7fffffff)); + } + for (const offset of offsets) { + const corrupt = Buffer.from(frame); + corrupt[offset] ^= 1; + expect(() => codec!.decode(corrupt)).to.throw(); + expect(() => codec!.decode(Buffer.concat([frame, corrupt]))).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}`, + ); + } + for (let missing = 1; missing <= 8; missing += 1) { + expect(() => codec!.decode(frame.subarray(0, -missing))).to.throw(); + } + }); + } + + it('preserves the native checksum failure reason and failing frame location', () => { + const frame = Buffer.from(fixtures.find((fixture) => fixture.contentChecksum)!.hex, 'hex'); + const corrupt = Buffer.from(frame); + corrupt[corrupt.length - 1] ^= 1; + const input = Buffer.concat([frame, corrupt]); + expect(() => codec!.decode(input)).to.throw( + HiveDriverError, + `LZ4 frame 1 at offset ${frame.length}: ContentChecksumError`, + ); + }); });