Skip to content
Open
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## Unreleased

- Decode every concatenated LZ4 frame and, with defaults, request LZ4 for supported Thrift Arrow results (inline and CloudFetch) when the native codec is available; CloudFetch decoding yields between files but remains synchronous. Set `useLZ4Compression: false` on the client or statement to disable compression; compressed results may arrive inline instead of CloudFetch (for example, a 200,000-row `range()` result).
- The shared Arrow converter now raises `HiveDriverError` with `Arrow row count mismatch` instead of silently returning fewer rows than declared. This applies to every Arrow result path on both backends, including metadata, inline results and uncompressed CloudFetch.
- Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends.
- Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged.
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
Expand Down
2 changes: 1 addition & 1 deletion CONNECTION_PARAMETERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ telemetry implementation, so only the options explicitly forwarded by
| `runAsync` | `boolean` | ❌ | ✅ | — / `false` | Thrift always sends asynchronously and ignores the option. Kernel `false`/omitted uses direct execution; `true` submits and polls. This execute option is not deprecated. |
| `maxRows` | `number \| bigint \| Int64 \| null` | ✅ | ❌ | `100000` / — | Thrift uses it for the initial Direct Results row count; `null` disables Direct Results. Kernel does not retain or forward the supplied value. |
| `useCloudFetch` | `boolean` | ✅ | ❌ | `true` / — | Kernel logs and ignores this per-statement hint; Kernel result fetching owns its CloudFetch behavior. |
| `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift uses it when supported and when the result is not CloudFetch. Kernel owns and auto-detects result compression. |
| `useLZ4Compression` | `boolean` | ✅ | ❌ | `true` / — | Thrift requests LZ4 for supported Arrow results, inline and CloudFetch, when the native codec is available. Kernel owns and auto-detects result compression. |
| `stagingAllowedLocalPath` | `string \| string[]` | ✅ | ❌ | — | Local allowlist for Thrift volume/staging operations. Kernel volume operations are unsupported. |
| `namedParameters` | `Record<string, DBSQLParameter \| DBSQLParameterValue>` | ✅ | ✅ | — | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries. |
| `ordinalParameters` | `Array<DBSQLParameter \| DBSQLParameterValue>` | ✅ | ✅ | — | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`. |
Expand Down
5 changes: 4 additions & 1 deletion lib/result/ArrowResultConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
}

const arrowBatch = await this.source.fetchNext(options); // eslint-disable-line no-await-in-loop
if (arrowBatch.batches.length > 0 && arrowBatch.rowCount > 0) {
if (arrowBatch.rowCount > 0) {
const reader = RecordBatchReader.from<TypeMap>(arrowBatch.batches);
this.recordBatchReader = reader[Symbol.iterator]();
this.remainingRows = arrowBatch.rowCount;
Expand All @@ -359,6 +359,9 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
// handles readers which are already empty
const item = this.recordBatchReader?.next() ?? { done: true, value: undefined };
if (item.done || item.value === undefined) {
if (this.remainingRows > 0) {
throw new HiveDriverError(`Arrow row count mismatch: ${this.remainingRows} rows missing`);
}
this.recordBatchReader = undefined;
} else {
// Skip empty batches
Expand Down
10 changes: 6 additions & 4 deletions lib/result/CloudFetchResultHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,13 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
throw settled.error;
}

const { batch } = settled;
if (this.isLZ4Compressed) {
batch.batches = batch.batches.map((buffer) => LZ4()!.decode(buffer));
if (this.isLZ4Compressed && settled.batch.batches.length > 0) {
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
settled.batch.batches = settled.batch.batches.flatMap((buffer) => LZ4()!.decodeFrames(buffer));
}
return batch;
return settled.batch;
}

private logDownloadMetrics(url: string, fileSizeBytes: number, downloadTimeMs: number): void {
Expand Down
2 changes: 1 addition & 1 deletion lib/thrift-backend/ThriftSessionBackend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ export default class ThriftSessionBackend implements ISessionBackend {
request.canDownloadResult = options.useCloudFetch ?? clientConfig.useCloudFetch;
}

if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion) && request.canDownloadResult !== true) {
if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion)) {
request.canDecompressLZ4Result = (options.useLZ4Compression ?? clientConfig.useLZ4Compression) && Boolean(LZ4());
}

Expand Down
78 changes: 77 additions & 1 deletion lib/utils/lz4.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,87 @@
// and cross-decoding both directions. We expose the same `{ encode, decode }`
// surface the rest of the driver already calls, so call sites are unchanged.

import HiveDriverError from '../errors/HiveDriverError';

type Lz4Napi = {
compressFrameSync(data: Buffer): Buffer;
decompressFrameSync(data: Buffer): Buffer;
};

type LZ4Frame = { bytes: Buffer; index: number; offset: number; contentSize?: bigint };

// Stable interface used across the driver (lib call sites do `LZ4()!.decode(buf)`).
export interface LZ4Codec {
encode(data: Buffer): Buffer;
decode(data: Buffer): Buffer;
decodeFrames(data: Buffer): Array<Buffer>;
}

function frameError(frame: { index: number; offset: number }, reason: unknown): HiveDriverError {
return new HiveDriverError(
`LZ4 frame ${frame.index} at offset ${frame.offset}: ${reason instanceof Error ? reason.message : String(reason)}`,
);
}

function* readFrames(data: Buffer): Generator<LZ4Frame> {
let offset = 0;
let index = 0;
let start = 0;
const advance = (size: number) => {
offset += size;
if (offset > data.length) {
throw frameError({ index, offset: start }, 'Truncated LZ4 frame');
}
};

while (offset < data.length) {
start = offset;
const frame = { index, offset: start };
advance(7);
if (data.readUInt32LE(start) !== 0x184d2204) {
throw frameError(frame, 'Invalid LZ4 frame magic');
}
const flags = data[start + 4];
let contentSize: bigint | undefined;
if (flags & 0x08) {
advance(8);
contentSize = data.readBigUInt64LE(start + 6);
}
if (flags & 0x01) advance(4);

let blockSize: number;
do {
advance(4);
blockSize = data.readUInt32LE(offset - 4);
if (blockSize === 0x80000000) {
throw frameError(frame, 'Invalid zero-length LZ4 block');
}
if (blockSize !== 0) {
advance((blockSize & 0x7fffffff) + (flags & 0x10 ? 4 : 0));
}
} while (blockSize !== 0);
if (flags & 0x04) advance(4);

yield { ...frame, bytes: data.subarray(start, offset), contentSize };
index += 1;
}
}

function validateContentSize(frame: LZ4Frame, decoded: Buffer): Buffer {
if (frame.contentSize !== undefined && BigInt(decoded.length) !== frame.contentSize) {
throw new Error(`Content size mismatch: expected ${frame.contentSize}, got ${decoded.length}`);
}
return decoded;
}

function decodeFrames(data: Buffer, napi: Lz4Napi): Array<Buffer> {
return Array.from(readFrames(data), (frame) => {
try {
return validateContentSize(frame, napi.decompressFrameSync(frame.bytes));
} catch (error) {
throw frameError(frame, error);
}
});
}

function tryLoadLZ4Module(): LZ4Codec | undefined {
Expand All @@ -47,7 +119,11 @@ function tryLoadLZ4Module(): LZ4Codec | undefined {
// hot path (once per Arrow batch).
return {
encode: (data: Buffer): Buffer => napi.compressFrameSync(data),
decode: (data: Buffer): Buffer => napi.decompressFrameSync(data),
decode: (data: Buffer): Buffer => {
const frames = decodeFrames(data, napi);
return frames.length === 1 ? frames[0] : Buffer.concat(frames);
},
decodeFrames: (data: Buffer): Array<Buffer> => decodeFrames(data, napi),
};
}

Expand Down
7 changes: 3 additions & 4 deletions tests/e2e/cloudfetch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,11 @@ describe('CloudFetch', () => {
expect(fetchedRowCount).to.be.equal(queriedRowsCount);
});

it('should not use LZ4 compression with cloud fetch', async () => {
it('should use LZ4 compression with cloud fetch', async () => {
const cloudFetchConcurrentDownloads = 5;
const session = await openSession({
cloudFetchConcurrentDownloads,
useLZ4Compression: true, // This is ignored when cloud fetch is enabled
useLZ4Compression: true,
});

const queriedRowsCount = 10000000; // result has to be quite big to enable CloudFetch
Expand All @@ -129,8 +129,7 @@ describe('CloudFetch', () => {
expect(resultHandler).to.be.instanceof(ResultSlicer);
expect(resultHandler.source).to.be.instanceof(ArrowResultConverter);
expect(resultHandler.source.source).to.be.instanceOf(CloudFetchResultHandler);
// LZ4 compression should not be enabled with cloud fetch
expect(resultHandler.source.source.isLZ4Compressed).to.be.false;
expect(resultHandler.source.source.isLZ4Compressed).to.be.true;

const chunk = await operation.fetchChunk({ maxRows: 100000, disableBuffering: true });
expect(chunk.length).to.be.gt(0);
Expand Down
64 changes: 64 additions & 0 deletions tests/e2e/lz4.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { createHash } from 'crypto';
import { expect } from 'chai';
import { DBSQLClient } from '../../lib';
import CloudFetchResultHandler from '../../lib/result/CloudFetchResultHandler';
import IOperation from '../../lib/contracts/IOperation';
import IDBSQLSession from '../../lib/contracts/IDBSQLSession';
import config from './utils/config';

describe('LZ4 result parity', () => {
for (const useCloudFetch of [false, true]) {
it(`preserves every value with CloudFetch=${useCloudFetch}`, async () => {
const client = new DBSQLClient();
let operation: IOperation | undefined;
let session: IDBSQLSession | undefined;
try {
await client.connect({ host: config.host, path: config.path, token: config.token });
const activeSession = await client.openSession();
session = activeSession;
const read = async (useLZ4Compression: boolean) => {
operation = await activeSession.executeStatement(
`SELECT id, IF(id % 7 = 0, NULL, id * 0.125) AS value,
IF(id % 5 = 0, NULL, sha2(CAST(id AS STRING), 512)) AS text,
IF(id % 3 = 0, NULL, id % 2 = 0) AS flag
FROM range(500000) ORDER BY id`,
{ useCloudFetch, useLZ4Compression, maxRows: null },
);
await operation.finished();
const handler = await (operation as any).getResultHandler();
expect(handler.source.source.isLZ4Compressed).to.equal(useLZ4Compression);
expect(handler.source.source instanceof CloudFetchResultHandler).to.equal(useCloudFetch);
const hash = createHash('sha256');
let count = 0;
for await (const chunk of operation.iterateChunks({ maxRows: 11111 })) {
for (const row of chunk as Array<{
id: number;
value: number | null;
text: string | null;
flag: boolean | null;
}>) {
expect(row.id).to.equal(count);
expect(row.value).to.equal(count % 7 === 0 ? null : count * 0.125);
expect(row.text).to.equal(
count % 5 === 0 ? null : createHash('sha512').update(String(count)).digest('hex'),
);
expect(row.flag).to.equal(count % 3 === 0 ? null : count % 2 === 0);
hash.update(JSON.stringify(row));
count += 1;
}
}
expect(count).to.equal(500000);
await operation.close();
operation = undefined;
return hash.digest('hex');
};
const uncompressed = await read(false);
expect(await read(true)).to.equal(uncompressed);
} finally {
await operation?.close();
await session?.close();
await client.close();
}
});
}
});
29 changes: 20 additions & 9 deletions tests/unit/DBSQLSession.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -165,12 +165,7 @@ describe('DBSQLSession', () => {
}

if (version >= TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6) {
// Since cloud fetch is enabled, canDecompressLZ4Result should not be set
if (req.canDownloadResult === true) {
expect(req.canDecompressLZ4Result).to.not.be.true;
} else {
expect(req.canDecompressLZ4Result).to.be.true;
}
expect(req.canDecompressLZ4Result).to.be.true;
} else {
expect(req.canDecompressLZ4Result).to.not.be.true;
}
Expand All @@ -194,7 +189,24 @@ describe('DBSQLSession', () => {
});

describe('LZ4 compression with cloud fetch', () => {
it('should not set canDecompressLZ4Result when cloud fetch is enabled (canDownloadResult=true)', async () => {
for (const clientSetting of [false, true]) {
for (const statementSetting of [undefined, false, true]) {
it(`honors client=${clientSetting}, statement=${statementSetting}`, async () => {
const context = new ClientContextStub({ useLZ4Compression: clientSetting });
const driver = sinon.spy(context.driver);
const session = createSessionForTest({
handle: sessionHandleStub,
context,
serverProtocolVersion: TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6,
});
await session.executeStatement('SELECT 1', { useCloudFetch: true, useLZ4Compression: statementSetting });
expect(driver.executeStatement.firstCall.args[0].canDecompressLZ4Result).to.equal(
statementSetting ?? clientSetting,
);
});
}
}
it('should set canDecompressLZ4Result when cloud fetch is enabled (canDownloadResult=true)', async () => {
const context = new ClientContextStub({ useLZ4Compression: true });
const driver = sinon.spy(context.driver);
const statement = 'SELECT * FROM table';
Expand All @@ -212,9 +224,8 @@ describe('DBSQLSession', () => {
expect(driver.executeStatement.callCount).to.eq(1);
const req = driver.executeStatement.firstCall.args[0];

// canDownloadResult should be true and canDecompressLZ4Result should NOT be set
expect(req.canDownloadResult).to.be.true;
expect(req.canDecompressLZ4Result).to.not.be.true;
expect(req.canDecompressLZ4Result).to.be.true;
});

it('should set canDecompressLZ4Result when cloud fetch is disabled (canDownloadResult=false)', async () => {
Expand Down
23 changes: 23 additions & 0 deletions tests/unit/result/ArrowResultConverter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import fs from 'fs';
import path from 'path';
import { Table, tableFromArrays, tableToIPC, RecordBatch, TypeMap } from 'apache-arrow';
import ArrowResultConverter, { bigNumDecimalToString } from '../../../lib/result/ArrowResultConverter';
import HiveDriverError from '../../../lib/errors/HiveDriverError';
import { ArrowBatch } from '../../../lib/result/utils';
import ResultsProviderStub from '../.stubs/ResultsProviderStub';
import { TTableSchema, TTypeId } from '../../../thrift/TCLIService_types';
Expand Down Expand Up @@ -87,6 +88,28 @@ function createSampleArrowBatch<T extends TypeMap>(...recordBatches: RecordBatch
}

describe('ArrowResultConverter', () => {
for (const disableRowMaterialization of [false, true]) {
for (const batches of [sampleArrowBatch, []]) {
it(`rejects missing rows, materialization disabled=${disableRowMaterialization}, buffers=${batches.length}`, async () => {
const source = new ResultsProviderStub([{ batches, rowCount: 2 }], emptyItem);
const converter = new ArrowResultConverter(
new ClientContextStub(),
source,
{ schema: sampleThriftSchema },
{ disableRowMaterialization },
);
let caught: unknown;
try {
await converter.fetchNext({ limit: 10000 });
} catch (error) {
caught = error;
}
expect(caught).to.be.instanceOf(HiveDriverError);
expect((caught as Error).message).to.equal(`Arrow row count mismatch: ${batches.length ? 1 : 2} rows missing`);
});
}
}

it('should convert data', async () => {
const rowSetProvider = new ResultsProviderStub(
[
Expand Down
Loading
Loading