From 061b595d86254aa8968f8c67212d0f0838c83d2c Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 15:49:24 +0000 Subject: [PATCH 1/5] Avoid decimal strings for safe Arrow decimal values Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + lib/result/ArrowResultConverter.ts | 26 +++-- .../result/ArrowRowMaterialization.test.ts | 103 ++++++++++++++++++ 3 files changed, 123 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c06d6f7..48139e6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends. - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index b5d558a6..22f81a56 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -192,6 +192,20 @@ export function bigNumDecimalToString(unscaled: bigint, scale: number): string { return `${negative ? '-' : ''}${digits.slice(0, cut)}.${digits.slice(cut)}`; } +function decimalToNumber(value: Uint32Array & Parameters[0]): number { + if (value.length === 4) { + const high = value[1] | 0; + const signExtension = high < 0 ? 0xffffffff : 0; + if (value[2] === signExtension && value[3] === signExtension) { + const unscaled = high * 0x100000000 + value[0]; + if (Number.isSafeInteger(unscaled)) { + return unscaled; + } + } + } + return Number(bigNumToBigInt(value)); +} + export default class ArrowResultConverter implements IResultsProvider> { private readonly context: IClientContext; @@ -407,7 +421,7 @@ export default class ArrowResultConverter implements IResultsProvider if (DataType.isDecimal(valueType)) { return this.preserveBigNumericPrecision ? (value) => bigNumDecimalToString(bigNumToBigInt(value), valueType.scale) - : (value) => Number(bigNumToBigInt(value)) / 10 ** valueType.scale; + : (value) => decimalToNumber(value) / 10 ** valueType.scale; } if (DataType.isBinary(valueType) || DataType.isFixedSizeBinary(valueType)) { return (value) => Buffer.from(value); @@ -504,18 +518,16 @@ export default class ArrowResultConverter implements IResultsProvider // Convert big number values to BigInt // Decimals are also represented as big numbers in Arrow, so additionally process them (convert to float) if (value instanceof Object && value[isArrowBigNumSymbol]) { - const result = bigNumToBigInt(value); if (DataType.isDecimal(valueType)) { // Preserve full precision as an exact string when requested (kernel); // otherwise keep the historical lossy `number` form. - if (this.preserveBigNumericPrecision) { - return bigNumDecimalToString(result, valueType.scale); - } - return Number(result) / 10 ** valueType.scale; + return this.preserveBigNumericPrecision + ? bigNumDecimalToString(bigNumToBigInt(value), valueType.scale) + : decimalToNumber(value) / 10 ** valueType.scale; } // A rewritten Duration Int64 surfaces as a raw `bigint`, not a BigNum // wrapper, so it is handled in the bigint branch below — not here. - return result; + return bigNumToBigInt(value); } // Convert binary data to Buffer. diff --git a/tests/unit/result/ArrowRowMaterialization.test.ts b/tests/unit/result/ArrowRowMaterialization.test.ts index 5b0c1930..7511e2d3 100644 --- a/tests/unit/result/ArrowRowMaterialization.test.ts +++ b/tests/unit/result/ArrowRowMaterialization.test.ts @@ -73,6 +73,109 @@ function decimal(value: string): Uint32Array { describe('Arrow row materialization', () => { for (const rowMode of ['object', 'array'] as const) { + it(`preserves Decimal128 rounding, signs and scales (${rowMode})`, async () => { + const unscaled = [ + '0', + '1', + '-1', + '4294967295', + '-4294967295', + '4294967296', + '-4294967296', + '9007199254740991', + '-9007199254740991', + '9007199254740992', + '-9007199254740992', + '9007199254740993', + '-9007199254740993', + '18446744073709551615', + '-18446744073709551615', + '18446744073709551616', + '-18446744073709551616', + '79228162514264337593543950336', + '-79228162514264337593543950336', + '99999999999999999999999999999999999999', + '-99999999999999999999999999999999999999', + null, + ]; + const columns: Column[] = [-2, 0, 2, 18, 38].map((scale) => ({ + field: new arrow.Field(`decimal${scale}`, new arrow.Decimal(scale, 38, 128)), + thriftType: TTypeId.DECIMAL_TYPE, + values: unscaled.map((value) => (value === null ? null : decimal(value))), + expected: unscaled.map((value) => (value === null ? null : Number(value) / 10 ** scale)), + })); + const converter = converterFor(columns, rowMode); + const expected = unscaled.map((value, rowIndex) => + rowMode === 'array' + ? columns.map((column) => column.expected[rowIndex]) + : Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])), + ); + deepStrictEqual(await converter.fetchNext({ limit: 100 }), expected); + const batch = arrow.tableFromIPC(ipc(columns)).batches[0]; + deepStrictEqual((converter as any).getRows(batch.slice(3, 15), 8), expected.slice(3, 11)); + const legacyRows = (converter as any).getObjectRows(batch.schema, batch.toArray()); + deepStrictEqual( + rowMode === 'array' + ? legacyRows.map((row: any) => columns.map((column) => row[column.field.name])) + : legacyRows, + expected, + ); + }); + + it(`preserves nested numeric decimals and nulls (${rowMode})`, async () => { + const amount = new arrow.Decimal(2, 38, 128); + const array = new arrow.List(new arrow.Field('element', amount)); + const columns: Column[] = [ + { + field: new arrow.Field('array', array), + thriftType: TTypeId.ARRAY_TYPE, + values: [[decimal('-4294967297'), null, decimal('9007199254740993')], null, []], + expected: [[-42949672.97, null, Number('9007199254740993') / 100], null, []], + }, + { + field: new arrow.Field( + 'struct', + new arrow.Struct([new arrow.Field('amount', amount), new arrow.Field('items', array)]), + ), + thriftType: TTypeId.STRUCT_TYPE, + values: [{ amount: decimal('-1'), items: [null, decimal('4294967296')] }, null, { amount: null, items: [] }], + expected: [{ amount: -0.01, items: [null, 42949672.96] }, null, { amount: null, items: [] }], + }, + { + field: new arrow.Field( + 'map', + new arrow.Map_( + new arrow.Field( + 'entries', + new arrow.Struct<{ key: arrow.Utf8; value: arrow.Decimal }>([ + new arrow.Field('key', new arrow.Utf8()), + new arrow.Field('value', amount), + ]), + ), + ), + ), + thriftType: TTypeId.MAP_TYPE, + values: [ + new Map([ + ['negative', decimal('-9007199254740991')], + ['null', null], + ]), + null, + new Map(), + ], + expected: [{ negative: Number('-9007199254740991') / 100, null: null }, null, {}], + }, + ]; + deepStrictEqual( + await converterFor(columns, rowMode).fetchNext({ limit: 100 }), + [0, 1, 2].map((rowIndex) => + rowMode === 'array' + ? columns.map((column) => column.expected[rowIndex]) + : Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])), + ), + ); + }); + for (const preservePrecision of [false, true]) { it(`preserves scalar values, types and nulls (${rowMode}, precision=${preservePrecision})`, async () => { const timestamp = new Date('2026-01-02T03:04:05.000Z'); From 3cd2d793577d3934b0889137dda8182776ac2787 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 17:20:38 +0000 Subject: [PATCH 2/5] Pin signed decimal fast-path selection in tests Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .../result/ArrowRowMaterialization.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/unit/result/ArrowRowMaterialization.test.ts b/tests/unit/result/ArrowRowMaterialization.test.ts index 7511e2d3..e1febe84 100644 --- a/tests/unit/result/ArrowRowMaterialization.test.ts +++ b/tests/unit/result/ArrowRowMaterialization.test.ts @@ -73,6 +73,39 @@ function decimal(value: string): Uint32Array { describe('Arrow row materialization', () => { for (const rowMode of ['object', 'array'] as const) { + it(`avoids decimal string conversion for safe signed values (${rowMode})`, () => { + const unscaled = ['0', '1', '-1', '4294967297', '-4294967297', '9007199254740991', '-9007199254740991']; + const columns: Column[] = [ + { + field: new arrow.Field('amount', new arrow.Decimal(2, 18, 128)), + thriftType: TTypeId.DECIMAL_TYPE, + values: unscaled.map(decimal), + expected: unscaled.map((value) => Number(value) / 100), + }, + ]; + const batch = arrow.tableFromIPC(ipc(columns)).batches[0]; + const vector = batch.getChildAt(0)!; + const values = unscaled.map((value, index) => vector.get(index)); + const byteLengthReads = values.map((value) => { + const read = sinon.stub().returns(value.byteLength); + Object.defineProperty(value, 'byteLength', { get: read }); + return read; + }); + const sandbox = sinon.createSandbox(); + try { + sandbox.stub(vector, 'get').callsFake((index) => values[index]); + sandbox.stub(batch, 'getChildAt').returns(vector); + const converter = converterFor(columns, rowMode); + deepStrictEqual( + (converter as any).getRows(batch, values.length), + columns[0].expected.map((amount) => (rowMode === 'array' ? [amount] : { amount })), + ); + byteLengthReads.forEach((read) => sinon.assert.notCalled(read)); + } finally { + sandbox.restore(); + } + }); + it(`preserves Decimal128 rounding, signs and scales (${rowMode})`, async () => { const unscaled = [ '0', From db6cd8369ffd193d6e29ec2f5dcf4020516c5c5e Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 20:29:03 +0000 Subject: [PATCH 3/5] Re-run CI after retargeting to main Co-authored-by: Isaac Signed-off-by: Vu Anh Phung From 65806c0d6c7563618935f0df7661e17824685f64 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 19 Nov 2026 23:34:19 +0000 Subject: [PATCH 4/5] Prove the decimal fast-path test reads through its stubs Assert the stubbed vector.get feeds getRows, and add unsafe values that must take the BigInt fallback, so the byteLength checks cannot pass vacuously if getRows stops reading through the stubs. Co-authored-by: Isaac Signed-off-by: Vu Anh Phung --- .../unit/result/ArrowRowMaterialization.test.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/unit/result/ArrowRowMaterialization.test.ts b/tests/unit/result/ArrowRowMaterialization.test.ts index e1febe84..7afb6e89 100644 --- a/tests/unit/result/ArrowRowMaterialization.test.ts +++ b/tests/unit/result/ArrowRowMaterialization.test.ts @@ -73,11 +73,14 @@ function decimal(value: string): Uint32Array { describe('Arrow row materialization', () => { for (const rowMode of ['object', 'array'] as const) { - it(`avoids decimal string conversion for safe signed values (${rowMode})`, () => { - const unscaled = ['0', '1', '-1', '4294967297', '-4294967297', '9007199254740991', '-9007199254740991']; + it(`skips BigInt conversion only for safe-integer decimals (${rowMode})`, () => { + const safe = ['0', '1', '-1', '4294967297', '-4294967297', '9007199254740991', '-9007199254740991']; + // The BigInt fallback reads `byteLength`; these must take it. + const unsafe = ['9007199254740993', '-9007199254740993', '18446744073709551616']; + const unscaled = [...safe, ...unsafe]; const columns: Column[] = [ { - field: new arrow.Field('amount', new arrow.Decimal(2, 18, 128)), + field: new arrow.Field('amount', new arrow.Decimal(2, 38, 128)), thriftType: TTypeId.DECIMAL_TYPE, values: unscaled.map(decimal), expected: unscaled.map((value) => Number(value) / 100), @@ -93,14 +96,17 @@ describe('Arrow row materialization', () => { }); const sandbox = sinon.createSandbox(); try { - sandbox.stub(vector, 'get').callsFake((index) => values[index]); + const get = sandbox.stub(vector, 'get').callsFake((index) => values[index]); sandbox.stub(batch, 'getChildAt').returns(vector); const converter = converterFor(columns, rowMode); deepStrictEqual( (converter as any).getRows(batch, values.length), columns[0].expected.map((amount) => (rowMode === 'array' ? [amount] : { amount })), ); - byteLengthReads.forEach((read) => sinon.assert.notCalled(read)); + // The stubbed values must be the ones converted, or the byteLength checks prove nothing. + sinon.assert.callCount(get, values.length); + byteLengthReads.slice(0, safe.length).forEach((read) => sinon.assert.notCalled(read)); + byteLengthReads.slice(safe.length).forEach((read) => sinon.assert.called(read)); } finally { sandbox.restore(); } From b9ea5cfcda50fe74a690dcf4e964b1267063930f Mon Sep 17 00:00:00 2001 From: "peco-engineer-bot[bot]" Date: Fri, 20 Nov 2026 04:00:22 +0000 Subject: [PATCH 5/5] engineer-bot: address @mention on PR #633 Signed-off-by: peco-engineer-bot[bot] --- lib/result/ArrowResultConverter.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index 22f81a56..441c495f 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -192,6 +192,13 @@ export function bigNumDecimalToString(unscaled: bigint, scale: number): string { return `${negative ? '-' : ''}${digits.slice(0, cut)}.${digits.slice(cut)}`; } +/** + * Returns the unscaled value of an Arrow Decimal128 as a number. + * + * Decimal128 is four little-endian 32-bit words. If words 2–3 are pure sign + * extension of word 1, the unscaled value fits in an int64, and we return it + * directly when it's a safe integer; otherwise fall back to `bigNumToBigInt`. + */ function decimalToNumber(value: Uint32Array & Parameters[0]): number { if (value.length === 4) { const high = value[1] | 0;