Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Unreleased

- Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends.
- Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged.
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
- Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608)
Expand Down
33 changes: 26 additions & 7 deletions lib/result/ArrowResultConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,27 @@ export function bigNumDecimalToString(unscaled: bigint, scale: number): string {
return `${negative ? '-' : ''}${digits.slice(0, cut)}.${digits.slice(cut)}`;
}

/**
* Returns the unscaled value of an Arrow Decimal128 as a number.
*
* Decimal128 is four little-endian 32-bit words. If words 2–3 are pure sign
* extension of word 1, the unscaled value fits in an int64, and we return it
* directly when it's a safe integer; otherwise fall back to `bigNumToBigInt`.
*/
function decimalToNumber(value: Uint32Array & Parameters<typeof bigNumToBigInt>[0]): number {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Could this get a short doc comment? For example: "Decimal128 is four little-endian 32-bit words. If words 2–3 are pure sign extension of word 1, the unscaled value fits in an int64, and we return it directly when it's a safe integer; otherwise fall back to bigNumToBigInt."

if (value.length === 4) {
const high = value[1] | 0;
const signExtension = high < 0 ? 0xffffffff : 0;
if (value[2] === signExtension && value[3] === signExtension) {
const unscaled = high * 0x100000000 + value[0];
if (Number.isSafeInteger(unscaled)) {
return unscaled;
}
}
}
return Number(bigNumToBigInt(value));
}

export default class ArrowResultConverter implements IResultsProvider<Array<any>> {
private readonly context: IClientContext;

Expand Down Expand Up @@ -407,7 +428,7 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
if (DataType.isDecimal(valueType)) {
return this.preserveBigNumericPrecision
? (value) => bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
: (value) => Number(bigNumToBigInt(value)) / 10 ** valueType.scale;
: (value) => decimalToNumber(value) / 10 ** valueType.scale;
}
if (DataType.isBinary(valueType) || DataType.isFixedSizeBinary(valueType)) {
return (value) => Buffer.from(value);
Expand Down Expand Up @@ -504,18 +525,16 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
// Convert big number values to BigInt
// Decimals are also represented as big numbers in Arrow, so additionally process them (convert to float)
if (value instanceof Object && value[isArrowBigNumSymbol]) {
const result = bigNumToBigInt(value);
if (DataType.isDecimal(valueType)) {
// Preserve full precision as an exact string when requested (kernel);
// otherwise keep the historical lossy `number` form.
if (this.preserveBigNumericPrecision) {
return bigNumDecimalToString(result, valueType.scale);
}
return Number(result) / 10 ** valueType.scale;
return this.preserveBigNumericPrecision
? bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
: decimalToNumber(value) / 10 ** valueType.scale;
}
// A rewritten Duration Int64 surfaces as a raw `bigint`, not a BigNum
// wrapper, so it is handled in the bigint branch below — not here.
return result;
return bigNumToBigInt(value);
}

// Convert binary data to Buffer.
Expand Down
142 changes: 142 additions & 0 deletions tests/unit/result/ArrowRowMaterialization.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,148 @@ function decimal(value: string): Uint32Array {

describe('Arrow row materialization', () => {
for (const rowMode of ['object', 'array'] as const) {
it(`skips BigInt conversion only for safe-integer decimals (${rowMode})`, () => {
const safe = ['0', '1', '-1', '4294967297', '-4294967297', '9007199254740991', '-9007199254740991'];
// The BigInt fallback reads `byteLength`; these must take it.
const unsafe = ['9007199254740993', '-9007199254740993', '18446744073709551616'];
const unscaled = [...safe, ...unsafe];
const columns: Column[] = [
{
field: new arrow.Field('amount', new arrow.Decimal(2, 38, 128)),
thriftType: TTypeId.DECIMAL_TYPE,
values: unscaled.map(decimal),
expected: unscaled.map((value) => Number(value) / 100),
},
];
const batch = arrow.tableFromIPC(ipc(columns)).batches[0];
const vector = batch.getChildAt(0)!;
const values = unscaled.map((value, index) => vector.get(index));
const byteLengthReads = values.map((value) => {
const read = sinon.stub().returns(value.byteLength);
Object.defineProperty(value, 'byteLength', { get: read });
return read;
});
const sandbox = sinon.createSandbox();
try {
const get = sandbox.stub(vector, 'get').callsFake((index) => values[index]);
sandbox.stub(batch, 'getChildAt').returns(vector);
const converter = converterFor(columns, rowMode);
deepStrictEqual(
(converter as any).getRows(batch, values.length),
columns[0].expected.map((amount) => (rowMode === 'array' ? [amount] : { amount })),
);
// The stubbed values must be the ones converted, or the byteLength checks prove nothing.
sinon.assert.callCount(get, values.length);
byteLengthReads.slice(0, safe.length).forEach((read) => sinon.assert.notCalled(read));
byteLengthReads.slice(safe.length).forEach((read) => sinon.assert.called(read));
} finally {
sandbox.restore();
}
});

it(`preserves Decimal128 rounding, signs and scales (${rowMode})`, async () => {
const unscaled = [
'0',
'1',
'-1',
'4294967295',
'-4294967295',
'4294967296',
'-4294967296',
'9007199254740991',
'-9007199254740991',
'9007199254740992',
'-9007199254740992',
'9007199254740993',
'-9007199254740993',
'18446744073709551615',
'-18446744073709551615',
'18446744073709551616',
'-18446744073709551616',
'79228162514264337593543950336',
'-79228162514264337593543950336',
'99999999999999999999999999999999999999',
'-99999999999999999999999999999999999999',
null,
];
const columns: Column[] = [-2, 0, 2, 18, 38].map((scale) => ({
field: new arrow.Field(`decimal${scale}`, new arrow.Decimal(scale, 38, 128)),
thriftType: TTypeId.DECIMAL_TYPE,
values: unscaled.map((value) => (value === null ? null : decimal(value))),
expected: unscaled.map((value) => (value === null ? null : Number(value) / 10 ** scale)),
}));
const converter = converterFor(columns, rowMode);
const expected = unscaled.map((value, rowIndex) =>
rowMode === 'array'
? columns.map((column) => column.expected[rowIndex])
: Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])),
);
deepStrictEqual(await converter.fetchNext({ limit: 100 }), expected);
const batch = arrow.tableFromIPC(ipc(columns)).batches[0];
deepStrictEqual((converter as any).getRows(batch.slice(3, 15), 8), expected.slice(3, 11));
const legacyRows = (converter as any).getObjectRows(batch.schema, batch.toArray());
deepStrictEqual(
rowMode === 'array'
? legacyRows.map((row: any) => columns.map((column) => row[column.field.name]))
: legacyRows,
expected,
);
});

it(`preserves nested numeric decimals and nulls (${rowMode})`, async () => {
const amount = new arrow.Decimal(2, 38, 128);
const array = new arrow.List(new arrow.Field('element', amount));
const columns: Column[] = [
{
field: new arrow.Field('array', array),
thriftType: TTypeId.ARRAY_TYPE,
values: [[decimal('-4294967297'), null, decimal('9007199254740993')], null, []],
expected: [[-42949672.97, null, Number('9007199254740993') / 100], null, []],
},
{
field: new arrow.Field(
'struct',
new arrow.Struct([new arrow.Field('amount', amount), new arrow.Field('items', array)]),
),
thriftType: TTypeId.STRUCT_TYPE,
values: [{ amount: decimal('-1'), items: [null, decimal('4294967296')] }, null, { amount: null, items: [] }],
expected: [{ amount: -0.01, items: [null, 42949672.96] }, null, { amount: null, items: [] }],
},
{
field: new arrow.Field(
'map',
new arrow.Map_(
new arrow.Field(
'entries',
new arrow.Struct<{ key: arrow.Utf8; value: arrow.Decimal }>([
new arrow.Field('key', new arrow.Utf8()),
new arrow.Field('value', amount),
]),
),
),
),
thriftType: TTypeId.MAP_TYPE,
values: [
new Map([
['negative', decimal('-9007199254740991')],
['null', null],
]),
null,
new Map(),
],
expected: [{ negative: Number('-9007199254740991') / 100, null: null }, null, {}],
},
];
deepStrictEqual(
await converterFor(columns, rowMode).fetchNext({ limit: 100 }),
[0, 1, 2].map((rowIndex) =>
rowMode === 'array'
? columns.map((column) => column.expected[rowIndex])
: Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])),
),
);
});

for (const preservePrecision of [false, true]) {
it(`preserves scalar values, types and nulls (${rowMode}, precision=${preservePrecision})`, async () => {
const timestamp = new Date('2026-01-02T03:04:05.000Z');
Expand Down
Loading