Skip to content

Commit fbe5057

Browse files
vuanhphungIsaac
andcommitted
Avoid decimal strings for safe Arrow decimal values
Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent fb040d8 commit fbe5057

3 files changed

Lines changed: 123 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
## Unreleased
44

5+
- Avoid intermediate decimal strings when materializing Arrow DECIMAL values with safe unscaled integers, on both backends.
56
- Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged.
67
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
78
- Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608)

‎lib/result/ArrowResultConverter.ts‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,20 @@ export function bigNumDecimalToString(unscaled: bigint, scale: number): string {
192192
return `${negative ? '-' : ''}${digits.slice(0, cut)}.${digits.slice(cut)}`;
193193
}
194194

195+
function decimalToNumber(value: Uint32Array & Parameters<typeof bigNumToBigInt>[0]): number {
196+
if (value.length === 4) {
197+
const high = value[1] | 0;
198+
const signExtension = high < 0 ? 0xffffffff : 0;
199+
if (value[2] === signExtension && value[3] === signExtension) {
200+
const unscaled = high * 0x100000000 + value[0];
201+
if (Number.isSafeInteger(unscaled)) {
202+
return unscaled;
203+
}
204+
}
205+
}
206+
return Number(bigNumToBigInt(value));
207+
}
208+
195209
export default class ArrowResultConverter implements IResultsProvider<Array<any>> {
196210
private readonly context: IClientContext;
197211

@@ -407,7 +421,7 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
407421
if (DataType.isDecimal(valueType)) {
408422
return this.preserveBigNumericPrecision
409423
? (value) => bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
410-
: (value) => Number(bigNumToBigInt(value)) / 10 ** valueType.scale;
424+
: (value) => decimalToNumber(value) / 10 ** valueType.scale;
411425
}
412426
if (DataType.isBinary(valueType) || DataType.isFixedSizeBinary(valueType)) {
413427
return (value) => Buffer.from(value);
@@ -504,18 +518,16 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
504518
// Convert big number values to BigInt
505519
// Decimals are also represented as big numbers in Arrow, so additionally process them (convert to float)
506520
if (value instanceof Object && value[isArrowBigNumSymbol]) {
507-
const result = bigNumToBigInt(value);
508521
if (DataType.isDecimal(valueType)) {
509522
// Preserve full precision as an exact string when requested (kernel);
510523
// otherwise keep the historical lossy `number` form.
511-
if (this.preserveBigNumericPrecision) {
512-
return bigNumDecimalToString(result, valueType.scale);
513-
}
514-
return Number(result) / 10 ** valueType.scale;
524+
return this.preserveBigNumericPrecision
525+
? bigNumDecimalToString(bigNumToBigInt(value), valueType.scale)
526+
: decimalToNumber(value) / 10 ** valueType.scale;
515527
}
516528
// A rewritten Duration Int64 surfaces as a raw `bigint`, not a BigNum
517529
// wrapper, so it is handled in the bigint branch below — not here.
518-
return result;
530+
return bigNumToBigInt(value);
519531
}
520532

521533
// Convert binary data to Buffer.

‎tests/unit/result/ArrowRowMaterialization.test.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,109 @@ function decimal(value: string): Uint32Array {
7373

7474
describe('Arrow row materialization', () => {
7575
for (const rowMode of ['object', 'array'] as const) {
76+
it(`preserves Decimal128 rounding, signs and scales (${rowMode})`, async () => {
77+
const unscaled = [
78+
'0',
79+
'1',
80+
'-1',
81+
'4294967295',
82+
'-4294967295',
83+
'4294967296',
84+
'-4294967296',
85+
'9007199254740991',
86+
'-9007199254740991',
87+
'9007199254740992',
88+
'-9007199254740992',
89+
'9007199254740993',
90+
'-9007199254740993',
91+
'18446744073709551615',
92+
'-18446744073709551615',
93+
'18446744073709551616',
94+
'-18446744073709551616',
95+
'79228162514264337593543950336',
96+
'-79228162514264337593543950336',
97+
'99999999999999999999999999999999999999',
98+
'-99999999999999999999999999999999999999',
99+
null,
100+
];
101+
const columns: Column[] = [-2, 0, 2, 18, 38].map((scale) => ({
102+
field: new arrow.Field(`decimal${scale}`, new arrow.Decimal(scale, 38, 128)),
103+
thriftType: TTypeId.DECIMAL_TYPE,
104+
values: unscaled.map((value) => (value === null ? null : decimal(value))),
105+
expected: unscaled.map((value) => (value === null ? null : Number(value) / 10 ** scale)),
106+
}));
107+
const converter = converterFor(columns, rowMode);
108+
const expected = unscaled.map((value, rowIndex) =>
109+
rowMode === 'array'
110+
? columns.map((column) => column.expected[rowIndex])
111+
: Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])),
112+
);
113+
deepStrictEqual(await converter.fetchNext({ limit: 100 }), expected);
114+
const batch = arrow.tableFromIPC(ipc(columns)).batches[0];
115+
deepStrictEqual((converter as any).getRows(batch.slice(3, 15), 8), expected.slice(3, 11));
116+
const legacyRows = (converter as any).getObjectRows(batch.schema, batch.toArray());
117+
deepStrictEqual(
118+
rowMode === 'array'
119+
? legacyRows.map((row: any) => columns.map((column) => row[column.field.name]))
120+
: legacyRows,
121+
expected,
122+
);
123+
});
124+
125+
it(`preserves nested numeric decimals and nulls (${rowMode})`, async () => {
126+
const amount = new arrow.Decimal(2, 38, 128);
127+
const array = new arrow.List(new arrow.Field('element', amount));
128+
const columns: Column[] = [
129+
{
130+
field: new arrow.Field('array', array),
131+
thriftType: TTypeId.ARRAY_TYPE,
132+
values: [[decimal('-4294967297'), null, decimal('9007199254740993')], null, []],
133+
expected: [[-42949672.97, null, Number('9007199254740993') / 100], null, []],
134+
},
135+
{
136+
field: new arrow.Field(
137+
'struct',
138+
new arrow.Struct([new arrow.Field('amount', amount), new arrow.Field('items', array)]),
139+
),
140+
thriftType: TTypeId.STRUCT_TYPE,
141+
values: [{ amount: decimal('-1'), items: [null, decimal('4294967296')] }, null, { amount: null, items: [] }],
142+
expected: [{ amount: -0.01, items: [null, 42949672.96] }, null, { amount: null, items: [] }],
143+
},
144+
{
145+
field: new arrow.Field(
146+
'map',
147+
new arrow.Map_(
148+
new arrow.Field(
149+
'entries',
150+
new arrow.Struct<{ key: arrow.Utf8; value: arrow.Decimal }>([
151+
new arrow.Field('key', new arrow.Utf8()),
152+
new arrow.Field('value', amount),
153+
]),
154+
),
155+
),
156+
),
157+
thriftType: TTypeId.MAP_TYPE,
158+
values: [
159+
new Map([
160+
['negative', decimal('-9007199254740991')],
161+
['null', null],
162+
]),
163+
null,
164+
new Map(),
165+
],
166+
expected: [{ negative: Number('-9007199254740991') / 100, null: null }, null, {}],
167+
},
168+
];
169+
deepStrictEqual(
170+
await converterFor(columns, rowMode).fetchNext({ limit: 100 }),
171+
[0, 1, 2].map((rowIndex) =>
172+
rowMode === 'array'
173+
? columns.map((column) => column.expected[rowIndex])
174+
: Object.fromEntries(columns.map((column) => [column.field.name, column.expected[rowIndex]])),
175+
),
176+
);
177+
});
178+
76179
for (const preservePrecision of [false, true]) {
77180
it(`preserves scalar values, types and nulls (${rowMode}, precision=${preservePrecision})`, async () => {
78181
const timestamp = new Date('2026-01-02T03:04:05.000Z');

0 commit comments

Comments
 (0)