diff --git a/CHANGELOG.md b/CHANGELOG.md index f8aeb947..53d2ef02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +Arrow results now materialize directly into output chunks on both backends, avoiding intermediate batch-row copies while preserving chunk sizes and row values. + - Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged. - The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611) - Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608) diff --git a/lib/result/ArrowResultConverter.ts b/lib/result/ArrowResultConverter.ts index b5d558a6..662c2873 100644 --- a/lib/result/ArrowResultConverter.ts +++ b/lib/result/ArrowResultConverter.ts @@ -19,7 +19,7 @@ import { TTableSchema, TColumnDesc, TTypeId } from '../../thrift/TCLIService_typ import IClientContext from '../contracts/IClientContext'; import { RowMode } from '../contracts/IDBSQLSession'; import HiveDriverError from '../errors/HiveDriverError'; -import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider'; +import IResultsProvider, { ResultsProviderFetchNextOptions, RowTarget } from './IResultsProvider'; import { ArrowBatch, getSchemaColumns, convertThriftValue, warnOnDuplicateColumnNames } from './utils'; const { isArrowBigNumSymbol, bigNumToBigInt } = arrowUtils; @@ -263,6 +263,10 @@ export default class ArrowResultConverter implements IResultsProvider } public async fetchNext(options: ResultsProviderFetchNextOptions) { + return this.fetchNextInto(options, { rows: [], count: 0 }, 0); + } + + public async fetchNextInto(options: ResultsProviderFetchNextOptions, target: RowTarget, limit: number) { if (this.schema.length === 0) { return []; } @@ -284,17 +288,31 @@ export default class ArrowResultConverter implements IResultsProvider // Get table rows, but not more than remaining count. When materialization // is disabled, skip `toArray()`/`getRows()` and return dense `null` placeholders. const batchRows = Math.min(batch.numRows, this.remainingRows); + const appendedRows = Math.min(batchRows, Math.max(0, limit - target.count)); + const end = target.count + appendedRows; + if (end > target.rows.length && target.rows.length < 2 ** 25) { + const capacity = target.count === 0 ? end : Math.min(limit, Math.max(end, target.count * 2, batchRows * 32)); + target.rows.length = Math.min(capacity, 2 ** 25); + } let result: Array; if (this.disableRowMaterialization) { - result = new Array(batchRows).fill(null); + target.rows.fill(null, target.count, end); + for (let index = target.rows.length; index < end; index += 1) { + target.rows[index] = null; + } + result = new Array(batchRows - appendedRows).fill(null); } else { - result = this.getRows(batch, batchRows); + if (appendedRows > 0) { + this.getRows(batch, appendedRows, target.rows, 0, target.count); + } + result = appendedRows < batchRows ? this.getRows(batch, batchRows - appendedRows, undefined, appendedRows) : []; } + target.count = end; // Reduce remaining rows count by a count of rows we just processed. // If the remaining count reached zero - we're done with current arrow // batch, so discard the batch reader - this.remainingRows -= result.length; + this.remainingRows -= batchRows; if (this.remainingRows === 0) { this.recordBatchReader = undefined; } @@ -358,7 +376,13 @@ export default class ArrowResultConverter implements IResultsProvider }); } - private getRows(batch: RecordBatch, rowCount: number): Array { + private getRows( + batch: RecordBatch, + rowCount: number, + target?: Array, + start: number = 0, + offset: number = 0, + ): Array { const arrayRows = this.rowMode === 'array'; const fields: Array = arrayRows ? batch.data.type.children : batch.schema.fields; const names = this.schema.map((column) => column.columnName); @@ -370,7 +394,14 @@ export default class ArrowResultConverter implements IResultsProvider fields.some((field, index) => field.name !== names[index])) ) { const table = new Table(batch); - return this.getObjectRows(table.schema, table.toArray().slice(0, rowCount)); + const rows = this.getObjectRows(table.schema, table.toArray().slice(start, start + rowCount)); + if (!target) { + return rows; + } + for (let index = 0; index < rows.length; index += 1) { + target[offset + index] = rows[index]; + } + return target; } const vectors = this.schema.map((column, index) => batch.getChildAt(index)); const keys = arrayRows ? names.map((name, index) => index) : names; @@ -384,14 +415,14 @@ export default class ArrowResultConverter implements IResultsProvider : descriptor; return (value: any) => (value === null ? null : convertThriftValue(thriftType, convert(value))); }); - const rows: Array = new Array(rowCount); + const rows: Array = target ?? new Array(Math.min(rowCount, 2 ** 25)); for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) { const row: any = arrayRows ? [] : {}; for (let columnIndex = 0; columnIndex < names.length; columnIndex += 1) { - const value = converters[columnIndex](vectors[columnIndex]?.get(rowIndex) ?? null); + const value = converters[columnIndex](vectors[columnIndex]?.get(start + rowIndex) ?? null); row[keys[columnIndex]] = value; } - rows[rowIndex] = row; + rows[offset + rowIndex] = row; } return rows; } diff --git a/lib/result/IResultsProvider.ts b/lib/result/IResultsProvider.ts index 0e521f71..41bd6472 100644 --- a/lib/result/IResultsProvider.ts +++ b/lib/result/IResultsProvider.ts @@ -7,3 +7,12 @@ export default interface IResultsProvider { hasMore(): Promise; } + +export interface RowTarget { + rows: Array; + count: number; +} + +export interface RowResultsProvider extends IResultsProvider> { + fetchNextInto?(options: ResultsProviderFetchNextOptions, target: RowTarget, limit: number): Promise>; +} diff --git a/lib/result/ResultSlicer.ts b/lib/result/ResultSlicer.ts index de5c1107..64607ff4 100644 --- a/lib/result/ResultSlicer.ts +++ b/lib/result/ResultSlicer.ts @@ -1,5 +1,5 @@ import IClientContext from '../contracts/IClientContext'; -import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider'; +import IResultsProvider, { ResultsProviderFetchNextOptions, RowResultsProvider } from './IResultsProvider'; export interface ResultSlicerFetchNextOptions extends ResultsProviderFetchNextOptions { // Setting this to `true` will disable slicer, and it will return unprocessed chunks @@ -33,7 +33,7 @@ export function concatChunks(chunks: Array>): Array { export default class ResultSlicer implements IResultsProvider> { private readonly context: IClientContext; - private readonly source: IResultsProvider>; + private readonly source: RowResultsProvider; private remainingResults: Array = []; @@ -90,6 +90,21 @@ export default class ResultSlicer implements IResultsProvider> { const limit = Number(options.limit); const sourceOptions = { ...options, limit }; + if (this.source.fetchNextInto && limit > 0 && (Number.isInteger(limit) || limit === Infinity)) { + const rows = this.remainingCount > 0 ? this.takeRemaining(limit) : []; + const target = { rows, count: rows.length }; + while (target.count < limit) { + // eslint-disable-next-line no-await-in-loop + if (!(await this.source.hasMore())) { + break; + } + // eslint-disable-next-line no-await-in-loop + this.remainingResults = await this.source.fetchNextInto(sourceOptions, target, limit); + this.remainingOffset = 0; + } + target.rows.length = target.count; + return target.rows; + } const result: Array> = []; let resultsCount = 0; diff --git a/tests/e2e/utils/rowMaterialization.ts b/tests/e2e/utils/rowMaterialization.ts index 9c862238..1177b30b 100644 --- a/tests/e2e/utils/rowMaterialization.ts +++ b/tests/e2e/utils/rowMaterialization.ts @@ -9,6 +9,58 @@ const largeInteger = BigInt('9007199254740993'); export default function rowMaterializationTests(useKernel: boolean) { describe(`Arrow row materialization (${useKernel ? 'kernel' : 'Thrift'})`, () => { for (const rowMode of ['object', 'array'] as const) { + for (const maxRows of [1000, 20000]) { + for (const api of ['fetchAll', 'fetchChunk', 'iterateChunks', 'iterateRows'] as const) { + it(`preserves chunked rows (${rowMode}, ${api}, maxRows=${maxRows})`, async () => { + const client = new DBSQLClient(); + try { + await client.connect({ + host: config.host, + path: config.path, + token: config.token, + useKernel, + } as ConnectionOptions & InternalConnectionOptions); + const session = await client.openSession(); + try { + const operation = await session.executeStatement( + 'SELECT CAST(id AS INT) AS id, IF(id % 17 = 0, NULL, id % 2 = 0) AS value FROM range(10003) ORDER BY id', + { rowMode, maxRows: 127 }, + ); + try { + const rows: object[] = []; + if (api === 'fetchAll') { + rows.push(...(await operation.fetchAll({ maxRows }))); + } else if (api === 'fetchChunk') { + const chunk = await operation.fetchChunk({ maxRows }); + expect(chunk.length).to.equal(Math.min(maxRows, 10003)); + rows.push(...chunk, ...(await operation.fetchAll({ maxRows }))); + } else if (api === 'iterateChunks') { + for await (const chunk of operation.iterateChunks({ maxRows })) { + expect(chunk.length).to.equal(Math.min(maxRows, 10003 - rows.length)); + rows.push(...chunk); + } + } else { + for await (const row of operation.iterateRows({ maxRows })) rows.push(row); + } + expect(rows).to.deep.equal( + Array.from({ length: 10003 }, (_, id) => { + const value = id % 17 === 0 ? null : id % 2 === 0; + return rowMode === 'array' ? [id, value] : { id, value }; + }), + ); + expect(await operation.hasMoreRows()).to.equal(false); + } finally { + await operation.close(); + } + } finally { + await session.close(); + } + } finally { + await client.close(); + } + }); + } + } for (const useArrowNativeTypes of [false, true]) { for (const preserveBigNumericPrecision of [false, true]) { it(`preserves values (${rowMode}, native=${useArrowNativeTypes}, precision=${preserveBigNumericPrecision})`, async () => { diff --git a/tests/unit/result/ArrowResultAppend.test.ts b/tests/unit/result/ArrowResultAppend.test.ts new file mode 100644 index 00000000..d7d753d8 --- /dev/null +++ b/tests/unit/result/ArrowResultAppend.test.ts @@ -0,0 +1,198 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as arrow from 'apache-arrow'; +import ArrowResultConverter from '../../../lib/result/ArrowResultConverter'; +import { ArrowBatch } from '../../../lib/result/utils'; +import ResultSlicer from '../../../lib/result/ResultSlicer'; +import { RowTarget } from '../../../lib/result/IResultsProvider'; +import { TTypeId } from '../../../thrift/TCLIService_types'; +import ClientContextStub from '../.stubs/ClientContextStub'; +import ResultsProviderStub from '../.stubs/ResultsProviderStub'; + +function fixture(rowMode: 'object' | 'array', disableRowMaterialization = false) { + const values = [true, null, false, true, null, false]; + const bytes = Buffer.from( + arrow.tableToIPC(new arrow.Table({ value: arrow.vectorFromArray(values, new arrow.Bool()) })), + ); + const converter = new ArrowResultConverter( + new ClientContextStub(), + new ResultsProviderStub( + [ + { batches: [bytes], rowCount: 6 }, + { batches: [bytes], rowCount: 5 }, + { batches: [], rowCount: 0 }, + { batches: [bytes], rowCount: 2 }, + ], + { batches: [], rowCount: 0 }, + ), + { + schema: { + columns: [ + { + columnName: 'value', + position: 1, + typeDesc: { types: [{ primitiveEntry: { type: TTypeId.BOOLEAN_TYPE } }] }, + }, + ], + }, + }, + { rowMode, disableRowMaterialization }, + ); + const rows = [...values, ...values.slice(0, 5), ...values.slice(0, 2)].map((value) => + disableRowMaterialization ? null : rowMode === 'array' ? [value] : { value }, + ); + return { converter, rows, slicer: new ResultSlicer(new ClientContextStub(), converter) }; +} + +describe('Arrow result append', () => { + for (const rowMode of ['object', 'array'] as const) { + for (const placeholders of [false, true]) { + for (const limit of [1, 4, 6, 10, 100, Infinity]) { + it(`preserves boundaries, nulls and owned tails (${rowMode}, placeholders=${placeholders}, limit=${limit})`, async () => { + const { converter, rows, slicer } = fixture(rowMode, placeholders); + const fetch = sinon.spy(converter, 'fetchNext'); + const chunks = []; + let count = 0; + while (await slicer.hasMore()) { + const chunk = await slicer.fetchNext({ limit }); + expect(chunk).to.deep.equal(rows.slice(count, count + limit)); + count += chunk.length; + chunks.push(...chunk); + const pending = slicer['remainingResults'].slice(slicer['remainingOffset']); + expect(pending).to.deep.equal(rows.slice(count, count + pending.length)); + expect(slicer['remainingResults'].slice(0, slicer['remainingOffset'])).to.satisfy((cleared: unknown[]) => + cleared.every((value) => value === undefined), + ); + } + expect(chunks).to.deep.equal(rows); + sinon.assert.notCalled(fetch); + }); + } + } + + it(`writes fitted materialized rows directly to the output once (${rowMode})`, async () => { + const { converter, rows } = fixture(rowMode); + const writes: number[] = []; + const target: RowTarget = { + count: 0, + rows: new Proxy([], { + set(array, property, value) { + if (typeof property === 'string' && /^\d+$/.test(property)) writes.push(Number(property)); + return Reflect.set(array, property, value); + }, + }), + }; + const materialize = sinon.spy(converter as any, 'getRows'); + while (await converter.hasMore()) { + expect(await converter.fetchNextInto({ limit: 100 }, target, 100)).to.deep.equal([]); + } + target.rows.length = target.count; + expect(target.rows).to.deep.equal(rows); + expect(writes).to.deep.equal(rows.map((_, index) => index)); + materialize.getCalls().forEach((call) => { + expect(call.args[2]).to.equal(target.rows); + expect(call.returnValue).to.equal(target.rows); + }); + }); + + for (const placeholders of [false, true]) { + it(`grows sequentially beyond the initial allocation guard (${rowMode}, placeholders=${placeholders})`, async () => { + const { converter, rows } = fixture(rowMode, placeholders); + const start = 2 ** 25 - 2; + let length = start; + const values = new Map(); + const reserved: number[] = []; + const target: RowTarget = { + count: start, + rows: new Proxy([], { + get(array, property) { + return property === 'length' ? length : Reflect.get(array, property); + }, + set(_array, property, value) { + if (property === 'length') { + reserved.push(value); + length = value; + } else { + const index = Number(property); + values.set(index, value); + length = Math.max(length, index + 1); + } + return true; + }, + }), + }; + while (await converter.hasMore()) { + expect(await converter.fetchNextInto({ limit: Infinity }, target, Infinity)).to.deep.equal([]); + } + expect(reserved).to.deep.equal([2 ** 25]); + expect(target.count).to.equal(start + rows.length); + expect([...values.keys()]).to.deep.equal(rows.map((_, index) => start + index)); + expect([...values.values()]).to.deep.equal(rows); + }); + } + } + + it('does not read back fitted rows to copy them in the slicer', async () => { + const { converter, slicer, rows } = fixture('object'); + const append = converter.fetchNextInto.bind(converter); + let reading = false; + sinon.stub(converter, 'fetchNextInto').callsFake(async (options, target, limit) => { + target.rows = new Proxy(target.rows, { + get(array, property) { + if (!reading && typeof property === 'string' && /^\d+$/.test(property)) throw new Error('copied output row'); + return Reflect.get(array, property); + }, + }); + return append(options, target, limit); + }); + const chunk = await slicer.fetchNext({ limit: 100 }); + reading = true; + expect(chunk).to.deep.equal(rows); + }); + + it('returns the owned tail before an unbuffered batch and never mutates returned chunks', async () => { + const { slicer, rows } = fixture('object'); + const first = await slicer.fetchNext({ limit: 4 }); + const second = await slicer.fetchNext({ limit: 100, disableBuffering: true }); + const third = await slicer.fetchNext({ limit: 100, disableBuffering: true }); + const fourth = await slicer.fetchNext({ limit: 100 }); + expect([first.length, second.length, third.length, fourth.length]).to.deep.equal([4, 2, 5, 2]); + expect([...first, ...second, ...third, ...fourth]).to.deep.equal(rows); + expect(await slicer.hasMore()).to.equal(false); + }); + + it('retains fractional and invalid limit behavior when the source supports append', async () => { + const { slicer, rows } = fixture('object'); + const first = await slicer.fetchNext({ limit: 3.5 }); + expect(first).to.deep.equal(rows.slice(0, 3)); + expect(await slicer.fetchNext({ limit: 0 })).to.deep.equal([]); + expect(await slicer.fetchNext({ limit: -1 })).to.deep.equal([]); + expect(await slicer.fetchNext({ limit: NaN })).to.deep.equal(rows.slice(3, 6)); + expect(await slicer.fetchNext({ limit: 100 })).to.deep.equal(rows.slice(6)); + }); + + it('propagates prefetch errors rather than returning a partial chunk', async () => { + const { converter, slicer } = fixture('object'); + const error = new Error('prefetch failed'); + sinon.stub(converter['source'], 'fetchNext').rejects(error); + try { + await slicer.fetchNext({ limit: 100 }); + expect.fail('fetch should reject'); + } catch (actual) { + expect(actual).to.equal(error); + } + }); + + for (const limit of [2 ** 25, 2 ** 25 + 1, Infinity]) { + it(`does not reserve a huge requested limit for a small result (${limit})`, async () => { + const { converter, rows } = fixture('object'); + const target: RowTarget = { rows: [], count: 0 }; + while (await converter.hasMore()) { + expect(await converter.fetchNextInto({ limit }, target, limit)).to.deep.equal([]); + expect(target.rows.length).to.be.at.most(6 * 32); + } + target.rows.length = target.count; + expect(target.rows).to.deep.equal(rows); + }); + } +}); diff --git a/tests/unit/result/ArrowRowMaterialization.test.ts b/tests/unit/result/ArrowRowMaterialization.test.ts index 5b0c1930..49c3e146 100644 --- a/tests/unit/result/ArrowRowMaterialization.test.ts +++ b/tests/unit/result/ArrowRowMaterialization.test.ts @@ -71,6 +71,15 @@ function decimal(value: string): Uint32Array { return Uint32Array.from([0, 32, 64, 96], (shift) => Number((unsigned >> BigInt(shift)) & BigInt('4294967295'))); } +async function expectAppended(converter: ArrowResultConverter, expected: unknown[]) { + const prefix = { prefix: true }; + const target = { rows: [prefix] as unknown[], count: 1 }; + const tail = await converter.fetchNextInto({ limit: 100 }, target, 2); + target.rows.length = target.count; + expect(target.rows[0]).to.equal(prefix); + deepStrictEqual([...target.rows.slice(1), ...tail], expected); +} + describe('Arrow row materialization', () => { for (const rowMode of ['object', 'array'] as const) { for (const preservePrecision of [false, true]) { @@ -166,6 +175,7 @@ describe('Arrow row materialization', () => { ]; const converter = converterFor(columns, rowMode, preservePrecision); const result = await converter.fetchNext({ limit: 100 }); + await expectAppended(converterFor(columns, rowMode, preservePrecision), result); const expected = [0, 1, 2].map((rowIndex) => { const values = columns.map( (column) => (preservePrecision ? column.precise ?? column.expected : column.expected)[rowIndex], @@ -207,6 +217,7 @@ describe('Arrow row materialization', () => { })), ]; const rows = await converterFor(columns, rowMode, preservePrecision).fetchNext({ limit: 100 }); + await expectAppended(converterFor(columns, rowMode, preservePrecision), rows); deepStrictEqual( rows, [0, 1, 2].map((rowIndex) => @@ -545,6 +556,7 @@ describe('Arrow row materialization', () => { const rows = await converter.fetchNext({ limit: 100 }); sinon.assert.calledOnce(fallback); deepStrictEqual(rows, expected); + await expectAppended(converterFor(columns, 'object', false, undefined, schemaColumns), expected); rows.forEach((row, index) => { deepStrictEqual(Object.keys(row), Object.keys(expected[index])); expect(Object.getPrototypeOf(row)).to.equal(Object.prototype);