Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## Unreleased

Arrow results now materialize directly into output chunks on both backends, avoiding intermediate batch-row copies while preserving chunk sizes and row values.

- Reduce Arrow row materialization overhead on both backends by reading vectors directly and resolving scalar conversions once per batch. Object and array row values are unchanged.
- The Thrift token federation exchange (`enableTokenFederation`) now goes through the connection's agent, so `proxy`, `checkServerCertificate`, `customCaCert` and `clientCert`/`clientKey` apply to it. Previously it connected directly with Node's default agent, so behind a required proxy the exchange failed and the driver fell back to the original token. (#611)
- Kernel backend: a failed status poll while waiting for a statement (a `runAsync: true` statement, or a default execution that outlives the server's inline wait) and a failed result fetch in `hasMoreRows()` now throw the mapped driver error, such as `HiveDriverError: HTTP 500: Internal Server Error` once retries on HTTP 5xx run out, instead of a raw `Error` whose message is the internal `__databricks_error__:{...}` envelope. As in `fetchChunk()`, a failed fetch in `hasMoreRows()` also closes the statement. (#608)
Expand Down
49 changes: 40 additions & 9 deletions lib/result/ArrowResultConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { TTableSchema, TColumnDesc, TTypeId } from '../../thrift/TCLIService_typ
import IClientContext from '../contracts/IClientContext';
import { RowMode } from '../contracts/IDBSQLSession';
import HiveDriverError from '../errors/HiveDriverError';
import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
import IResultsProvider, { ResultsProviderFetchNextOptions, RowTarget } from './IResultsProvider';
import { ArrowBatch, getSchemaColumns, convertThriftValue, warnOnDuplicateColumnNames } from './utils';

const { isArrowBigNumSymbol, bigNumToBigInt } = arrowUtils;
Expand Down Expand Up @@ -263,6 +263,10 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
}

public async fetchNext(options: ResultsProviderFetchNextOptions) {
return this.fetchNextInto(options, { rows: [], count: 0 }, 0);
}

public async fetchNextInto(options: ResultsProviderFetchNextOptions, target: RowTarget<any>, limit: number) {
if (this.schema.length === 0) {
return [];
}
Expand All @@ -284,17 +288,31 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
// Get table rows, but not more than remaining count. When materialization
// is disabled, skip `toArray()`/`getRows()` and return dense `null` placeholders.
const batchRows = Math.min(batch.numRows, this.remainingRows);
const appendedRows = Math.min(batchRows, Math.max(0, limit - target.count));
const end = target.count + appendedRows;
if (end > target.rows.length && target.rows.length < 2 ** 25) {
const capacity = target.count === 0 ? end : Math.min(limit, Math.max(end, target.count * 2, batchRows * 32));
target.rows.length = Math.min(capacity, 2 ** 25);
}
let result: Array<any>;
if (this.disableRowMaterialization) {
result = new Array(batchRows).fill(null);
target.rows.fill(null, target.count, end);
for (let index = target.rows.length; index < end; index += 1) {
target.rows[index] = null;
}
result = new Array(batchRows - appendedRows).fill(null);
} else {
result = this.getRows(batch, batchRows);
if (appendedRows > 0) {
this.getRows(batch, appendedRows, target.rows, 0, target.count);
}
result = appendedRows < batchRows ? this.getRows(batch, batchRows - appendedRows, undefined, appendedRows) : [];
}
target.count = end;

// Reduce remaining rows count by a count of rows we just processed.
// If the remaining count reached zero - we're done with current arrow
// batch, so discard the batch reader
this.remainingRows -= result.length;
this.remainingRows -= batchRows;
if (this.remainingRows === 0) {
this.recordBatchReader = undefined;
}
Expand Down Expand Up @@ -358,7 +376,13 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
});
}

private getRows(batch: RecordBatch<TypeMap>, rowCount: number): Array<any> {
private getRows(
batch: RecordBatch<TypeMap>,
rowCount: number,
target?: Array<any>,
start: number = 0,
offset: number = 0,
): Array<any> {
const arrayRows = this.rowMode === 'array';
const fields: Array<ArrowSchemaField> = arrayRows ? batch.data.type.children : batch.schema.fields;
const names = this.schema.map((column) => column.columnName);
Expand All @@ -370,7 +394,14 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
fields.some((field, index) => field.name !== names[index]))
) {
const table = new Table(batch);
return this.getObjectRows(table.schema, table.toArray().slice(0, rowCount));
const rows = this.getObjectRows(table.schema, table.toArray().slice(start, start + rowCount));
if (!target) {
return rows;
}
for (let index = 0; index < rows.length; index += 1) {
target[offset + index] = rows[index];
}
return target;
}
const vectors = this.schema.map((column, index) => batch.getChildAt(index));
const keys = arrayRows ? names.map((name, index) => index) : names;
Expand All @@ -384,14 +415,14 @@ export default class ArrowResultConverter implements IResultsProvider<Array<any>
: descriptor;
return (value: any) => (value === null ? null : convertThriftValue(thriftType, convert(value)));
});
const rows: Array<any> = new Array(rowCount);
const rows: Array<any> = target ?? new Array(Math.min(rowCount, 2 ** 25));
for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
const row: any = arrayRows ? [] : {};
for (let columnIndex = 0; columnIndex < names.length; columnIndex += 1) {
const value = converters[columnIndex](vectors[columnIndex]?.get(rowIndex) ?? null);
const value = converters[columnIndex](vectors[columnIndex]?.get(start + rowIndex) ?? null);
row[keys[columnIndex]] = value;
}
rows[rowIndex] = row;
rows[offset + rowIndex] = row;
}
return rows;
}
Expand Down
9 changes: 9 additions & 0 deletions lib/result/IResultsProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,12 @@ export default interface IResultsProvider<T> {

hasMore(): Promise<boolean>;
}

export interface RowTarget<T> {
rows: Array<T>;
count: number;
}

export interface RowResultsProvider<T> extends IResultsProvider<Array<T>> {
fetchNextInto?(options: ResultsProviderFetchNextOptions, target: RowTarget<T>, limit: number): Promise<Array<T>>;
}
19 changes: 17 additions & 2 deletions lib/result/ResultSlicer.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import IClientContext from '../contracts/IClientContext';
import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
import IResultsProvider, { ResultsProviderFetchNextOptions, RowResultsProvider } from './IResultsProvider';

export interface ResultSlicerFetchNextOptions extends ResultsProviderFetchNextOptions {
// Setting this to `true` will disable slicer, and it will return unprocessed chunks
Expand Down Expand Up @@ -33,7 +33,7 @@ export function concatChunks<T>(chunks: Array<Array<T>>): Array<T> {
export default class ResultSlicer<T> implements IResultsProvider<Array<T>> {
private readonly context: IClientContext;

private readonly source: IResultsProvider<Array<T>>;
private readonly source: RowResultsProvider<T>;

private remainingResults: Array<T | undefined> = [];

Expand Down Expand Up @@ -90,6 +90,21 @@ export default class ResultSlicer<T> implements IResultsProvider<Array<T>> {

const limit = Number(options.limit);
const sourceOptions = { ...options, limit };
if (this.source.fetchNextInto && limit > 0 && (Number.isInteger(limit) || limit === Infinity)) {
const rows = this.remainingCount > 0 ? this.takeRemaining(limit) : [];
const target = { rows, count: rows.length };
while (target.count < limit) {
// eslint-disable-next-line no-await-in-loop
if (!(await this.source.hasMore())) {
break;
}
// eslint-disable-next-line no-await-in-loop
this.remainingResults = await this.source.fetchNextInto(sourceOptions, target, limit);
this.remainingOffset = 0;
}
target.rows.length = target.count;
return target.rows;
}
const result: Array<Array<T>> = [];
let resultsCount = 0;

Expand Down
52 changes: 52 additions & 0 deletions tests/e2e/utils/rowMaterialization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,58 @@ const largeInteger = BigInt('9007199254740993');
export default function rowMaterializationTests(useKernel: boolean) {
describe(`Arrow row materialization (${useKernel ? 'kernel' : 'Thrift'})`, () => {
for (const rowMode of ['object', 'array'] as const) {
for (const maxRows of [1000, 20000]) {
for (const api of ['fetchAll', 'fetchChunk', 'iterateChunks', 'iterateRows'] as const) {
it(`preserves chunked rows (${rowMode}, ${api}, maxRows=${maxRows})`, async () => {
const client = new DBSQLClient();
try {
await client.connect({
host: config.host,
path: config.path,
token: config.token,
useKernel,
} as ConnectionOptions & InternalConnectionOptions);
const session = await client.openSession();
try {
const operation = await session.executeStatement(
'SELECT CAST(id AS INT) AS id, IF(id % 17 = 0, NULL, id % 2 = 0) AS value FROM range(10003) ORDER BY id',
{ rowMode, maxRows: 127 },
);
try {
const rows: object[] = [];
if (api === 'fetchAll') {
rows.push(...(await operation.fetchAll({ maxRows })));
} else if (api === 'fetchChunk') {
const chunk = await operation.fetchChunk({ maxRows });
expect(chunk.length).to.equal(Math.min(maxRows, 10003));
rows.push(...chunk, ...(await operation.fetchAll({ maxRows })));
} else if (api === 'iterateChunks') {
for await (const chunk of operation.iterateChunks({ maxRows })) {
expect(chunk.length).to.equal(Math.min(maxRows, 10003 - rows.length));
rows.push(...chunk);
}
} else {
for await (const row of operation.iterateRows({ maxRows })) rows.push(row);
}
expect(rows).to.deep.equal(
Array.from({ length: 10003 }, (_, id) => {
const value = id % 17 === 0 ? null : id % 2 === 0;
return rowMode === 'array' ? [id, value] : { id, value };
}),
);
expect(await operation.hasMoreRows()).to.equal(false);
} finally {
await operation.close();
}
} finally {
await session.close();
}
} finally {
await client.close();
}
});
}
}
for (const useArrowNativeTypes of [false, true]) {
for (const preserveBigNumericPrecision of [false, true]) {
it(`preserves values (${rowMode}, native=${useArrowNativeTypes}, precision=${preserveBigNumericPrecision})`, async () => {
Expand Down
Loading
Loading