Skip to content

Commit 5dab295

Browse files
authored
Merge branch 'main' into perf/row-convert-2
2 parents 703d675 + 386d242 commit 5dab295

4 files changed

Lines changed: 192 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
- Fix `DECIMAL` parameters losing their fractional digits on the Thrift backend: `"123.45"` was bound as `decimal(3,0)` and came back as `123`. Both backends now send `DECIMAL(p,s)` derived from the value (also for the `DEC` / `NUMERIC` synonyms), stringify `Int64` values beyond 2^53 exactly instead of as `Infinity`, accept exponent forms such as `1e-7` (rewritten as a plain numeral), and throw a `ParameterError` for non-numeric values or values that need more than 38 digits. (#474)
3030
- `customHeaders` are now sent on every Thrift request (session, statement and metadata calls), as the kernel backend already did, instead of only on telemetry and feature-flag requests. On these requests, caller values for driver-managed headers (`Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`) are ignored on both backends, and the Thrift backend rejects header names or values that cannot be sent at connect time. (#491)
3131
- Error telemetry now reports only the error class name and the server SQLSTATE. It no longer includes the error message or stack trace, which could contain table and column names from the query. On the Thrift backend, `statement.complete` telemetry now also reports the result delivery format (inline Arrow, inline columnar, or CloudFetch) and whether the server LZ4-compressed the result. (#508)
32+
- Avoid an extra full-buffer copy when downloading Thrift CloudFetch results.
3233

3334
## 3.0.0
3435

‎lib/result/CloudFetchResultHandler.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,12 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
144144
this.chunkIndex += 1;
145145

146146
return {
147-
batches: [Buffer.from(result)],
147+
batches: [result],
148148
rowCount: link.rowCount.toNumber(true),
149149
};
150150
}
151151

152-
private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: ArrayBuffer }> {
152+
private async fetch(url: RequestInfo, init?: RequestInit): Promise<{ response: Response; body?: Buffer }> {
153153
const connectionProvider = await this.context.getConnectionProvider();
154154
let requestUrl = url;
155155
if (typeof requestUrl !== 'string') {
@@ -175,9 +175,9 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
175175
// Read the body inside the retried block. CloudFetch downloads are large
176176
// GETs against pre-signed cloud-storage URLs that frequently surface
177177
// `socket hang up` / "Premature close" once the stream is mid-transfer.
178-
// Pulling `arrayBuffer()` in here lets the retry policy treat those
178+
// Pulling `buffer()` in here lets the retry policy treat those
179179
// body-stream failures the same as connect-time failures.
180-
let downloaded: ArrayBuffer | undefined;
180+
let downloaded: Buffer | undefined;
181181
let result: { request: Request; response: Response };
182182
try {
183183
const download = retryPolicy.invokeWithRetry(async () => {
@@ -194,7 +194,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
194194
try {
195195
const response = await fetch(request);
196196
if (response.ok) {
197-
downloaded = await response.arrayBuffer();
197+
downloaded = await response.buffer();
198198
} else {
199199
lastFailure = { request, response };
200200
}
@@ -232,7 +232,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
232232
// callback). In production the body is always read inside the retried
233233
// block above, so this path is a no-op.
234234
if (downloaded === undefined && result.response.ok) {
235-
downloaded = await result.response.arrayBuffer();
235+
downloaded = await result.response.buffer();
236236
}
237237
return { response: result.response, body: downloaded };
238238
}

‎tests/unit/result/ArrowRowMaterialization.test.ts‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,127 @@ describe('Arrow row materialization', () => {
460460
);
461461
});
462462

463+
for (const numeric of [
464+
{
465+
name: 'BIGINT',
466+
type: () => new arrow.Int64(),
467+
thriftType: TTypeId.BIGINT_TYPE,
468+
values: [BigInt('9007199254740993'), BigInt('-9223372036854775808'), BigInt(0)],
469+
expected: [BigInt('9007199254740993'), BigInt('-9223372036854775808'), BigInt(0)],
470+
},
471+
{
472+
name: 'DECIMAL',
473+
type: () => new arrow.Decimal(4, 38, 128),
474+
thriftType: TTypeId.DECIMAL_TYPE,
475+
values: [decimal('90071992547409931234'), decimal('-12500'), decimal('0')],
476+
expected: ['9007199254740993.1234', '-1.2500', '0.0000'],
477+
// Dictionary decimals retain the legacy unscaled string representation.
478+
dictionaryExpected: ['90071992547409931234', '-12500', '0'],
479+
},
480+
]) {
481+
for (const dictionaryEncoded of [false, true]) {
482+
it(`matches legacy precise ${numeric.name} in scalars and containers (${rowMode}, dictionary=${dictionaryEncoded})`, async () => {
483+
const type = () =>
484+
dictionaryEncoded ? new arrow.Dictionary(numeric.type(), new arrow.Int32()) : numeric.type();
485+
const [positive, negative, zero] = numeric.values;
486+
const [expectedPositive, expectedNegative, expectedZero] =
487+
(dictionaryEncoded && numeric.dictionaryExpected) || numeric.expected;
488+
const columns: Column[] = [
489+
{
490+
field: new arrow.Field('scalar', type(), true),
491+
thriftType: numeric.thriftType,
492+
values: [positive, null, negative, positive, zero],
493+
expected: [expectedPositive, null, expectedNegative, expectedPositive, expectedZero],
494+
},
495+
{
496+
field: new arrow.Field('struct', new arrow.Struct([new arrow.Field('value', type(), true)]), true),
497+
thriftType: TTypeId.STRUCT_TYPE,
498+
values: [{ value: positive }, null, { value: null }, { value: negative }, { value: zero }],
499+
expected: [
500+
{ value: expectedPositive },
501+
null,
502+
{ value: null },
503+
{ value: expectedNegative },
504+
{ value: expectedZero },
505+
],
506+
},
507+
{
508+
field: new arrow.Field('list', new arrow.List(new arrow.Field('element', type(), true)), true),
509+
thriftType: TTypeId.ARRAY_TYPE,
510+
values: [[positive, null, negative, positive, zero], null, [], [null], [negative]],
511+
expected: [
512+
[expectedPositive, null, expectedNegative, expectedPositive, expectedZero],
513+
null,
514+
[],
515+
[null],
516+
[expectedNegative],
517+
],
518+
},
519+
{
520+
field: new arrow.Field(
521+
'map',
522+
new arrow.Map_(
523+
new arrow.Field(
524+
'entries',
525+
new arrow.Struct<{ key: arrow.Utf8; value: ReturnType<typeof type> }>([
526+
new arrow.Field('key', new arrow.Utf8(), false),
527+
new arrow.Field('value', type(), true),
528+
]),
529+
false,
530+
),
531+
),
532+
true,
533+
),
534+
thriftType: TTypeId.MAP_TYPE,
535+
values: [
536+
new Map<string, any>([
537+
['positive', positive],
538+
['missing', null],
539+
['negative', negative],
540+
['repeated', positive],
541+
['zero', zero],
542+
]),
543+
null,
544+
new Map(),
545+
new Map([['missing', null]]),
546+
new Map([['negative', negative]]),
547+
],
548+
expected: [
549+
{
550+
positive: expectedPositive,
551+
missing: null,
552+
negative: expectedNegative,
553+
repeated: expectedPositive,
554+
zero: expectedZero,
555+
},
556+
null,
557+
{},
558+
{ missing: null },
559+
{ negative: expectedNegative },
560+
],
561+
},
562+
];
563+
const converter = converterFor(columns, rowMode, true);
564+
const rows = await converter.fetchNext({ limit: 100 });
565+
const expected = columns[0].values.map((_, rowIndex) => {
566+
const values = columns.map((column) => column.expected[rowIndex]);
567+
return rowMode === 'array'
568+
? values
569+
: Object.fromEntries(columns.map((column, index) => [column.field.name, values[index]]));
570+
});
571+
deepStrictEqual(rows, expected);
572+
const table = arrow.tableFromIPC(ipc(columns));
573+
const legacyRows = (converter as any).getObjectRows(table.schema, table.toArray());
574+
deepStrictEqual(
575+
rows,
576+
rowMode === 'array'
577+
? legacyRows.map((row: any) => columns.map((column) => row[column.field.name]))
578+
: legacyRows,
579+
);
580+
});
581+
}
582+
}
583+
463584
for (const type of [new arrow.Binary(), new arrow.FixedSizeBinary(2)]) {
464585
it(`copies ${type} values out of the batch (${rowMode})`, async () => {
465586
const columns: Column[] = [

‎tests/unit/result/CloudFetchResultHandler.test.ts‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ import { ShouldRetryResult } from '../../../lib/connection/contracts/IRetryPolic
99
import { HttpTransactionDetails } from '../../../lib/connection/contracts/IConnectionProvider';
1010
import { LogLevel } from '../../../lib/contracts/IDBSQLLogger';
1111
import CloudFetchResultHandler from '../../../lib/result/CloudFetchResultHandler';
12+
import ArrowResultConverter from '../../../lib/result/ArrowResultConverter';
1213
import ResultsProviderStub from '../.stubs/ResultsProviderStub';
13-
import { TRowSet, TSparkArrowResultLink, TStatusCode } from '../../../thrift/TCLIService_types';
14+
import { TRowSet, TSparkArrowResultLink, TStatusCode, TTypeId } from '../../../thrift/TCLIService_types';
1415
import { ArrowBatch } from '../../../lib/result/utils';
1516
import RetryError, { RetryErrorCode } from '../../../lib/errors/RetryError';
1617

@@ -725,6 +726,32 @@ describe('CloudFetchResultHandler', () => {
725726
expect(batches).to.deep.eq([sampleArrowBatch]);
726727
});
727728

729+
[Buffer.alloc(0), Buffer.from([0, 255, 128, 42]), Buffer.from([99, 0, 255, 128, 42, 98]).subarray(1, 5)].forEach(
730+
(body, index) => {
731+
it(`should reuse the fallback response buffer without copying (${index})`, async () => {
732+
const context = new ClientContextStub({ cloudFetchConcurrentDownloads: 1 });
733+
const rowSetProvider = new ResultsProviderStub([sampleRowSet1], undefined);
734+
const result = new CloudFetchResultHandler(context, rowSetProvider, {
735+
lz4Compressed: false,
736+
status: { statusCode: TStatusCode.SUCCESS_STATUS },
737+
});
738+
const response = new Response(body, { status: 200 });
739+
const buffer = sinon.spy(response, 'buffer');
740+
const arrayBuffer = sinon.spy(response, 'arrayBuffer');
741+
context.invokeWithRetryStub.resolves({ request: new Request('localhost'), response });
742+
743+
const { batches, rowCount } = await result.fetchNext({ limit: 10000 });
744+
745+
expect(batches).to.have.lengthOf(1);
746+
expect(batches[0]).to.equal(body);
747+
expect(batches[0].byteLength).to.equal(body.byteLength);
748+
expect(rowCount).to.equal(1);
749+
expect(buffer.calledOnce).to.be.true;
750+
expect(arrayBuffer.called).to.be.false;
751+
});
752+
},
753+
);
754+
728755
describe('download deadline', () => {
729756
let server: http.Server | undefined;
730757

@@ -789,6 +816,42 @@ describe('CloudFetchResultHandler', () => {
789816
return { ...outcome, elapsedMs: Date.now() - startedAt };
790817
}
791818

819+
it('should decode a pooled Arrow response read inside the retry callback', async () => {
820+
const body = Buffer.concat([sampleArrowSchema, sampleArrowBatch]);
821+
const url = await startServer((req, res) => res.end(body));
822+
const handler = createHandler(url, { retriesTimeout: 1000 });
823+
const buffer = sinon.spy(Response.prototype, 'buffer');
824+
const arrayBuffer = sinon.spy(Response.prototype, 'arrayBuffer');
825+
const fetchNext = sinon.spy(handler, 'fetchNext');
826+
const converter = new ArrowResultConverter(new BaseClientContextStub(), handler, {
827+
schema: {
828+
columns: [
829+
{
830+
columnName: '1',
831+
typeDesc: { types: [{ primitiveEntry: { type: TTypeId.INT_TYPE } }] },
832+
position: 1,
833+
},
834+
],
835+
},
836+
});
837+
838+
try {
839+
expect(await converter.fetchNext({ limit: 10000 })).to.deep.eq([{ 1: 1 }]);
840+
expect(buffer.calledOnce).to.be.true;
841+
expect(arrayBuffer.called).to.be.false;
842+
expect(buffer.firstCall.thisValue.body).to.have.property('pipe').that.is.a('function');
843+
const downloaded = await buffer.firstCall.returnValue;
844+
expect(downloaded).to.deep.eq(body);
845+
expect(downloaded.length).to.be.lt(Buffer.poolSize >>> 1);
846+
expect(downloaded.buffer.byteLength).to.be.gt(downloaded.length);
847+
expect((await fetchNext.firstCall.returnValue).batches[0]).to.equal(downloaded);
848+
} finally {
849+
fetchNext.restore();
850+
arrayBuffer.restore();
851+
buffer.restore();
852+
}
853+
});
854+
792855
it('should fail a download whose response headers never arrive with a timeout error', async () => {
793856
const closedRequests: Array<Promise<void>> = [];
794857
const url = await startServer((req) => {

0 commit comments

Comments
 (0)