diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml b/acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt new file mode 100644 index 00000000000..23a66b39851 --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt @@ -0,0 +1,13 @@ +>>> [CLI] experimental aitools tools discover-schema main.default.events +COLUMNS: + id: bigint + #tag: string + region: string + + +=== wire SQL + +>>> print_requests.py --sort //api/2.0/sql/statements +DESCRIBE TABLE `main`.`default`.`events` +SELECT * FROM `main`.`default`.`events` LIMIT 5 +SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `#tag` IS NULL THEN 1 ELSE 0 END) AS `#tag_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events` diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/script b/acceptance/experimental/aitools/tools/discover-schema-partitioned/script new file mode 100644 index 00000000000..05fc5329979 --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/script @@ -0,0 +1,7 @@ +export DATABRICKS_WAREHOUSE_ID=$TEST_DEFAULT_WAREHOUSE_ID + +# Only the COLUMNS section is meaningful: the stub returns the DESCRIBE output for every statement. +trace $CLI experimental aitools tools discover-schema main.default.events 2>&1 | sed -n -e '/^>>>/p' -e '/^COLUMNS:/,/^$/p' + +title "wire SQL" +trace print_requests.py --sort //api/2.0/sql/statements | jq -r .body.statement diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml b/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml new file mode 100644 index 00000000000..a70c6392510 --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml @@ -0,0 +1,22 @@ +RecordRequests = true + +# Every statement gets the same canned response: the DESCRIBE TABLE output of a +# table partitioned by `region`. Real DESCRIBE TABLE lists partition columns in +# the main column section and then again under "# Partition Information". +[[Server]] +Pattern = "POST /api/2.0/sql/statements" +Response.Body = ''' +{ + "statement_id": "s1", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": {"columns": [{"name": "col_name"}, {"name": "data_type"}, {"name": "comment"}]}, "total_chunk_count": 1}, + "result": {"data_array": [ + ["id", "bigint", null], + ["#tag", "string", null], + ["region", "string", null], + ["# Partition Information", "", ""], + ["# col_name", "data_type", "comment"], + ["region", "string", null] + ]} +} +''' diff --git a/experimental/aitools/cmd/discover_schema.go b/experimental/aitools/cmd/discover_schema.go index 28f8bcb730b..133ea037c1f 100644 --- a/experimental/aitools/cmd/discover_schema.go +++ b/experimental/aitools/cmd/discover_schema.go @@ -328,9 +328,12 @@ func parseDescribeResult(result *sqlexec.Result) (columns, types []string) { } colName := row[0] colType := row[1] - // skip partition/metadata rows (they start with #) - if strings.HasPrefix(colName, "#") || colName == "" { - continue + // The column list ends at the first row without a type: the blank separator + // or a "# Partition Information" style header. The sections that follow + // repeat columns already listed above. The name can't be the signal because + // Delta allows column names starting with "#". + if colType == "" { + break } columns = append(columns, colName) types = append(types, colType) diff --git a/experimental/aitools/cmd/discover_schema_test.go b/experimental/aitools/cmd/discover_schema_test.go index d0fae4b24c2..a771519fbe9 100644 --- a/experimental/aitools/cmd/discover_schema_test.go +++ b/experimental/aitools/cmd/discover_schema_test.go @@ -51,19 +51,26 @@ func TestQuoteTableName(t *testing.T) { } func TestParseDescribeResultSkipsMetadataRows(t *testing.T) { + // DESCRIBE TABLE lists partition columns in the main section and then + // repeats them under "# Partition Information". result := &sqlexec.Result{ Rows: [][]string{ {"id", "BIGINT", ""}, + {"#tag", "STRING", ""}, {"name", "STRING", ""}, + {"region", "STRING", ""}, {"# Partition Information", "", ""}, + {"# col_name", "data_type", "comment"}, {"region", "STRING", ""}, - {"", "STRING", ""}, + {"", "", ""}, + {"# Clustering Information", "", ""}, + {"name", "STRING", ""}, }, } cols, types := parseDescribeResult(result) - assert.Equal(t, []string{"id", "name", "region"}, cols) - assert.Equal(t, []string{"BIGINT", "STRING", "STRING"}, types) + assert.Equal(t, []string{"id", "#tag", "name", "region"}, cols) + assert.Equal(t, []string{"BIGINT", "STRING", "STRING", "STRING"}, types) } func TestSQLGateRunPinsOnWaitTimeoutAndRecordsID(t *testing.T) {