From a6cf4e6f2bdc213bcfa77bc20285d244887e232a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 09:19:38 +0000 Subject: [PATCH 1/4] aitools: acceptance test for discover-schema on a partitioned table Records current behavior: partition columns are listed twice under COLUMNS and get a duplicate null-count expression, because parseDescribeResult reads past the '# Partition Information' section of DESCRIBE TABLE. Co-authored-by: Isaac --- .../discover-schema-partitioned/out.test.toml | 1 + .../discover-schema-partitioned/output.txt | 13 +++++++++ .../tools/discover-schema-partitioned/script | 5 ++++ .../discover-schema-partitioned/test.toml | 29 +++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml create mode 100644 acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt create mode 100644 acceptance/experimental/aitools/tools/discover-schema-partitioned/script create mode 100644 acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml b/acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt new file mode 100644 index 00000000000..42a98122fa8 --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt @@ -0,0 +1,13 @@ +>>> [CLI] experimental aitools tools discover-schema main.default.events +COLUMNS: + id: bigint + region: string + region: string + + +=== wire SQL + +>>> print_requests.py --sort //api/2.0/sql/statements +DESCRIBE TABLE `main`.`default`.`events` +SELECT * FROM `main`.`default`.`events` LIMIT 5 +SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events` diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/script b/acceptance/experimental/aitools/tools/discover-schema-partitioned/script new file mode 100644 index 00000000000..861575bcaad --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/script @@ -0,0 +1,5 @@ +# Only the COLUMNS section is meaningful: the stub returns the DESCRIBE output for every statement. +trace $CLI experimental aitools tools discover-schema main.default.events 2>&1 | sed -n -e '/^>>>/p' -e '/^COLUMNS:/,/^$/p' + +title "wire SQL" +trace print_requests.py --sort //api/2.0/sql/statements | jq -r .body.statement diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml b/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml new file mode 100644 index 00000000000..8974944c08c --- /dev/null +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml @@ -0,0 +1,29 @@ +RecordRequests = true + +Env.DATABRICKS_WAREHOUSE_ID = "wh-test" + +[[Server]] +Pattern = "GET /api/2.0/sql/warehouses/wh-test" +Response.Body = ''' +{"id": "wh-test", "name": "Test Warehouse", "state": "RUNNING"} +''' + +# Every statement gets the same canned response: the DESCRIBE TABLE output of a +# table partitioned by `region`. Real DESCRIBE TABLE lists partition columns in +# the main column section and then again under "# Partition Information". +[[Server]] +Pattern = "POST /api/2.0/sql/statements" +Response.Body = ''' +{ + "statement_id": "s1", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": {"columns": [{"name": "col_name"}, {"name": "data_type"}, {"name": "comment"}]}, "total_chunk_count": 1}, + "result": {"data_array": [ + ["id", "bigint", null], + ["region", "string", null], + ["# Partition Information", "", ""], + ["# col_name", "data_type", "comment"], + ["region", "string", null] + ]} +} +''' From 41d611adb23dcbdf72bfdd8462b673ec7ebb56d9 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 09:21:40 +0000 Subject: [PATCH 2/4] aitools: stop discover-schema column list at DESCRIBE metadata sections DESCRIBE TABLE lists partition (and clustering) columns in the main column section and then repeats them under '# Partition Information' / '# Clustering Information'. parseDescribeResult skipped the '#' header rows but kept reading, so those columns were reported twice and got duplicate null-count expressions. Stop at the first blank or '#' row. Co-authored-by: Isaac --- .../cli/aitools-discover-schema-partition-columns.md | 1 + .../aitools/tools/discover-schema-partitioned/output.txt | 3 +-- experimental/aitools/cmd/discover_schema.go | 6 ++++-- experimental/aitools/cmd/discover_schema_test.go | 8 +++++++- 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 .nextchanges/cli/aitools-discover-schema-partition-columns.md diff --git a/.nextchanges/cli/aitools-discover-schema-partition-columns.md b/.nextchanges/cli/aitools-discover-schema-partition-columns.md new file mode 100644 index 00000000000..2d19299b94b --- /dev/null +++ b/.nextchanges/cli/aitools-discover-schema-partition-columns.md @@ -0,0 +1 @@ +* Fix `databricks experimental aitools tools discover-schema` listing partition and clustering columns twice. diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt index 42a98122fa8..63daa4768d5 100644 --- a/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt @@ -2,7 +2,6 @@ COLUMNS: id: bigint region: string - region: string === wire SQL @@ -10,4 +9,4 @@ COLUMNS: >>> print_requests.py --sort //api/2.0/sql/statements DESCRIBE TABLE `main`.`default`.`events` SELECT * FROM `main`.`default`.`events` LIMIT 5 -SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events` +SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events` diff --git a/experimental/aitools/cmd/discover_schema.go b/experimental/aitools/cmd/discover_schema.go index 28f8bcb730b..7e3952d523f 100644 --- a/experimental/aitools/cmd/discover_schema.go +++ b/experimental/aitools/cmd/discover_schema.go @@ -328,9 +328,11 @@ func parseDescribeResult(result *sqlexec.Result) (columns, types []string) { } colName := row[0] colType := row[1] - // skip partition/metadata rows (they start with #) + // The column list ends at the first blank or "#" row. The sections that + // follow ("# Partition Information", "# Clustering Information", ...) + // repeat columns already listed above. if strings.HasPrefix(colName, "#") || colName == "" { - continue + break } columns = append(columns, colName) types = append(types, colType) diff --git a/experimental/aitools/cmd/discover_schema_test.go b/experimental/aitools/cmd/discover_schema_test.go index d0fae4b24c2..694a2d002cb 100644 --- a/experimental/aitools/cmd/discover_schema_test.go +++ b/experimental/aitools/cmd/discover_schema_test.go @@ -51,13 +51,19 @@ func TestQuoteTableName(t *testing.T) { } func TestParseDescribeResultSkipsMetadataRows(t *testing.T) { + // DESCRIBE TABLE lists partition columns in the main section and then + // repeats them under "# Partition Information". result := &sqlexec.Result{ Rows: [][]string{ {"id", "BIGINT", ""}, {"name", "STRING", ""}, + {"region", "STRING", ""}, {"# Partition Information", "", ""}, + {"# col_name", "data_type", "comment"}, {"region", "STRING", ""}, - {"", "STRING", ""}, + {"", "", ""}, + {"# Clustering Information", "", ""}, + {"name", "STRING", ""}, }, } From bbd5d39c329aea6386b9828ec8b85dfb9f88357d Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Mon, 16 Nov 2026 09:20:52 +0000 Subject: [PATCH 3/4] Drop changelog fragment: experimental commands have no entries Co-authored-by: Isaac --- .nextchanges/cli/aitools-discover-schema-partition-columns.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .nextchanges/cli/aitools-discover-schema-partition-columns.md diff --git a/.nextchanges/cli/aitools-discover-schema-partition-columns.md b/.nextchanges/cli/aitools-discover-schema-partition-columns.md deleted file mode 100644 index 2d19299b94b..00000000000 --- a/.nextchanges/cli/aitools-discover-schema-partition-columns.md +++ /dev/null @@ -1 +0,0 @@ -* Fix `databricks experimental aitools tools discover-schema` listing partition and clustering columns twice. From d4a76ecc9e9fb0120f7c9a0297ee294c506d42a6 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Mon, 16 Nov 2026 13:35:31 +0000 Subject: [PATCH 4/4] aitools: end the DESCRIBE column list at the first row without a type Delta allows column names that start with '#', so the name can't mark the end of the column list; a real column such as '#tag' cut the list short. The blank separator and the section headers are the only rows without a data_type. Also use the fake workspace's default warehouse in the test. Co-authored-by: Isaac --- .../aitools/tools/discover-schema-partitioned/output.txt | 3 ++- .../aitools/tools/discover-schema-partitioned/script | 2 ++ .../aitools/tools/discover-schema-partitioned/test.toml | 9 +-------- experimental/aitools/cmd/discover_schema.go | 9 +++++---- experimental/aitools/cmd/discover_schema_test.go | 5 +++-- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt index 63daa4768d5..23a66b39851 100644 --- a/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/output.txt @@ -1,6 +1,7 @@ >>> [CLI] experimental aitools tools discover-schema main.default.events COLUMNS: id: bigint + #tag: string region: string @@ -9,4 +10,4 @@ COLUMNS: >>> print_requests.py --sort //api/2.0/sql/statements DESCRIBE TABLE `main`.`default`.`events` SELECT * FROM `main`.`default`.`events` LIMIT 5 -SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events` +SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `#tag` IS NULL THEN 1 ELSE 0 END) AS `#tag_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events` diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/script b/acceptance/experimental/aitools/tools/discover-schema-partitioned/script index 861575bcaad..05fc5329979 100644 --- a/acceptance/experimental/aitools/tools/discover-schema-partitioned/script +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/script @@ -1,3 +1,5 @@ +export DATABRICKS_WAREHOUSE_ID=$TEST_DEFAULT_WAREHOUSE_ID + # Only the COLUMNS section is meaningful: the stub returns the DESCRIBE output for every statement. trace $CLI experimental aitools tools discover-schema main.default.events 2>&1 | sed -n -e '/^>>>/p' -e '/^COLUMNS:/,/^$/p' diff --git a/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml b/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml index 8974944c08c..a70c6392510 100644 --- a/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml +++ b/acceptance/experimental/aitools/tools/discover-schema-partitioned/test.toml @@ -1,13 +1,5 @@ RecordRequests = true -Env.DATABRICKS_WAREHOUSE_ID = "wh-test" - -[[Server]] -Pattern = "GET /api/2.0/sql/warehouses/wh-test" -Response.Body = ''' -{"id": "wh-test", "name": "Test Warehouse", "state": "RUNNING"} -''' - # Every statement gets the same canned response: the DESCRIBE TABLE output of a # table partitioned by `region`. Real DESCRIBE TABLE lists partition columns in # the main column section and then again under "# Partition Information". @@ -20,6 +12,7 @@ Response.Body = ''' "manifest": {"schema": {"columns": [{"name": "col_name"}, {"name": "data_type"}, {"name": "comment"}]}, "total_chunk_count": 1}, "result": {"data_array": [ ["id", "bigint", null], + ["#tag", "string", null], ["region", "string", null], ["# Partition Information", "", ""], ["# col_name", "data_type", "comment"], diff --git a/experimental/aitools/cmd/discover_schema.go b/experimental/aitools/cmd/discover_schema.go index 7e3952d523f..133ea037c1f 100644 --- a/experimental/aitools/cmd/discover_schema.go +++ b/experimental/aitools/cmd/discover_schema.go @@ -328,10 +328,11 @@ func parseDescribeResult(result *sqlexec.Result) (columns, types []string) { } colName := row[0] colType := row[1] - // The column list ends at the first blank or "#" row. The sections that - // follow ("# Partition Information", "# Clustering Information", ...) - // repeat columns already listed above. - if strings.HasPrefix(colName, "#") || colName == "" { + // The column list ends at the first row without a type: the blank separator + // or a "# Partition Information" style header. The sections that follow + // repeat columns already listed above. The name can't be the signal because + // Delta allows column names starting with "#". + if colType == "" { break } columns = append(columns, colName) diff --git a/experimental/aitools/cmd/discover_schema_test.go b/experimental/aitools/cmd/discover_schema_test.go index 694a2d002cb..a771519fbe9 100644 --- a/experimental/aitools/cmd/discover_schema_test.go +++ b/experimental/aitools/cmd/discover_schema_test.go @@ -56,6 +56,7 @@ func TestParseDescribeResultSkipsMetadataRows(t *testing.T) { result := &sqlexec.Result{ Rows: [][]string{ {"id", "BIGINT", ""}, + {"#tag", "STRING", ""}, {"name", "STRING", ""}, {"region", "STRING", ""}, {"# Partition Information", "", ""}, @@ -68,8 +69,8 @@ func TestParseDescribeResultSkipsMetadataRows(t *testing.T) { } cols, types := parseDescribeResult(result) - assert.Equal(t, []string{"id", "name", "region"}, cols) - assert.Equal(t, []string{"BIGINT", "STRING", "STRING"}, types) + assert.Equal(t, []string{"id", "#tag", "name", "region"}, cols) + assert.Equal(t, []string{"BIGINT", "STRING", "STRING", "STRING"}, types) } func TestSQLGateRunPinsOnWaitTimeoutAndRecordsID(t *testing.T) {