Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
>>> [CLI] experimental aitools tools discover-schema main.default.events
COLUMNS:
id: bigint
#tag: string
region: string


=== wire SQL

>>> print_requests.py --sort //api/2.0/sql/statements
DESCRIBE TABLE `main`.`default`.`events`
SELECT * FROM `main`.`default`.`events` LIMIT 5
SELECT COUNT(*) AS total_rows, SUM(CASE WHEN `id` IS NULL THEN 1 ELSE 0 END) AS `id_nulls`, SUM(CASE WHEN `#tag` IS NULL THEN 1 ELSE 0 END) AS `#tag_nulls`, SUM(CASE WHEN `region` IS NULL THEN 1 ELSE 0 END) AS `region_nulls` FROM `main`.`default`.`events`
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export DATABRICKS_WAREHOUSE_ID=$TEST_DEFAULT_WAREHOUSE_ID

# Only the COLUMNS section is meaningful: the stub returns the DESCRIBE output for every statement.
trace $CLI experimental aitools tools discover-schema main.default.events 2>&1 | sed -n -e '/^>>>/p' -e '/^COLUMNS:/,/^$/p'

title "wire SQL"
trace print_requests.py --sort //api/2.0/sql/statements | jq -r .body.statement
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
RecordRequests = true

# Every statement gets the same canned response: the DESCRIBE TABLE output of a
# table partitioned by `region`. Real DESCRIBE TABLE lists partition columns in
# the main column section and then again under "# Partition Information".
[[Server]]
Pattern = "POST /api/2.0/sql/statements"
Response.Body = '''
{
"statement_id": "s1",
"status": {"state": "SUCCEEDED"},
"manifest": {"schema": {"columns": [{"name": "col_name"}, {"name": "data_type"}, {"name": "comment"}]}, "total_chunk_count": 1},
"result": {"data_array": [
["id", "bigint", null],
["#tag", "string", null],
["region", "string", null],
["# Partition Information", "", ""],
["# col_name", "data_type", "comment"],
["region", "string", null]
]}
}
'''
9 changes: 6 additions & 3 deletions experimental/aitools/cmd/discover_schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,9 +328,12 @@ func parseDescribeResult(result *sqlexec.Result) (columns, types []string) {
}
colName := row[0]
colType := row[1]
// skip partition/metadata rows (they start with #)
if strings.HasPrefix(colName, "#") || colName == "" {
continue
// The column list ends at the first row without a type: the blank separator
// or a "# Partition Information" style header. The sections that follow
// repeat columns already listed above. The name can't be the signal because
// Delta allows column names starting with "#".
if colType == "" {
break
}
columns = append(columns, colName)
types = append(types, colType)
Expand Down
13 changes: 10 additions & 3 deletions experimental/aitools/cmd/discover_schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,26 @@ func TestQuoteTableName(t *testing.T) {
}

func TestParseDescribeResultSkipsMetadataRows(t *testing.T) {
// DESCRIBE TABLE lists partition columns in the main section and then
// repeats them under "# Partition Information".
result := &sqlexec.Result{
Rows: [][]string{
{"id", "BIGINT", ""},
{"#tag", "STRING", ""},
{"name", "STRING", ""},
{"region", "STRING", ""},
{"# Partition Information", "", ""},
{"# col_name", "data_type", "comment"},
{"region", "STRING", ""},
{"", "STRING", ""},
{"", "", ""},
{"# Clustering Information", "", ""},
{"name", "STRING", ""},
},
}

cols, types := parseDescribeResult(result)
assert.Equal(t, []string{"id", "name", "region"}, cols)
assert.Equal(t, []string{"BIGINT", "STRING", "STRING"}, types)
assert.Equal(t, []string{"id", "#tag", "name", "region"}, cols)
assert.Equal(t, []string{"BIGINT", "STRING", "STRING", "STRING"}, types)
}

func TestSQLGateRunPinsOnWaitTimeoutAndRecordsID(t *testing.T) {
Expand Down
Loading