Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions shim/analyzer_shim.cc
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,9 @@ int GoogleSQL_AnalyzeNextStatementWithCatalogParams(
}

googlesql::AnalyzerOptions options(handle->lang);
// Scans list only the columns the statement references, so a window or a sort carries no column
// its consumer never reads.
options.set_prune_unused_columns(true);
// BigQuery reads a TIMESTAMP literal with no zone, and every other
// zoneless civil time the analyzer folds, as UTC.
options.set_default_time_zone(absl::UTCTimeZone());
Expand Down Expand Up @@ -709,6 +712,7 @@ int GoogleSQL_AnalyzeNextStatementParams(
}

googlesql::AnalyzerOptions options(lang);
options.set_prune_unused_columns(true);
// See the note in the handle variant on the default time zone.
options.set_default_time_zone(absl::UTCTimeZone());
// options_json carries the parameter mode: `{"parameter_mode":"positional"}`
Expand Down
48 changes: 32 additions & 16 deletions src/internal/transformer/scan_aggregate.zig
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,17 @@ pub fn transform(
return q;
}

/// The select item of an aggregate whose columns are all unread. The aggregate still yields a row
/// per group, and a SELECT needs a column to yield them in.
fn unreadSelectItem(alloc: std.mem.Allocator) Error!*sqlbuilder.SelectListItem {
const item = try alloc.create(sqlbuilder.SelectListItem);
item.* = .{
.expression = try sqlbuilder.newLiteralExpression(alloc, "NULL"),
.alias = "zigquery_unread",
};
return item;
}

/// GROUPING SETS, ROLLUP and CUBE as a UNION ALL of one SELECT per grouping set, with NULL
/// projected for the columns a set does not group by, ordered by the grouping columns.
fn buildGroupingSetsQuery(
Expand Down Expand Up @@ -172,7 +183,11 @@ fn buildGroupingSetsQuery(
if (!set_column_ids.contains(id.*)) try null_column_ids.put(id.*, {});
}

const select_list = try alloc.alloc(*sqlbuilder.SelectListItem, output_columns.len);
const select_list = try alloc.alloc(
*sqlbuilder.SelectListItem,
@max(output_columns.len, 1),
);
if (output_columns.len == 0) select_list[0] = try unreadSelectItem(alloc);
for (output_columns, 0..) |original_col, i| {
// GROUPING(x): 1 when this set rolled x up, 0 when it grouped by
// it. Constant within the set, so it is written as a literal rather
Expand Down Expand Up @@ -246,26 +261,27 @@ fn buildGroupingSetsQuery(

// ORDER BY the grouping columns with the value-aware collation so the
// grouping levels come out in a deterministic ZetaSQL-like order.
if (group_by_map.count() > 0) {
var order_by_items: std.ArrayList(*sqlbuilder.OrderByItem) = .empty;
var val_it = group_by_map.valueIterator();
while (val_it.next()) |group_by_col| {
// Copy the expression: the ORDER BY term must not alias the GROUP BY one, which
// gets `collation` assigned in place.
const order_expr = try alloc.create(SQLExpression);
order_expr.* = group_by_col.*.*;

const item = try alloc.create(sqlbuilder.OrderByItem);
item.* = .{ .expression = order_expr, .direction = "ASC" };
try order_by_items.append(alloc, item);
}
union_stmt.order_by_list = try order_by_items.toOwnedSlice(alloc);
// An ORDER BY over a compound SELECT names result columns, so its terms are the grouping
// columns the output holds.
var order_by_items: std.ArrayList(*sqlbuilder.OrderByItem) = .empty;
for (column_data) |column| {
const group_by_col = group_by_map.get(column.id) orelse continue;
// Copy the expression: the ORDER BY term must not alias the GROUP BY one, which
// gets `collation` assigned in place.
const order_expr = try alloc.create(SQLExpression);
order_expr.* = group_by_col.*;

const item = try alloc.create(sqlbuilder.OrderByItem);
item.* = .{ .expression = order_expr, .direction = "ASC" };
try order_by_items.append(alloc, item);
}
union_stmt.order_by_list = try order_by_items.toOwnedSlice(alloc);

// Re-project through an inner subquery to introduce the new logical
// columns (aliases from the first set's SELECT items).
const statement = try alloc.create(sqlbuilder.SelectStatement);
const outer_list = try alloc.alloc(*sqlbuilder.SelectListItem, column_data.len);
const outer_list = try alloc.alloc(*sqlbuilder.SelectListItem, @max(column_data.len, 1));
if (column_data.len == 0) outer_list[0] = try unreadSelectItem(alloc);
for (column_data, 0..) |column, i| {
const item_column = set_operation.items[0].select_list[i];
const item = try alloc.create(sqlbuilder.SelectListItem);
Expand Down
14 changes: 14 additions & 0 deletions testdata/spec_baseline.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2052,8 +2052,12 @@ googlesql/compliance/grouping_sets_queries grouping_sets_with_alias
googlesql/compliance/grouping_sets_queries grouping_sets_with_ordinal_columns
googlesql/compliance/grouping_sets_queries grouping_sets_with_mix_of_column_alias_ordinal_index
googlesql/compliance/grouping_sets_queries grouping_sets_with_duplicate_simple_expressions
googlesql/compliance/grouping_sets_queries grouping_sets_with_duplicate_subquery_expressions
googlesql/compliance/grouping_sets_queries grouping_sets_with_collation_columns
googlesql/compliance/grouping_sets_queries grouping_sets_with_analytic_functions
googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_constant
googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_from_the_table
googlesql/compliance/grouping_sets_queries grouping_sets_with_correlated_subquery
googlesql/compliance/grouping_sets_queries grouping_sets_with_keyword_column_name
googlesql/compliance/grouping_sets_queries grouping_sets_with_rollup_cube
googlesql/compliance/grouping_sets_queries grouping_sets_with_rollup_cube_on_multi_columns
Expand Down Expand Up @@ -2081,6 +2085,7 @@ googlesql/compliance/grouping_sets_queries rollup_with_duplicate_mix_of_column_a
googlesql/compliance/grouping_sets_queries rollup_with_mix_of_column_alias_ordinal_index
googlesql/compliance/grouping_sets_queries rollup_expression_match
googlesql/compliance/grouping_sets_queries rollup_multi_column_expression_match
googlesql/compliance/grouping_sets_queries rollup_with_subquery
googlesql/compliance/grouping_sets_queries rollup_with_collation_columns
googlesql/compliance/grouping_sets_queries cube_simple_query
googlesql/compliance/grouping_sets_queries cube_simple_query_no_grouping
Expand All @@ -2100,6 +2105,7 @@ googlesql/compliance/grouping_sets_queries cube_with_unpivot
googlesql/compliance/grouping_sets_queries cube_and_with_recursive
googlesql/compliance/grouping_sets_queries cube_expression_match
googlesql/compliance/grouping_sets_queries cube_multi_column_expression_match
googlesql/compliance/grouping_sets_queries cube_with_subquery
googlesql/compliance/grouping_sets_queries cube_with_keyword_column_name
googlesql/compliance/grouping_sets_queries rollup_with_analytic_functions
googlesql/compliance/grouping_sets_queries rollup_with_qualify_clauses
Expand Down Expand Up @@ -5063,6 +5069,10 @@ zigquery/functions/utility/generate_uuid generate_uuid
zigquery/functions/utility/safe_prefix safe on a function that is neither aggregate nor analytic
zigquery/functions/utility/safe_prefix safe names the function it was applied to
zigquery/functions/utility/safe_prefix a function with no implementation names itself
zigquery/functions/window/analytic_option_rejections string_agg with distinct over a window is rejected
zigquery/functions/window/analytic_option_rejections array_agg with ignore nulls over a window is rejected
zigquery/functions/window/analytic_option_rejections string_agg over a window without distinct
zigquery/functions/window/analytic_option_rejections array_agg over a window without ignore nulls
zigquery/functions/window/cume_dist cume_dist
zigquery/functions/window/dense_rank window dense_rank
zigquery/functions/window/dense_rank window dense_rank with group
Expand Down Expand Up @@ -5217,6 +5227,10 @@ zigquery/syntax/query/function_arg_limit function call with many arguments - abo
zigquery/syntax/query/group_by group by
zigquery/syntax/query/group_by group by rollup with one column
zigquery/syntax/query/group_by group by rollup with two columns
zigquery/syntax/query/group_by rollup selecting none of its grouping columns
zigquery/syntax/query/group_by rollup selecting only its second grouping column
zigquery/syntax/query/group_by grouping sets with a constant select list
zigquery/syntax/query/group_by group by with a constant select list
zigquery/syntax/query/group_by group by having
zigquery/syntax/query/group_by group by struct values basic
zigquery/syntax/query/group_by group by struct values with aggregation
Expand Down
6 changes: 0 additions & 6 deletions testdata/spec_known_failures.txt
Original file line number Diff line number Diff line change
Expand Up @@ -926,15 +926,9 @@ googlesql/compliance/groupby_queries groupby_single_base_column_uuid
googlesql/compliance/groupby_queries_2 group_by_empty_columns_select_literals
googlesql/compliance/groupby_queries_2 group_by_rollup_struct_constructor
googlesql/compliance/groupby_queries_2 grouping_function_supports_struct_constructor
googlesql/compliance/grouping_sets_queries grouping_sets_with_duplicate_subquery_expressions
googlesql/compliance/grouping_sets_queries grouping_sets_with_aggregate_functions
googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_constant
googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_from_the_table
googlesql/compliance/grouping_sets_queries grouping_sets_with_correlated_subquery
googlesql/compliance/grouping_sets_queries rollup_with_subquery
googlesql/compliance/grouping_sets_queries cube_with_multi_columns_grouping_set
googlesql/compliance/grouping_sets_queries cube_with_multi_columns_grouping_set_no_grouping
googlesql/compliance/grouping_sets_queries cube_with_subquery
googlesql/compliance/grouping_sets_queries grouping_func_in_having_clause
googlesql/compliance/grouping_sets_queries grouping_func_in_qualify_clause
googlesql/compliance/hints reject_unknown_hints_in
Expand Down
116 changes: 116 additions & 0 deletions testdata/specs/zigquery/syntax/query/group_by.zon
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,122 @@
.{ 789, 3, 1.99 },
},
},
// A grouping set yields one row per group whether or not the select list reads its
// columns. The rows are five (sku, day) groups, three sku groups and the grand total.
// query-syntax.md#group_by_rollup
.{
.desc = "rollup selecting none of its grouping columns",
.sql =
\\WITH Sales AS (
\\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL
\\ SELECT 123, 1, 8.99 UNION ALL
\\ SELECT 456, 1, 4.56 UNION ALL
\\ SELECT 123, 2, 9.99 UNION ALL
\\ SELECT 789, 3, 1.00 UNION ALL
\\ SELECT 456, 3, 4.25 UNION ALL
\\ SELECT 789, 3, 0.99
\\)
\\SELECT COUNT(*) AS n
\\FROM Sales
\\GROUP BY ROLLUP(sku, day)
,
.unordered = true,
.rows = .{
.{ 2 },
.{ 1 },
.{ 1 },
.{ 1 },
.{ 2 },
.{ 3 },
.{ 2 },
.{ 2 },
.{ 7 },
},
},
// day is NULL in the rows of the sets that roll it up, which are the three sku groups and
// the grand total.
// query-syntax.md#group_by_rollup
.{
.desc = "rollup selecting only its second grouping column",
.sql =
\\WITH Sales AS (
\\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL
\\ SELECT 123, 1, 8.99 UNION ALL
\\ SELECT 456, 1, 4.56 UNION ALL
\\ SELECT 123, 2, 9.99 UNION ALL
\\ SELECT 789, 3, 1.00 UNION ALL
\\ SELECT 456, 3, 4.25 UNION ALL
\\ SELECT 789, 3, 0.99
\\)
\\SELECT day, COUNT(*) AS n
\\FROM Sales
\\GROUP BY ROLLUP(sku, day)
,
.unordered = true,
.rows = .{
.{ 1, 2 },
.{ 2, 1 },
.{ 1, 1 },
.{ 3, 1 },
.{ 3, 2 },
.{ null, 3 },
.{ null, 2 },
.{ null, 2 },
.{ null, 7 },
},
},
// One row per group of each set, three skus and three days, each carrying the constant.
// query-syntax.md#group_by_grouping_sets
.{
.desc = "grouping sets with a constant select list",
.sql =
\\WITH Sales AS (
\\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL
\\ SELECT 123, 1, 8.99 UNION ALL
\\ SELECT 456, 1, 4.56 UNION ALL
\\ SELECT 123, 2, 9.99 UNION ALL
\\ SELECT 789, 3, 1.00 UNION ALL
\\ SELECT 456, 3, 4.25 UNION ALL
\\ SELECT 789, 3, 0.99
\\)
\\SELECT 'x' AS c
\\FROM Sales
\\GROUP BY GROUPING SETS(sku, day)
,
.unordered = true,
.rows = .{
.{ "x" },
.{ "x" },
.{ "x" },
.{ "x" },
.{ "x" },
.{ "x" },
},
},
// One row per sku, though no column of the aggregation is read.
.{
.desc = "group by with a constant select list",
.sql =
\\WITH Sales AS (
\\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL
\\ SELECT 123, 1, 8.99 UNION ALL
\\ SELECT 456, 1, 4.56 UNION ALL
\\ SELECT 123, 2, 9.99 UNION ALL
\\ SELECT 789, 3, 1.00 UNION ALL
\\ SELECT 456, 3, 4.25 UNION ALL
\\ SELECT 789, 3, 0.99
\\)
\\SELECT 'x' AS c
\\FROM Sales
\\GROUP BY sku
,
.unordered = true,
.rows = .{
.{ "x" },
.{ "x" },
.{ "x" },
},
},
.{
.desc = "group by having",
.sql =
Expand Down
Loading