diff --git a/shim/analyzer_shim.cc b/shim/analyzer_shim.cc index 862341f..bdf546e 100644 --- a/shim/analyzer_shim.cc +++ b/shim/analyzer_shim.cc @@ -619,6 +619,9 @@ int GoogleSQL_AnalyzeNextStatementWithCatalogParams( } googlesql::AnalyzerOptions options(handle->lang); + // Scans list only the columns the statement references, so a window or a sort carries no column + // its consumer never reads. + options.set_prune_unused_columns(true); // BigQuery reads a TIMESTAMP literal with no zone, and every other // zoneless civil time the analyzer folds, as UTC. options.set_default_time_zone(absl::UTCTimeZone()); @@ -709,6 +712,7 @@ int GoogleSQL_AnalyzeNextStatementParams( } googlesql::AnalyzerOptions options(lang); + options.set_prune_unused_columns(true); // See the note in the handle variant on the default time zone. options.set_default_time_zone(absl::UTCTimeZone()); // options_json carries the parameter mode: `{"parameter_mode":"positional"}` diff --git a/src/internal/transformer/scan_aggregate.zig b/src/internal/transformer/scan_aggregate.zig index 1aa08e1..a1930af 100644 --- a/src/internal/transformer/scan_aggregate.zig +++ b/src/internal/transformer/scan_aggregate.zig @@ -134,6 +134,17 @@ pub fn transform( return q; } +/// The select item of an aggregate whose columns are all unread. The aggregate still yields a row +/// per group, and a SELECT needs a column to yield them in. +fn unreadSelectItem(alloc: std.mem.Allocator) Error!*sqlbuilder.SelectListItem { + const item = try alloc.create(sqlbuilder.SelectListItem); + item.* = .{ + .expression = try sqlbuilder.newLiteralExpression(alloc, "NULL"), + .alias = "zigquery_unread", + }; + return item; +} + /// GROUPING SETS, ROLLUP and CUBE as a UNION ALL of one SELECT per grouping set, with NULL /// projected for the columns a set does not group by, ordered by the grouping columns. fn buildGroupingSetsQuery( @@ -172,7 +183,11 @@ fn buildGroupingSetsQuery( if (!set_column_ids.contains(id.*)) try null_column_ids.put(id.*, {}); } - const select_list = try alloc.alloc(*sqlbuilder.SelectListItem, output_columns.len); + const select_list = try alloc.alloc( + *sqlbuilder.SelectListItem, + @max(output_columns.len, 1), + ); + if (output_columns.len == 0) select_list[0] = try unreadSelectItem(alloc); for (output_columns, 0..) |original_col, i| { // GROUPING(x): 1 when this set rolled x up, 0 when it grouped by // it. Constant within the set, so it is written as a literal rather @@ -246,26 +261,27 @@ fn buildGroupingSetsQuery( // ORDER BY the grouping columns with the value-aware collation so the // grouping levels come out in a deterministic ZetaSQL-like order. - if (group_by_map.count() > 0) { - var order_by_items: std.ArrayList(*sqlbuilder.OrderByItem) = .empty; - var val_it = group_by_map.valueIterator(); - while (val_it.next()) |group_by_col| { - // Copy the expression: the ORDER BY term must not alias the GROUP BY one, which - // gets `collation` assigned in place. - const order_expr = try alloc.create(SQLExpression); - order_expr.* = group_by_col.*.*; - - const item = try alloc.create(sqlbuilder.OrderByItem); - item.* = .{ .expression = order_expr, .direction = "ASC" }; - try order_by_items.append(alloc, item); - } - union_stmt.order_by_list = try order_by_items.toOwnedSlice(alloc); + // An ORDER BY over a compound SELECT names result columns, so its terms are the grouping + // columns the output holds. + var order_by_items: std.ArrayList(*sqlbuilder.OrderByItem) = .empty; + for (column_data) |column| { + const group_by_col = group_by_map.get(column.id) orelse continue; + // Copy the expression: the ORDER BY term must not alias the GROUP BY one, which + // gets `collation` assigned in place. + const order_expr = try alloc.create(SQLExpression); + order_expr.* = group_by_col.*; + + const item = try alloc.create(sqlbuilder.OrderByItem); + item.* = .{ .expression = order_expr, .direction = "ASC" }; + try order_by_items.append(alloc, item); } + union_stmt.order_by_list = try order_by_items.toOwnedSlice(alloc); // Re-project through an inner subquery to introduce the new logical // columns (aliases from the first set's SELECT items). const statement = try alloc.create(sqlbuilder.SelectStatement); - const outer_list = try alloc.alloc(*sqlbuilder.SelectListItem, column_data.len); + const outer_list = try alloc.alloc(*sqlbuilder.SelectListItem, @max(column_data.len, 1)); + if (column_data.len == 0) outer_list[0] = try unreadSelectItem(alloc); for (column_data, 0..) |column, i| { const item_column = set_operation.items[0].select_list[i]; const item = try alloc.create(sqlbuilder.SelectListItem); diff --git a/testdata/spec_baseline.txt b/testdata/spec_baseline.txt index bd1d6d0..bcaf5bb 100644 --- a/testdata/spec_baseline.txt +++ b/testdata/spec_baseline.txt @@ -2052,8 +2052,12 @@ googlesql/compliance/grouping_sets_queries grouping_sets_with_alias googlesql/compliance/grouping_sets_queries grouping_sets_with_ordinal_columns googlesql/compliance/grouping_sets_queries grouping_sets_with_mix_of_column_alias_ordinal_index googlesql/compliance/grouping_sets_queries grouping_sets_with_duplicate_simple_expressions +googlesql/compliance/grouping_sets_queries grouping_sets_with_duplicate_subquery_expressions googlesql/compliance/grouping_sets_queries grouping_sets_with_collation_columns googlesql/compliance/grouping_sets_queries grouping_sets_with_analytic_functions +googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_constant +googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_from_the_table +googlesql/compliance/grouping_sets_queries grouping_sets_with_correlated_subquery googlesql/compliance/grouping_sets_queries grouping_sets_with_keyword_column_name googlesql/compliance/grouping_sets_queries grouping_sets_with_rollup_cube googlesql/compliance/grouping_sets_queries grouping_sets_with_rollup_cube_on_multi_columns @@ -2081,6 +2085,7 @@ googlesql/compliance/grouping_sets_queries rollup_with_duplicate_mix_of_column_a googlesql/compliance/grouping_sets_queries rollup_with_mix_of_column_alias_ordinal_index googlesql/compliance/grouping_sets_queries rollup_expression_match googlesql/compliance/grouping_sets_queries rollup_multi_column_expression_match +googlesql/compliance/grouping_sets_queries rollup_with_subquery googlesql/compliance/grouping_sets_queries rollup_with_collation_columns googlesql/compliance/grouping_sets_queries cube_simple_query googlesql/compliance/grouping_sets_queries cube_simple_query_no_grouping @@ -2100,6 +2105,7 @@ googlesql/compliance/grouping_sets_queries cube_with_unpivot googlesql/compliance/grouping_sets_queries cube_and_with_recursive googlesql/compliance/grouping_sets_queries cube_expression_match googlesql/compliance/grouping_sets_queries cube_multi_column_expression_match +googlesql/compliance/grouping_sets_queries cube_with_subquery googlesql/compliance/grouping_sets_queries cube_with_keyword_column_name googlesql/compliance/grouping_sets_queries rollup_with_analytic_functions googlesql/compliance/grouping_sets_queries rollup_with_qualify_clauses @@ -5063,6 +5069,10 @@ zigquery/functions/utility/generate_uuid generate_uuid zigquery/functions/utility/safe_prefix safe on a function that is neither aggregate nor analytic zigquery/functions/utility/safe_prefix safe names the function it was applied to zigquery/functions/utility/safe_prefix a function with no implementation names itself +zigquery/functions/window/analytic_option_rejections string_agg with distinct over a window is rejected +zigquery/functions/window/analytic_option_rejections array_agg with ignore nulls over a window is rejected +zigquery/functions/window/analytic_option_rejections string_agg over a window without distinct +zigquery/functions/window/analytic_option_rejections array_agg over a window without ignore nulls zigquery/functions/window/cume_dist cume_dist zigquery/functions/window/dense_rank window dense_rank zigquery/functions/window/dense_rank window dense_rank with group @@ -5217,6 +5227,10 @@ zigquery/syntax/query/function_arg_limit function call with many arguments - abo zigquery/syntax/query/group_by group by zigquery/syntax/query/group_by group by rollup with one column zigquery/syntax/query/group_by group by rollup with two columns +zigquery/syntax/query/group_by rollup selecting none of its grouping columns +zigquery/syntax/query/group_by rollup selecting only its second grouping column +zigquery/syntax/query/group_by grouping sets with a constant select list +zigquery/syntax/query/group_by group by with a constant select list zigquery/syntax/query/group_by group by having zigquery/syntax/query/group_by group by struct values basic zigquery/syntax/query/group_by group by struct values with aggregation diff --git a/testdata/spec_known_failures.txt b/testdata/spec_known_failures.txt index b80b9be..7c9c228 100644 --- a/testdata/spec_known_failures.txt +++ b/testdata/spec_known_failures.txt @@ -926,15 +926,9 @@ googlesql/compliance/groupby_queries groupby_single_base_column_uuid googlesql/compliance/groupby_queries_2 group_by_empty_columns_select_literals googlesql/compliance/groupby_queries_2 group_by_rollup_struct_constructor googlesql/compliance/groupby_queries_2 grouping_function_supports_struct_constructor -googlesql/compliance/grouping_sets_queries grouping_sets_with_duplicate_subquery_expressions googlesql/compliance/grouping_sets_queries grouping_sets_with_aggregate_functions -googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_constant -googlesql/compliance/grouping_sets_queries grouping_sets_with_subquery_from_the_table -googlesql/compliance/grouping_sets_queries grouping_sets_with_correlated_subquery -googlesql/compliance/grouping_sets_queries rollup_with_subquery googlesql/compliance/grouping_sets_queries cube_with_multi_columns_grouping_set googlesql/compliance/grouping_sets_queries cube_with_multi_columns_grouping_set_no_grouping -googlesql/compliance/grouping_sets_queries cube_with_subquery googlesql/compliance/grouping_sets_queries grouping_func_in_having_clause googlesql/compliance/grouping_sets_queries grouping_func_in_qualify_clause googlesql/compliance/hints reject_unknown_hints_in diff --git a/testdata/specs/zigquery/syntax/query/group_by.zon b/testdata/specs/zigquery/syntax/query/group_by.zon index cff90cf..d19fb2a 100644 --- a/testdata/specs/zigquery/syntax/query/group_by.zon +++ b/testdata/specs/zigquery/syntax/query/group_by.zon @@ -87,6 +87,122 @@ .{ 789, 3, 1.99 }, }, }, + // A grouping set yields one row per group whether or not the select list reads its + // columns. The rows are five (sku, day) groups, three sku groups and the grand total. + // query-syntax.md#group_by_rollup + .{ + .desc = "rollup selecting none of its grouping columns", + .sql = + \\WITH Sales AS ( + \\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL + \\ SELECT 123, 1, 8.99 UNION ALL + \\ SELECT 456, 1, 4.56 UNION ALL + \\ SELECT 123, 2, 9.99 UNION ALL + \\ SELECT 789, 3, 1.00 UNION ALL + \\ SELECT 456, 3, 4.25 UNION ALL + \\ SELECT 789, 3, 0.99 + \\) + \\SELECT COUNT(*) AS n + \\FROM Sales + \\GROUP BY ROLLUP(sku, day) + , + .unordered = true, + .rows = .{ + .{ 2 }, + .{ 1 }, + .{ 1 }, + .{ 1 }, + .{ 2 }, + .{ 3 }, + .{ 2 }, + .{ 2 }, + .{ 7 }, + }, + }, + // day is NULL in the rows of the sets that roll it up, which are the three sku groups and + // the grand total. + // query-syntax.md#group_by_rollup + .{ + .desc = "rollup selecting only its second grouping column", + .sql = + \\WITH Sales AS ( + \\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL + \\ SELECT 123, 1, 8.99 UNION ALL + \\ SELECT 456, 1, 4.56 UNION ALL + \\ SELECT 123, 2, 9.99 UNION ALL + \\ SELECT 789, 3, 1.00 UNION ALL + \\ SELECT 456, 3, 4.25 UNION ALL + \\ SELECT 789, 3, 0.99 + \\) + \\SELECT day, COUNT(*) AS n + \\FROM Sales + \\GROUP BY ROLLUP(sku, day) + , + .unordered = true, + .rows = .{ + .{ 1, 2 }, + .{ 2, 1 }, + .{ 1, 1 }, + .{ 3, 1 }, + .{ 3, 2 }, + .{ null, 3 }, + .{ null, 2 }, + .{ null, 2 }, + .{ null, 7 }, + }, + }, + // One row per group of each set, three skus and three days, each carrying the constant. + // query-syntax.md#group_by_grouping_sets + .{ + .desc = "grouping sets with a constant select list", + .sql = + \\WITH Sales AS ( + \\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL + \\ SELECT 123, 1, 8.99 UNION ALL + \\ SELECT 456, 1, 4.56 UNION ALL + \\ SELECT 123, 2, 9.99 UNION ALL + \\ SELECT 789, 3, 1.00 UNION ALL + \\ SELECT 456, 3, 4.25 UNION ALL + \\ SELECT 789, 3, 0.99 + \\) + \\SELECT 'x' AS c + \\FROM Sales + \\GROUP BY GROUPING SETS(sku, day) + , + .unordered = true, + .rows = .{ + .{ "x" }, + .{ "x" }, + .{ "x" }, + .{ "x" }, + .{ "x" }, + .{ "x" }, + }, + }, + // One row per sku, though no column of the aggregation is read. + .{ + .desc = "group by with a constant select list", + .sql = + \\WITH Sales AS ( + \\ SELECT 123 AS sku, 1 AS day, 9.99 AS price UNION ALL + \\ SELECT 123, 1, 8.99 UNION ALL + \\ SELECT 456, 1, 4.56 UNION ALL + \\ SELECT 123, 2, 9.99 UNION ALL + \\ SELECT 789, 3, 1.00 UNION ALL + \\ SELECT 456, 3, 4.25 UNION ALL + \\ SELECT 789, 3, 0.99 + \\) + \\SELECT 'x' AS c + \\FROM Sales + \\GROUP BY sku + , + .unordered = true, + .rows = .{ + .{ "x" }, + .{ "x" }, + .{ "x" }, + }, + }, .{ .desc = "group by having", .sql =