-
Notifications
You must be signed in to change notification settings - Fork 96
feat(core): add MaskExpression POJO and ReadRel projection support #782
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
nielspardon
merged 17 commits into
substrait-io:main
from
flex-team:feat/add-mask-expression
Apr 10, 2026
Merged
Changes from all commits
Commits
Show all changes
17 commits
Select commit
Hold shift + click to select a range
6b12f87
feat(core): add MaskExpression POJO and projection support for ReadRel
flex-seongmin 8bd8942
test(core): add MaskExpression roundtrip tests for nested list, map, …
flex-seongmin b723737
chore(core): format code for spotlessJavaApply lint
flex-seongmin ab49ad6
fix(core): apply MaskExpression projection to ReadRel deriveRecordType
flex-seongmin 2888f2d
chore(core): apply lint
flex-seongmin 821a424
refactor(core): PR: replace MaskExpression Select union with visitor …
flex-seongmin ecfd05f
refactor(core): PR: split MaskExpressionProtoConverter into toProto a…
flex-seongmin 2330204
refactor(core): PR: merge MaskExpr into MaskExpression interface
flex-seongmin deb0469
docs(core): add javadoc to MaskExpression proto converter classes
flex-seongmin b010578
refactor(core): use MaskExpressionVisitor for Select dispatch and cle…
flex-seongmin a285952
docs(core): add javadoc to MaskExpression
flex-seongmin 70aa512
docs(core): rewrite javadoc
flex-seongmin 4ca34b7
chore(core): renaming argument of projectStruct function
flex-seongmin 05a38f2
chroe(core): renaming argument of project function and add javadoc
flex-seongmin 3adb196
refactor(core): remove MaskExpression.Mask by combining @Value.Immuta…
flex-seongmin dd9cac0
docs(core): add javadoc return to MaskExpression interface
flex-seongmin 375704c
docs(core): lint javadoc
flex-seongmin File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
422 changes: 422 additions & 0 deletions
422
core/src/main/java/io/substrait/expression/MaskExpression.java
Large diffs are not rendered by default.
Oops, something went wrong.
133 changes: 133 additions & 0 deletions
133
core/src/main/java/io/substrait/expression/MaskExpressionTypeProjector.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| package io.substrait.expression; | ||
|
|
||
| import io.substrait.type.Type; | ||
| import io.substrait.type.TypeCreator; | ||
| import io.substrait.util.EmptyVisitationContext; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * Applies a {@link MaskExpression} projection to a {@link Type.Struct}, returning a pruned struct. | ||
| */ | ||
| public final class MaskExpressionTypeProjector { | ||
|
|
||
| private MaskExpressionTypeProjector() {} | ||
|
|
||
| /** | ||
| * Applies the given projection to a struct type, returning a pruned struct. | ||
| * | ||
| * @param projection the mask expression projection | ||
| * @param structType the struct type to project | ||
| * @return a pruned struct containing only the selected fields | ||
| */ | ||
| public static Type.Struct project(MaskExpression projection, Type.Struct structType) { | ||
| return projectStruct(projection.getSelect(), structType); | ||
| } | ||
|
|
||
| private static Type.Struct projectStruct( | ||
| MaskExpression.StructSelect structSelect, Type.Struct structType) { | ||
| List<Type> fields = structType.fields(); | ||
| List<MaskExpression.StructItem> items = structSelect.getStructItems(); | ||
|
|
||
| return TypeCreator.of(structType.nullable()) | ||
| .struct(items.stream().map(item -> projectItem(item, fields.get(item.getField())))); | ||
| } | ||
|
|
||
| private static Type projectItem(MaskExpression.StructItem item, Type fieldType) { | ||
| if (!item.getChild().isPresent()) { | ||
| return fieldType; | ||
| } | ||
|
|
||
| MaskExpression.Select select = item.getChild().get(); | ||
|
|
||
| return select.accept( | ||
| new MaskExpressionVisitor<Type, EmptyVisitationContext, RuntimeException>() { | ||
| @Override | ||
| public Type visit( | ||
| MaskExpression.StructSelect structSelect, EmptyVisitationContext context) { | ||
| return projectStruct(structSelect, (Type.Struct) fieldType); | ||
| } | ||
|
|
||
| @Override | ||
| public Type visit(MaskExpression.ListSelect listSelect, EmptyVisitationContext context) { | ||
| return projectList(listSelect, (Type.ListType) fieldType); | ||
| } | ||
|
|
||
| @Override | ||
| public Type visit(MaskExpression.MapSelect mapSelect, EmptyVisitationContext context) { | ||
| return projectMap(mapSelect, (Type.Map) fieldType); | ||
| } | ||
| }, | ||
| EmptyVisitationContext.INSTANCE); | ||
| } | ||
|
|
||
| private static Type.ListType projectList( | ||
| MaskExpression.ListSelect listSelect, Type.ListType listType) { | ||
| if (!listSelect.getChild().isPresent()) { | ||
| return listType; | ||
| } | ||
|
|
||
| MaskExpression.Select childSelect = listSelect.getChild().get(); | ||
| Type elementType = listType.elementType(); | ||
|
|
||
| return childSelect.accept( | ||
| new MaskExpressionVisitor<Type.ListType, EmptyVisitationContext, RuntimeException>() { | ||
| @Override | ||
| public Type.ListType visit( | ||
| MaskExpression.StructSelect structSelect, EmptyVisitationContext context) { | ||
| if (elementType instanceof Type.Struct) { | ||
| Type.Struct prunedElement = projectStruct(structSelect, (Type.Struct) elementType); | ||
| return TypeCreator.of(listType.nullable()).list(prunedElement); | ||
| } | ||
| return listType; | ||
| } | ||
|
|
||
| @Override | ||
| public Type.ListType visit( | ||
| MaskExpression.ListSelect listSelect, EmptyVisitationContext context) { | ||
| return listType; | ||
| } | ||
|
|
||
| @Override | ||
| public Type.ListType visit( | ||
| MaskExpression.MapSelect mapSelect, EmptyVisitationContext context) { | ||
| return listType; | ||
| } | ||
| }, | ||
| EmptyVisitationContext.INSTANCE); | ||
| } | ||
|
|
||
| private static Type.Map projectMap(MaskExpression.MapSelect mapSelect, Type.Map mapType) { | ||
| if (!mapSelect.getChild().isPresent()) { | ||
| return mapType; | ||
| } | ||
|
|
||
| MaskExpression.Select childSelect = mapSelect.getChild().get(); | ||
| Type valueType = mapType.value(); | ||
|
|
||
| return childSelect.accept( | ||
| new MaskExpressionVisitor<Type.Map, EmptyVisitationContext, RuntimeException>() { | ||
| @Override | ||
| public Type.Map visit( | ||
| MaskExpression.StructSelect structSelect, EmptyVisitationContext context) { | ||
| if (valueType instanceof Type.Struct) { | ||
| Type.Struct prunedValue = projectStruct(structSelect, (Type.Struct) valueType); | ||
| return TypeCreator.of(mapType.nullable()).map(mapType.key(), prunedValue); | ||
| } | ||
| return mapType; | ||
| } | ||
|
|
||
| @Override | ||
| public Type.Map visit( | ||
| MaskExpression.ListSelect listSelect, EmptyVisitationContext context) { | ||
| return mapType; | ||
| } | ||
|
|
||
| @Override | ||
| public Type.Map visit( | ||
| MaskExpression.MapSelect mapSelect, EmptyVisitationContext context) { | ||
| return mapType; | ||
| } | ||
| }, | ||
| EmptyVisitationContext.INSTANCE); | ||
| } | ||
| } |
43 changes: 43 additions & 0 deletions
43
core/src/main/java/io/substrait/expression/MaskExpressionVisitor.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package io.substrait.expression; | ||
|
|
||
| import io.substrait.util.VisitationContext; | ||
|
|
||
| /** | ||
| * Visitor for {@link MaskExpression} select nodes. | ||
| * | ||
| * @param <R> result type returned by each visit | ||
| * @param <C> visitation context type | ||
| * @param <E> throwable type that visit methods may throw | ||
| */ | ||
| public interface MaskExpressionVisitor<R, C extends VisitationContext, E extends Throwable> { | ||
|
|
||
| /** | ||
| * Visit a struct select. | ||
| * | ||
| * @param structSelect the struct select | ||
| * @param context visitation context | ||
| * @return visit result | ||
| * @throws E on visit failure | ||
| */ | ||
| R visit(MaskExpression.StructSelect structSelect, C context) throws E; | ||
|
|
||
| /** | ||
| * Visit a list select. | ||
| * | ||
| * @param listSelect the list select | ||
| * @param context visitation context | ||
| * @return visit result | ||
| * @throws E on visit failure | ||
| */ | ||
| R visit(MaskExpression.ListSelect listSelect, C context) throws E; | ||
|
|
||
| /** | ||
| * Visit a map select. | ||
| * | ||
| * @param mapSelect the map select | ||
| * @param context visitation context | ||
| * @return visit result | ||
| * @throws E on visit failure | ||
| */ | ||
| R visit(MaskExpression.MapSelect mapSelect, C context) throws E; | ||
| } |
137 changes: 137 additions & 0 deletions
137
core/src/main/java/io/substrait/expression/proto/MaskExpressionProtoConverter.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| package io.substrait.expression.proto; | ||
|
|
||
| import io.substrait.expression.MaskExpression; | ||
| import io.substrait.expression.MaskExpression.ListSelect; | ||
| import io.substrait.expression.MaskExpression.ListSelectItem; | ||
| import io.substrait.expression.MaskExpression.MapSelect; | ||
| import io.substrait.expression.MaskExpression.Select; | ||
| import io.substrait.expression.MaskExpression.StructItem; | ||
| import io.substrait.expression.MaskExpression.StructSelect; | ||
| import io.substrait.expression.MaskExpressionVisitor; | ||
| import io.substrait.proto.Expression; | ||
| import io.substrait.util.EmptyVisitationContext; | ||
|
|
||
| /** | ||
| * Converts from {@link io.substrait.expression.MaskExpression} to {@link Expression.MaskExpression} | ||
| */ | ||
| public final class MaskExpressionProtoConverter { | ||
|
|
||
| private MaskExpressionProtoConverter() {} | ||
|
|
||
| private static final MaskExpressionVisitor< | ||
| Expression.MaskExpression.Select, EmptyVisitationContext, RuntimeException> | ||
| SELECT_TO_PROTO_VISITOR = | ||
| new MaskExpressionVisitor< | ||
| Expression.MaskExpression.Select, EmptyVisitationContext, RuntimeException>() { | ||
| @Override | ||
| public Expression.MaskExpression.Select visit( | ||
| MaskExpression.StructSelect structSelect, EmptyVisitationContext context) { | ||
| return Expression.MaskExpression.Select.newBuilder() | ||
| .setStruct(toProto(structSelect)) | ||
| .build(); | ||
| } | ||
|
|
||
| @Override | ||
| public Expression.MaskExpression.Select visit( | ||
| MaskExpression.ListSelect listSelect, EmptyVisitationContext context) { | ||
| return Expression.MaskExpression.Select.newBuilder() | ||
| .setList(toProtoListSelect(listSelect)) | ||
| .build(); | ||
| } | ||
|
|
||
| @Override | ||
| public Expression.MaskExpression.Select visit( | ||
| MaskExpression.MapSelect mapSelect, EmptyVisitationContext context) { | ||
| return Expression.MaskExpression.Select.newBuilder() | ||
| .setMap(toProtoMapSelect(mapSelect)) | ||
| .build(); | ||
| } | ||
| }; | ||
|
|
||
| /** | ||
| * Converts a POJO {@link MaskExpression} to its proto representation. | ||
| * | ||
| * @param mask the POJO {@link MaskExpression} | ||
| * @return the proto {@link Expression.MaskExpression} | ||
| */ | ||
| public static Expression.MaskExpression toProto(MaskExpression mask) { | ||
| return Expression.MaskExpression.newBuilder() | ||
| .setSelect(toProto(mask.getSelect())) | ||
| .setMaintainSingularStruct(mask.getMaintainSingularStruct()) | ||
| .build(); | ||
| } | ||
nielspardon marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| private static Expression.MaskExpression.StructSelect toProto(StructSelect structSelect) { | ||
| Expression.MaskExpression.StructSelect.Builder builder = | ||
| Expression.MaskExpression.StructSelect.newBuilder(); | ||
| for (StructItem item : structSelect.getStructItems()) { | ||
| builder.addStructItems(toProto(item)); | ||
| } | ||
| return builder.build(); | ||
| } | ||
|
|
||
| private static Expression.MaskExpression.StructItem toProto(StructItem structItem) { | ||
| Expression.MaskExpression.StructItem.Builder builder = | ||
| Expression.MaskExpression.StructItem.newBuilder().setField(structItem.getField()); | ||
| structItem.getChild().ifPresent(child -> builder.setChild(toProtoSelect(child))); | ||
| return builder.build(); | ||
| } | ||
|
|
||
| private static Expression.MaskExpression.Select toProtoSelect(Select select) { | ||
| return select.accept(SELECT_TO_PROTO_VISITOR, EmptyVisitationContext.INSTANCE); | ||
| } | ||
|
|
||
| private static Expression.MaskExpression.ListSelect toProtoListSelect(ListSelect listSelect) { | ||
| Expression.MaskExpression.ListSelect.Builder builder = | ||
| Expression.MaskExpression.ListSelect.newBuilder(); | ||
| for (ListSelectItem item : listSelect.getSelection()) { | ||
| builder.addSelection(toProtoListSelectItem(item)); | ||
| } | ||
| listSelect.getChild().ifPresent(child -> builder.setChild(toProtoSelect(child))); | ||
| return builder.build(); | ||
| } | ||
|
|
||
| private static Expression.MaskExpression.ListSelect.ListSelectItem toProtoListSelectItem( | ||
| ListSelectItem item) { | ||
| Expression.MaskExpression.ListSelect.ListSelectItem.Builder builder = | ||
| Expression.MaskExpression.ListSelect.ListSelectItem.newBuilder(); | ||
| if (item.getItem().isPresent()) { | ||
| builder.setItem( | ||
| Expression.MaskExpression.ListSelect.ListSelectItem.ListElement.newBuilder() | ||
| .setField(item.getItem().get().getField()) | ||
| .build()); | ||
| } else if (item.getSlice().isPresent()) { | ||
| builder.setSlice( | ||
| Expression.MaskExpression.ListSelect.ListSelectItem.ListSlice.newBuilder() | ||
| .setStart(item.getSlice().get().getStart()) | ||
| .setEnd(item.getSlice().get().getEnd()) | ||
| .build()); | ||
| } else { | ||
| throw new IllegalArgumentException("ListSelectItem must have either item or slice set"); | ||
| } | ||
| return builder.build(); | ||
| } | ||
|
|
||
| private static Expression.MaskExpression.MapSelect toProtoMapSelect(MapSelect mapSelect) { | ||
| Expression.MaskExpression.MapSelect.Builder builder = | ||
| Expression.MaskExpression.MapSelect.newBuilder(); | ||
| mapSelect | ||
| .getKey() | ||
| .ifPresent( | ||
| key -> | ||
| builder.setKey( | ||
| Expression.MaskExpression.MapSelect.MapKey.newBuilder() | ||
| .setMapKey(key.getMapKey()) | ||
| .build())); | ||
| mapSelect | ||
| .getExpression() | ||
| .ifPresent( | ||
| expr -> | ||
| builder.setExpression( | ||
| Expression.MaskExpression.MapSelect.MapKeyExpression.newBuilder() | ||
| .setMapKeyExpression(expr.getMapKeyExpression()) | ||
| .build())); | ||
| mapSelect.getChild().ifPresent(child -> builder.setChild(toProtoSelect(child))); | ||
| return builder.build(); | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.