Skip to content

Add getRightEquiConditionKeys to JoinConditionAnalysis #9287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gianm merged 3 commits into from
Jan 30, 2020
Merged

Add getRightEquiConditionKeys to JoinConditionAnalysis #9287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f98720
Add getRightColumns to JoinConditionAnalysis
suneet-s Jan 29, 2020
5b54942
Address some review comments
suneet-s Jan 29, 2020
e70dca1
more code review stuff
suneet-s Jan 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions processing/src/main/java/org/apache/druid/segment/join/JoinConditionAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

/**
* Represents analysis of a join condition.
Expand All @@ -55,6 +57,7 @@ public class JoinConditionAnalysis
private final boolean isAlwaysFalse;
private final boolean isAlwaysTrue;
private final boolean canHashJoin;
private final Set<String> rightKeyColumns;

private JoinConditionAnalysis(
final String originalExpression,
Expand All @@ -76,6 +79,7 @@ private JoinConditionAnalysis(
.allMatch(expr -> expr.isLiteral() && expr.eval(
ExprUtils.nilBindings()).asBoolean());
canHashJoin = nonEquiConditions.stream().allMatch(Expr::isLiteral);
rightKeyColumns = getEquiConditions().stream().map(Equality::getRightColumn).distinct().collect(Collectors.toSet());
}

/**
Expand Down Expand Up @@ -176,6 +180,14 @@ public boolean canHashJoin()
return canHashJoin;
}

/**
* Returns the distinct column keys from the RHS required to evaluate the equi conditions.
*/
public Set<String> getRightEquiConditionKeys()
{
Copy link
Copy Markdown
Contributor

@gianm gianm Jan 29, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if it's clear that this method only applies to the equi-conditions. The javadoc explains it but the method name might be misleading? What do you think?

I don't know if I have a good solution, btw. getRightKeyColumnsForEquiConditions is clear but quite a mouthful. Maybe getRightEquiConditionKeys.

Copy link
Copy Markdown
Contributor

@gianm gianm Jan 29, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Btw, a Set may be more appropriate than a List here.

Copy link
Copy Markdown
Contributor Author

@suneet-s suneet-s Jan 29, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the suggestion. Done.

return rightKeyColumns;
}

@Override
public boolean equals(Object o)
{
Expand Down
4 changes: 2 additions & 2 deletions processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinMatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,9 @@ public static LookupJoinMatcher create(
keyExprs = null;
} else if (!condition.getNonEquiConditions().isEmpty()) {
throw new IAE("Cannot join lookup with non-equi condition: %s", condition);
} else if (!condition.getEquiConditions()
} else if (!condition.getRightEquiConditionKeys()
.stream()
.allMatch(eq -> eq.getRightColumn().equals(LookupColumnSelectorFactory.KEY_COLUMN))) {
.allMatch(LookupColumnSelectorFactory.KEY_COLUMN::equals)) {
throw new IAE("Cannot join lookup with condition referring to non-key column: %s", condition);
} else {
keyExprs = condition.getEquiConditions().stream().map(Equality::getLeftExpr).collect(Collectors.toList());
Expand Down
3 changes: 2 additions & 1 deletion processing/src/main/java/org/apache/druid/segment/join/table/IndexedTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import javax.annotation.Nullable;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* An interface to a table where some columns (the 'key columns') have indexes that enable fast lookups.
Expand All @@ -36,7 +37,7 @@ public interface IndexedTable
/**
* Returns the columns of this table that have indexes.
*/
List<String> keyColumns();
Set<String> keyColumns();

/**
* Returns all columns of this table, including the key and non-key columns.
Expand Down
7 changes: 4 additions & 3 deletions processing/src/main/java/org/apache/druid/segment/join/table/RowBasedIndexedTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand All @@ -48,13 +49,13 @@ public class RowBasedIndexedTable<RowType> implements IndexedTable
private final List<String> columns;
private final List<ValueType> columnTypes;
private final List<Function<RowType, Object>> columnFunctions;
private final List<String> keyColumns;
private final Set<String> keyColumns;

public RowBasedIndexedTable(
final List<RowType> table,
final RowAdapter<RowType> rowAdapter,
final Map<String, ValueType> rowSignature,
final List<String> keyColumns
final Set<String> keyColumns
)
{
this.table = table;
Expand Down Expand Up @@ -107,7 +108,7 @@ public RowBasedIndexedTable(
}

@Override
public List<String> keyColumns()
public Set<String> keyColumns()
{
return keyColumns;
}
Expand Down
16 changes: 14 additions & 2 deletions processing/src/test/java/org/apache/druid/segment/join/JoinConditionAnalysisTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.druid.segment.join;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.Pair;
Expand Down Expand Up @@ -60,6 +61,7 @@ public void test_forExpression_simple()
ImmutableList.of(),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertEquals(analysis.getRightEquiConditionKeys(), ImmutableSet.of("y"));
}

@Test
Expand All @@ -80,6 +82,7 @@ public void test_forExpression_simpleFlipped()
ImmutableList.of(),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertEquals(analysis.getRightEquiConditionKeys(), ImmutableSet.of("y"));
}

@Test
Expand All @@ -100,6 +103,7 @@ public void test_forExpression_leftFunction()
ImmutableList.of(),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertEquals(analysis.getRightEquiConditionKeys(), ImmutableSet.of("z"));
}

@Test
Expand All @@ -120,6 +124,7 @@ public void test_forExpression_rightFunction()
ImmutableList.of("(== (+ j.x j.y) z)"),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertTrue(analysis.getRightEquiConditionKeys().isEmpty());
}

@Test
Expand All @@ -140,6 +145,7 @@ public void test_forExpression_mixedFunction()
ImmutableList.of("(== (+ x j.y) j.z)"),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertTrue(analysis.getRightEquiConditionKeys().isEmpty());
}

@Test
Expand All @@ -160,6 +166,7 @@ public void test_forExpression_trueConstant()
ImmutableList.of("2"),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertTrue(analysis.getRightEquiConditionKeys().isEmpty());
}

@Test
Expand All @@ -180,6 +187,7 @@ public void test_forExpression_falseConstant()
ImmutableList.of("0"),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertTrue(analysis.getRightEquiConditionKeys().isEmpty());
}

@Test
Expand All @@ -200,6 +208,7 @@ public void test_forExpression_onlyLeft()
ImmutableList.of("(== x 1)"),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertTrue(analysis.getRightEquiConditionKeys().isEmpty());
}

@Test
Expand All @@ -220,6 +229,7 @@ public void test_forExpression_onlyRight()
ImmutableList.of(),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertEquals(analysis.getRightEquiConditionKeys(), ImmutableSet.of("x"));
Copy link
Copy Markdown
Contributor

@gianm gianm Jan 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are backwards. (I think putting actual before expected is more natural, but JUnit disagrees and who am I to judge?)

}

@Test
Expand All @@ -240,6 +250,7 @@ public void test_forExpression_andOfThreeConditions()
ImmutableList.of(),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertEquals(analysis.getRightEquiConditionKeys(), ImmutableSet.of("y", "z", "zz"));
}

@Test
Expand All @@ -260,6 +271,7 @@ public void test_forExpression_mixedAndWithOr()
ImmutableList.of("(|| (== (+ x y) j.z) (== z j.zz))"),
exprsToStrings(analysis.getNonEquiConditions())
);
Assert.assertEquals(analysis.getRightEquiConditionKeys(), ImmutableSet.of("y"));
}

@Test
Expand All @@ -270,8 +282,8 @@ public void test_equals()
.withIgnoredFields(
// These fields are tightly coupled with originalExpression
"equiConditions", "nonEquiConditions",
// These fields are calculated from nonEquiConditions
"isAlwaysTrue", "isAlwaysFalse", "canHashJoin")
// These fields are calculated from other other fields in the class
Copy link
Copy Markdown
Contributor

@gianm gianm Jan 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo (other other)

"isAlwaysTrue", "isAlwaysFalse", "canHashJoin", "rightKeyColumns")
Comment thread
suneet-s marked this conversation as resolved.
.verify();
}

Expand Down
5 changes: 3 additions & 2 deletions processing/src/test/java/org/apache/druid/segment/join/JoinTestHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.InputRow;
Expand Down Expand Up @@ -252,7 +253,7 @@ public static RowBasedIndexedTable<Map<String, Object>> createCountriesIndexedTa
rows,
createMapRowAdapter(COUNTRIES_SIGNATURE),
COUNTRIES_SIGNATURE,
ImmutableList.of("countryNumber", "countryIsoCode")
ImmutableSet.of("countryNumber", "countryIsoCode")
)
);
}
Expand All @@ -265,7 +266,7 @@ public static RowBasedIndexedTable<Map<String, Object>> createRegionsIndexedTabl
rows,
createMapRowAdapter(REGIONS_SIGNATURE),
REGIONS_SIGNATURE,
ImmutableList.of("regionIsoCode", "countryIsoCode")
ImmutableSet.of("regionIsoCode", "countryIsoCode")
)
);
}
Expand Down
3 changes: 2 additions & 1 deletion processing/src/test/java/org/apache/druid/segment/join/table/IndexedTableJoinableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.druid.segment.join.table;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.InlineDataSource;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
Expand Down Expand Up @@ -72,7 +73,7 @@ public ColumnCapabilities getColumnCapabilities(String columnName)
inlineDataSource.getRowsAsList(),
inlineDataSource.rowAdapter(),
inlineDataSource.getRowSignature(),
ImmutableList.of("str")
ImmutableSet.of("str")
);

@Test
Expand Down
3 changes: 2 additions & 1 deletion processing/src/test/java/org/apache/druid/segment/join/table/RowBasedIndexedTableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.join.JoinTestHelper;
Expand Down Expand Up @@ -64,7 +65,7 @@ public void setUp() throws IOException
@Test
public void test_keyColumns_countries()
{
Assert.assertEquals(ImmutableList.of("countryNumber", "countryIsoCode"), countriesTable.keyColumns());
Assert.assertEquals(ImmutableSet.of("countryNumber", "countryIsoCode"), countriesTable.keyColumns());
}

@Test
Expand Down
6 changes: 2 additions & 4 deletions server/src/main/java/org/apache/druid/segment/join/InlineJoinableFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,8 @@
import org.apache.druid.segment.join.table.IndexedTableJoinable;
import org.apache.druid.segment.join.table.RowBasedIndexedTable;

import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.Set;

/**
* A {@link JoinableFactory} for {@link InlineDataSource}. It works by building an {@link IndexedTable}.
Expand All @@ -39,8 +38,7 @@ public Optional<Joinable> build(final DataSource dataSource, final JoinCondition
{
if (condition.canHashJoin() && dataSource instanceof InlineDataSource) {
final InlineDataSource inlineDataSource = (InlineDataSource) dataSource;
final List<String> rightKeyColumns =
condition.getEquiConditions().stream().map(Equality::getRightColumn).distinct().collect(Collectors.toList());
final Set<String> rightKeyColumns = condition.getRightEquiConditionKeys();

return Optional.of(
new IndexedTableJoinable(
Expand Down