diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverBroadcastDistributionRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverBroadcastDistributionRule.java index ef5094cbea4a..3fc808450ebf 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverBroadcastDistributionRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverBroadcastDistributionRule.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -56,6 +57,12 @@ public boolean appliesTo(Interval interval, DateTime referenceTimestamp) return true; } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return Intervals.ETERNITY; + } + @Override public boolean equals(Object o) { @@ -75,4 +82,10 @@ public int hashCode() { return Objects.hash(getType()); } + + @Override + public String toString() + { + return "ForeverBroadcastDistributionRule{}"; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverDropRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverDropRule.java index 19479c015b91..7c5b54bf8c2d 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverDropRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverDropRule.java @@ -20,6 +20,7 @@ package org.apache.druid.server.coordinator.rules; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -46,4 +47,16 @@ public boolean appliesTo(Interval interval, DateTime referenceTimestamp) { return true; } + + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return Intervals.ETERNITY; + } + + @Override + public String toString() + { + return "ForeverDropRule{}"; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverLoadRule.java index 35f22fa555f8..55a9a992a00f 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverLoadRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverLoadRule.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -60,4 +61,18 @@ public boolean appliesTo(Interval interval, DateTime referenceTimestamp) return true; } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return Intervals.ETERNITY; + } + + @Override + public String toString() + { + return "ForeverLoadRule{" + + "tieredReplicants=" + getTieredReplicants() + + ", useDefaultTierForNull=" + useDefaultTierForNull() + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalBroadcastDistributionRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalBroadcastDistributionRule.java index b1bf29eedd20..96e7123630ad 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalBroadcastDistributionRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalBroadcastDistributionRule.java @@ -65,6 +65,12 @@ public Interval getInterval() return interval; } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return interval; + } + @Override public boolean equals(Object o) { @@ -83,4 +89,12 @@ public int hashCode() { return Objects.hash(getInterval()); } + + @Override + public String toString() + { + return "IntervalBroadcastDistributionRule{" + + "interval=" + interval + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalDropRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalDropRule.java index b2959e925d28..6ed11d49bf60 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalDropRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalDropRule.java @@ -66,6 +66,12 @@ public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) return interval.contains(theInterval); } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return interval; + } + @Override public boolean equals(Object o) { @@ -84,4 +90,12 @@ public int hashCode() { return Objects.hash(interval); } + + @Override + public String toString() + { + return "IntervalDropRule{" + + "interval=" + interval + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalLoadRule.java index 209f5c24d1e4..c66675c75fa4 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalLoadRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalLoadRule.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -34,8 +33,6 @@ */ public class IntervalLoadRule extends LoadRule { - private static final Logger log = new Logger(IntervalLoadRule.class); - private final Interval interval; @JsonCreator @@ -74,6 +71,12 @@ public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) return Rules.eligibleForLoad(interval, theInterval); } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return interval; + } + @Override public boolean equals(Object o) { @@ -95,4 +98,14 @@ public int hashCode() { return Objects.hash(super.hashCode(), interval); } + + @Override + public String toString() + { + return "IntervalLoadRule{" + + "interval=" + interval + + ", tieredReplicants=" + getTieredReplicants() + + ", useDefaultTierForNull=" + useDefaultTierForNull() + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodBroadcastDistributionRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodBroadcastDistributionRule.java index d48353d3e50a..17bbf7be789b 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodBroadcastDistributionRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodBroadcastDistributionRule.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -65,6 +67,13 @@ public boolean appliesTo(Interval interval, DateTime referenceTimestamp) return Rules.eligibleForLoad(period, interval, referenceTimestamp, includeFuture); } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return includeFuture ? new Interval(referenceTimestamp.minus(period), DateTimes.utc(JodaUtils.MAX_INSTANT)) + : new Interval(referenceTimestamp.minus(period), referenceTimestamp); + } + @JsonProperty public Period getPeriod() { @@ -96,4 +105,13 @@ public int hashCode() { return Objects.hash(getPeriod(), isIncludeFuture()); } + + @Override + public String toString() + { + return "PeriodBroadcastDistributionRule{" + + "period=" + period + + ", includeFuture=" + includeFuture + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropBeforeRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropBeforeRule.java index 6654b385eac3..7198e5af7006 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropBeforeRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropBeforeRule.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -63,4 +64,25 @@ public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) final DateTime periodAgo = referenceTimestamp.minus(period); return theInterval.getEndMillis() <= periodAgo.getMillis(); } + + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + final DateTime end = referenceTimestamp.minus(period); + if (end.isBefore(DateTimes.MIN)) { + // We use Long.MIN_VALUE as the start here (instead of DateTimes.MIN) when end is < DateTimes.MIN because the + // resulting interval will be invalid where start > end. This is true for referenceTimestamp = DateTimes.MIN. + return new Interval(DateTimes.utc(Long.MIN_VALUE), end); + } else { + return new Interval(DateTimes.MIN, end); + } + } + + @Override + public String toString() + { + return "PeriodDropBeforeRule{" + + "period=" + period + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropRule.java index da17b4c2a9d9..e92e55a3c9a8 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodDropRule.java @@ -21,12 +21,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; /** + * */ public class PeriodDropRule extends DropRule { @@ -80,4 +82,20 @@ public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) return currInterval.contains(theInterval); } } + + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return includeFuture ? new Interval(referenceTimestamp.minus(period), DateTimes.MAX) + : new Interval(referenceTimestamp.minus(period), referenceTimestamp); + } + + @Override + public String toString() + { + return "PeriodDropRule{" + + "period=" + period + + ", includeFuture=" + includeFuture + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodLoadRule.java index 1d2b4e187716..b799a664eb45 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodLoadRule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodLoadRule.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -35,7 +35,6 @@ */ public class PeriodLoadRule extends LoadRule { - private static final Logger log = new Logger(PeriodLoadRule.class); static final boolean DEFAULT_INCLUDE_FUTURE = true; private final Period period; @@ -85,6 +84,13 @@ public boolean appliesTo(Interval interval, DateTime referenceTimestamp) return Rules.eligibleForLoad(period, interval, referenceTimestamp, includeFuture); } + @Override + public Interval getEligibleInterval(DateTime referenceTimestamp) + { + return includeFuture ? new Interval(referenceTimestamp.minus(period), DateTimes.MAX) + : new Interval(referenceTimestamp.minus(period), referenceTimestamp); + } + @Override public boolean equals(Object o) { @@ -106,4 +112,15 @@ public int hashCode() { return Objects.hash(super.hashCode(), period, includeFuture); } + + @Override + public String toString() + { + return "PeriodLoadRule{" + + "period=" + period + + ", includeFuture=" + includeFuture + + ", tieredReplicants=" + getTieredReplicants() + + ", useDefaultTierForNull=" + useDefaultTierForNull() + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java index a66101d0fe14..dfb03102fe51 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java @@ -50,4 +50,11 @@ public interface Rule boolean appliesTo(Interval interval, DateTime referenceTimestamp); void run(DataSegment segment, SegmentActionHandler segmentHandler); + + /** + * Returns the interval eligible for this rule. The interval must be computed based on the rule type + * optionally using {@code referenceTimestamp}. {@code referenceTimestamp} must be a {@link DateTime} + * between [{@link org.apache.druid.java.util.common.DateTimes#MIN}, {@link org.apache.druid.java.util.common.DateTimes#MAX}]. + */ + Interval getEligibleInterval(DateTime referenceTimestamp); } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/Rules.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/Rules.java index 9cb6094886c7..5dc714d4f451 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/Rules.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/Rules.java @@ -19,10 +19,15 @@ package org.apache.druid.server.coordinator.rules; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; +import java.util.List; + public class Rules { public static boolean eligibleForLoad(Interval src, Interval target) @@ -43,4 +48,57 @@ public static boolean eligibleForLoad(Period period, Interval interval, DateTime private Rules() { } + + /** + * Validates the given list of retention rules for a datasource (or cluster default). + * A rule is considered valid only if it will be evaluated at some point, i.e. its eligible interval + * is not fully covered by the eligible interval of any preceding rule in the list. + *

+ * Consider two rules r1 and r2. Assume r1 and r2's eligible intervals at the time when the rules are evaluated are + * i1 and i2, respectively. r1 and r2 are invalid if: + *

+ *

+ * @throws org.apache.druid.error.DruidException with error code "invalidInput" if any of the given rules is not valid. + */ + public static void validateRules(final List rules) + { + if (rules == null) { + return; + } + + final DateTime now = DateTimes.nowUtc(); + for (int i = 0; i < rules.size(); i++) { + final Rule currRule = rules.get(i); + final Interval currInterval = currRule.getEligibleInterval(now); + + for (int j = i + 1; j < rules.size(); j++) { + final Rule nextRule = rules.get(j); + final Interval nextInterval = nextRule.getEligibleInterval(now); + if (currInterval.contains(nextInterval)) { + if (Intervals.ETERNITY.equals(currInterval) || + (currRule.getEligibleInterval(currInterval.getStart()) + .contains(nextRule.getEligibleInterval(currInterval.getStart())) + && currRule.getEligibleInterval(currInterval.getEnd()) + .contains(nextRule.getEligibleInterval(currInterval.getEnd())))) { + throw InvalidInput.exception( + "Rule[%s] has an interval that fully contains the interval for rule[%s]." + + " i.e., interval[%s] hides interval[%s]. Please fix the rules and retry.", + currRule, + nextRule, + currInterval, + nextInterval + ); + } + } + } + } + } } diff --git a/server/src/main/java/org/apache/druid/server/http/RulesResource.java b/server/src/main/java/org/apache/druid/server/http/RulesResource.java index beb223a6cc65..99d7d5d1caf4 100644 --- a/server/src/main/java/org/apache/druid/server/http/RulesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/RulesResource.java @@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.metadata.MetadataRuleManager; import org.apache.druid.server.coordinator.rules.Rule; +import org.apache.druid.server.coordinator.rules.Rules; import org.apache.druid.server.http.security.RulesResourceFilter; import org.apache.druid.server.http.security.StateResourceFilter; import org.joda.time.Interval; @@ -108,6 +109,7 @@ public Response setDatasourceRules( ) { try { + Rules.validateRules(rules); final AuditInfo auditInfo = new AuditInfo(author, comment, req.getRemoteAddr()); if (databaseRuleManager.overrideRule(dataSourceName, rules, auditInfo)) { return Response.ok().build(); @@ -180,5 +182,4 @@ private List getRuleHistory( } return auditManager.fetchAuditHistory(AUDIT_HISTORY_TYPE, theInterval); } - } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/RulesEligibleIntervalTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/RulesEligibleIntervalTest.java new file mode 100644 index 000000000000..83fafbde0f34 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/RulesEligibleIntervalTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.JodaUtils; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class RulesEligibleIntervalTest +{ + private final DateTime referenceTime; + + @Parameterized.Parameters + public static Object[] getReferenceTimestamps() + { + final DateTime now = DateTimes.nowUtc(); + return new Object[]{ + now, + now.minusYears(5), + now.plusYears(10), + DateTimes.utc(0), + DateTimes.utc(JodaUtils.MIN_INSTANT + 1), + DateTimes.utc(JodaUtils.MIN_INSTANT), + DateTimes.utc(JodaUtils.MAX_INSTANT - 1), + DateTimes.utc(JodaUtils.MAX_INSTANT) + }; + } + + public RulesEligibleIntervalTest(final DateTime referenceTimestamp) + { + this.referenceTime = referenceTimestamp; + } + + @Test + public void testPeriodLoadEligibleInterval() + { + final Period period = new Period("P50Y"); + final PeriodLoadRule loadPT1H = new PeriodLoadRule( + period, + true, + null, + null + ); + Assert.assertEquals( + new Interval(this.referenceTime.minus(period), DateTimes.MAX), + loadPT1H.getEligibleInterval(this.referenceTime) + ); + } + + @Test + public void testPeriodLoadExcludingFutureEligibleInterval() + { + final Period period = new Period("PT1H"); + final PeriodLoadRule rule = new PeriodLoadRule( + period, + false, + null, + null + ); + Assert.assertEquals(new Interval(period, this.referenceTime), rule.getEligibleInterval(this.referenceTime)); + } + + @Test + public void testIntervalLoadEligibleInterval() + { + final Interval interval = Intervals.of("2000/3000"); + final IntervalLoadRule rule = new IntervalLoadRule( + interval, + null, + null + ); + Assert.assertEquals(interval, rule.getEligibleInterval(this.referenceTime)); + } + + @Test + public void testForeverLoadEligibleInterval() + { + final ForeverLoadRule rule = new ForeverLoadRule(ImmutableMap.of(), false); + Assert.assertEquals(Intervals.ETERNITY, rule.getEligibleInterval(this.referenceTime)); + } + + @Test + public void testPeriodDropEligibleInterval() + { + final Period period = new Period("P5000Y"); + final PeriodDropRule rule = new PeriodDropRule( + period, + true + ); + Assert.assertEquals( + new Interval(this.referenceTime.minus(period), DateTimes.utc(JodaUtils.MAX_INSTANT)), + rule.getEligibleInterval(this.referenceTime) + ); + } + + @Test + public void testPeriodDropExcludingFutureEligibleInterval() + { + final Period period = new Period("P50Y"); + final PeriodDropRule rule = new PeriodDropRule( + period, + false + ); + Assert.assertEquals( + new Interval(this.referenceTime.minus(period), this.referenceTime), + rule.getEligibleInterval(this.referenceTime) + ); + } + + @Test + public void testPeriodDropBeforeEligibleInterval() + { + final Period period = new Period("P50Y"); + final PeriodDropBeforeRule rule = new PeriodDropBeforeRule(period); + + if (this.referenceTime.minus(period).isBefore(DateTimes.MIN)) { + Assert.assertEquals( + new Interval(DateTimes.utc(Long.MIN_VALUE), this.referenceTime.minus(period)), + rule.getEligibleInterval(this.referenceTime) + ); + } else { + Assert.assertEquals( + new Interval(DateTimes.MIN, this.referenceTime.minus(period)), + rule.getEligibleInterval(this.referenceTime) + ); + } + } + + @Test + public void testForeverDropEligibleInterval() + { + final ForeverDropRule rule = new ForeverDropRule(); + Assert.assertEquals(Intervals.ETERNITY, rule.getEligibleInterval(this.referenceTime)); + } + + @Test + public void testPeriodBroadcastEligibleInterval() + { + final Period period = new Period("P15Y"); + final PeriodBroadcastDistributionRule rule = new PeriodBroadcastDistributionRule(period, true); + Assert.assertEquals( + new Interval(referenceTime.minus(period), DateTimes.MAX), + rule.getEligibleInterval(referenceTime) + ); + } + + @Test + public void testPeriodBroadcastExcludingFutureEligibleInterval() + { + final Period period = new Period("P15Y"); + final PeriodBroadcastDistributionRule rule = new PeriodBroadcastDistributionRule(period, false); + Assert.assertEquals(new Interval(period, this.referenceTime), rule.getEligibleInterval(this.referenceTime)); + } + + @Test + public void testForeverBroadcastEligibleInterval() + { + final ForeverBroadcastDistributionRule rule = new ForeverBroadcastDistributionRule(); + Assert.assertEquals(Intervals.ETERNITY, rule.getEligibleInterval(this.referenceTime)); + } + + @Test + public void testIntervalBroadcastEligibleInterval() + { + final Interval interval = Intervals.of("1993/2070"); + final IntervalBroadcastDistributionRule rule = new IntervalBroadcastDistributionRule(interval); + Assert.assertEquals(interval, rule.getEligibleInterval(this.referenceTime)); + } +} diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/RulesTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/RulesTest.java new file mode 100644 index 000000000000..b6bde979a302 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/RulesTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; +import org.joda.time.Period; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(Parameterized.class) +public class RulesTest +{ + // Load rules + private static final PeriodLoadRule LOAD_PT1H = new PeriodLoadRule( + new Period("PT1H"), true, null, null + ); + private static final PeriodLoadRule LOAD_PT1H_EXLUDE_FUTURE = new PeriodLoadRule( + new Period("PT1H"), false, null, null + ); + private static final PeriodLoadRule LOAD_P3M = new PeriodLoadRule( + new Period("P3M"), true, null, null + ); + private static final IntervalLoadRule LOAD_2020_2023 = new IntervalLoadRule( + Intervals.of("2020/2023"), null, null + ); + private static final IntervalLoadRule LOAD_2021_2022 = new IntervalLoadRule( + Intervals.of("2021/2022"), null, null + ); + private static final IntervalLoadRule LOAD_1980_2050 = new IntervalLoadRule( + Intervals.of("1980/2050"), null, null + ); + private static final ForeverLoadRule LOAD_FOREVER = new ForeverLoadRule(null, null); + + // Drop rules + private static final PeriodDropBeforeRule DROP_BEFORE_P3M = new PeriodDropBeforeRule(new Period("P3M")); + private static final PeriodDropBeforeRule DROP_BEFORE_P6M = new PeriodDropBeforeRule(new Period("P6M")); + private static final PeriodDropRule DROP_P1M = new PeriodDropRule(new Period("P1M"), true); + private static final PeriodDropRule DROP_P2M = new PeriodDropRule(new Period("P2M"), true); + private static final IntervalDropRule DROP_2000_2020 = new IntervalDropRule(Intervals.of("2000/2020")); + private static final IntervalDropRule DROP_2010_2020 = new IntervalDropRule(Intervals.of("2010/2020")); + private static final ForeverDropRule DROP_FOREVER = new ForeverDropRule(); + + // Broadcast rules + private static final PeriodBroadcastDistributionRule BROADCAST_PT1H = new PeriodBroadcastDistributionRule( + new Period("PT1H"), true + ); + private static final PeriodBroadcastDistributionRule BROADCAST_PT1H_EXCLUDE_FUTURE = new PeriodBroadcastDistributionRule( + new Period("PT1H"), false + ); + private static final PeriodBroadcastDistributionRule BROADCAST_PT2H = new PeriodBroadcastDistributionRule( + new Period("PT2H"), true + ); + private static final IntervalBroadcastDistributionRule BROADCAST_2000_2050 = new IntervalBroadcastDistributionRule( + Intervals.of("2000/2050") + ); + private static final IntervalBroadcastDistributionRule BROADCAST_2010_2020 = new IntervalBroadcastDistributionRule( + Intervals.of("2010/2020") + ); + private static final ForeverBroadcastDistributionRule BROADCAST_FOREVER = new ForeverBroadcastDistributionRule(); + + private final List rules; + private final boolean isInvalid; + private final Rule invalidRule1; + private final Rule invalidRule2; + + public RulesTest(final List rules, final boolean isInvalid, final Rule invalidRule1, final Rule invalidRule2) + { + this.rules = rules; + this.isInvalid = isInvalid; + this.invalidRule1 = invalidRule1; + this.invalidRule2 = invalidRule2; + } + + @Parameterized.Parameters + public static Object[] inputsAndExpectations() + { + return new Object[][] { + // Invalid rules + {getRules(LOAD_PT1H, LOAD_2021_2022, LOAD_FOREVER, LOAD_P3M), true, LOAD_FOREVER, LOAD_P3M}, + {getRules(LOAD_PT1H, LOAD_P3M, LOAD_PT1H_EXLUDE_FUTURE, LOAD_1980_2050), true, LOAD_PT1H, LOAD_PT1H_EXLUDE_FUTURE}, + {getRules(LOAD_PT1H, LOAD_P3M, LOAD_2020_2023, LOAD_P3M), true, LOAD_P3M, LOAD_P3M}, + {getRules(LOAD_2020_2023, LOAD_2021_2022, LOAD_P3M, LOAD_FOREVER), true, LOAD_2020_2023, LOAD_2021_2022}, + {getRules(LOAD_P3M, LOAD_2021_2022, LOAD_PT1H, LOAD_2020_2023), true, LOAD_P3M, LOAD_PT1H}, + {getRules(LOAD_P3M, LOAD_2021_2022, LOAD_FOREVER, LOAD_2020_2023), true, LOAD_FOREVER, LOAD_2020_2023}, + {getRules(DROP_BEFORE_P3M, DROP_P1M, DROP_BEFORE_P6M, DROP_FOREVER), true, DROP_BEFORE_P3M, DROP_BEFORE_P6M}, + {getRules(DROP_P2M, DROP_P1M, DROP_FOREVER), true, DROP_P2M, DROP_P1M}, + {getRules(DROP_2000_2020, DROP_P1M, DROP_P2M, DROP_2010_2020), true, DROP_2000_2020, DROP_2010_2020}, + {getRules(DROP_P1M, DROP_FOREVER, DROP_P2M), true, DROP_FOREVER, DROP_P2M}, + {getRules(BROADCAST_2000_2050, BROADCAST_PT1H, BROADCAST_2010_2020, BROADCAST_FOREVER), true, BROADCAST_2000_2050, BROADCAST_2010_2020}, + {getRules(BROADCAST_PT2H, BROADCAST_2000_2050, BROADCAST_PT1H, BROADCAST_FOREVER), true, BROADCAST_PT2H, BROADCAST_PT1H}, + {getRules(BROADCAST_PT1H, BROADCAST_PT1H_EXCLUDE_FUTURE, BROADCAST_FOREVER), true, BROADCAST_PT1H, BROADCAST_PT1H_EXCLUDE_FUTURE}, + {getRules(BROADCAST_PT1H, BROADCAST_PT2H, BROADCAST_2010_2020, BROADCAST_FOREVER, BROADCAST_2000_2050), true, BROADCAST_FOREVER, BROADCAST_2000_2050}, + {getRules(LOAD_PT1H, LOAD_1980_2050, LOAD_P3M, LOAD_FOREVER, DROP_FOREVER), true, LOAD_FOREVER, DROP_FOREVER}, + + // Valid rules + {null, false, null, null}, + {getRules(), false, null, null}, + {getRules(LOAD_FOREVER), false, null, null}, + {getRules(LOAD_PT1H_EXLUDE_FUTURE, LOAD_PT1H, LOAD_P3M, LOAD_FOREVER), false, null, null}, + {getRules(DROP_2010_2020, DROP_2000_2020, DROP_P1M, DROP_P2M, DROP_BEFORE_P3M, DROP_FOREVER), false, null, null}, + {getRules(BROADCAST_PT1H, BROADCAST_PT2H, BROADCAST_2010_2020, BROADCAST_2000_2050, BROADCAST_FOREVER), false, null, null}, + {getRules(DROP_BEFORE_P6M, DROP_P1M, DROP_BEFORE_P3M, LOAD_2020_2023, DROP_FOREVER), false, null, null}, + {getRules(DROP_2000_2020, LOAD_PT1H, BROADCAST_2000_2050, LOAD_1980_2050, LOAD_P3M, LOAD_FOREVER), false, null, null}, + }; + } + + private static ArrayList getRules(Rule... rules) + { + return new ArrayList<>(Arrays.asList(rules)); + } + + @Test + public void testValidateRules() + { + if (this.isInvalid) { + DruidExceptionMatcher.invalidInput().expectMessageContains( + StringUtils.format("Rule[%s] has an interval that fully contains the interval for rule[%s].", + invalidRule1, invalidRule2 + ) + ).assertThrowsAndMatches(() -> Rules.validateRules(rules)); + } else { + Rules.validateRules(rules); + } + } +} diff --git a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java index 5b52e3d30ee9..16e394ccc3c3 100644 --- a/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/RulesResourceTest.java @@ -23,16 +23,28 @@ import org.apache.druid.audit.AuditEntry; import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; +import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.MetadataRuleManager; +import org.apache.druid.server.coordinator.rules.ForeverLoadRule; +import org.apache.druid.server.coordinator.rules.IntervalDropRule; +import org.apache.druid.server.coordinator.rules.PeriodBroadcastDistributionRule; +import org.apache.druid.server.coordinator.rules.PeriodDropBeforeRule; +import org.apache.druid.server.coordinator.rules.PeriodDropRule; +import org.apache.druid.server.coordinator.rules.PeriodLoadRule; +import org.apache.druid.server.coordinator.rules.Rule; import org.easymock.EasyMock; import org.joda.time.Interval; +import org.joda.time.Period; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -41,6 +53,17 @@ public class RulesResourceTest private MetadataRuleManager databaseRuleManager; private AuditManager auditManager; + private static final PeriodLoadRule LOAD_P3M = new PeriodLoadRule( + new Period("P3M"), true, null, null + ); + private final ForeverLoadRule LOAD_FOREVER = new ForeverLoadRule(null, null); + private static final PeriodDropRule DROP_P2M = new PeriodDropRule(new Period("P2M"), true); + private static final PeriodDropBeforeRule DROP_BEFORE_P6M = new PeriodDropBeforeRule(new Period("P6M")); + private static final IntervalDropRule DROP_2010_2020 = new IntervalDropRule(Intervals.of("2010/2020")); + private static final PeriodBroadcastDistributionRule BROADCAST_PT2H = new PeriodBroadcastDistributionRule( + new Period("PT2H"), true + ); + @Before public void setUp() { @@ -151,6 +174,64 @@ public void testGetDatasourceRuleHistoryWithWrongCount() EasyMock.verify(auditManager); } + @Test + public void testSetDatasourceRulesWithEffectivelyNoRule() + { + EasyMock.expect(databaseRuleManager.overrideRule(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())) + .andReturn(true).times(2); + EasyMock.replay(databaseRuleManager); + + final RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); + final Response resp1 = rulesResource.setDatasourceRules("dataSource1", null, null, null, EasyMock.createMock(HttpServletRequest.class)); + Assert.assertEquals(200, resp1.getStatus()); + + final Response resp2 = rulesResource.setDatasourceRules("dataSource1", new ArrayList<>(), null, null, EasyMock.createMock(HttpServletRequest.class)); + Assert.assertEquals(200, resp2.getStatus()); + EasyMock.verify(databaseRuleManager); + } + + @Test + public void testSetDatasourceRulesWithValidRules() + { + EasyMock.expect(databaseRuleManager.overrideRule(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())) + .andReturn(true).anyTimes(); + EasyMock.replay(databaseRuleManager); + final RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); + + final List rules = new ArrayList<>(); + rules.add(BROADCAST_PT2H); + rules.add(DROP_P2M); + rules.add(DROP_2010_2020); + rules.add(LOAD_P3M); + rules.add(DROP_BEFORE_P6M); + rules.add(LOAD_FOREVER); + + final Response resp = rulesResource.setDatasourceRules("dataSource1", rules, null, null, EasyMock.createMock(HttpServletRequest.class)); + Assert.assertEquals(200, resp.getStatus()); + EasyMock.verify(databaseRuleManager); + } + + @Test + public void testSetDatasourceRulesWithInvalidRules() + { + EasyMock.replay(auditManager); + + final RulesResource rulesResource = new RulesResource(databaseRuleManager, auditManager); + final HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); + + final List rules = new ArrayList<>(); + rules.add(DROP_P2M); + rules.add(LOAD_P3M); + rules.add(DROP_BEFORE_P6M); + rules.add(BROADCAST_PT2H); + rules.add(DROP_2010_2020); + rules.add(LOAD_FOREVER); + + DruidExceptionMatcher.invalidInput().expectMessageContains( + StringUtils.format("Rule[%s] has an interval that fully contains the interval for rule[%s].", DROP_P2M, BROADCAST_PT2H) + ).assertThrowsAndMatches(() -> rulesResource.setDatasourceRules("dataSource1", rules, null, null, req)); + } + @Test public void testGetAllDatasourcesRuleHistoryWithCount() {