Skip to content

[improvement](tablet scheduler) fix higher priority tablet add failed due to pending queue full #41076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 5 commits into from
Sep 25, 2024
Merged

[improvement](tablet scheduler) fix higher priority tablet add failed due to pending queue full #41076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a311232
improve sched add tablet
yujun777 Sep 20, 2024
5229a39
update
yujun777 Sep 20, 2024
45c3da5
update
yujun777 Sep 23, 2024
3dd7f82
update test
yujun777 Sep 23, 2024
df5eb69
add exceed limit counter log
yujun777 Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,15 +561,17 @@ private void matchGroups() {
tabletCtx.setIsUniqKeyMergeOnWrite(isUniqKeyMergeOnWrite);

AddResult res = tabletScheduler.addTablet(tabletCtx, false /* not force */);
if (res == AddResult.LIMIT_EXCEED || res == AddResult.DISABLED) {
if (res == AddResult.DISABLED) {
// tablet in scheduler exceed limit, or scheduler is disabled,
// skip this group and check next one.
LOG.info("tablet scheduler return: {}. stop colocate table check", res.name());
break OUT;
} else if (res == AddResult.ADDED) {
counter.addToSchedulerTabletNum++;
} else {
} else if (res == AddResult.ALREADY_IN) {
counter.tabletInScheduler++;
} else if (res == AddResult.REPLACE_ADDED || res == AddResult.LIMIT_EXCEED) {
counter.tabletExceedLimit++;
}
}
}
Expand All @@ -589,9 +591,10 @@ private void matchGroups() {
} // end for groups

long cost = System.currentTimeMillis() - start;
LOG.info("finished to check tablets. unhealth/total/added/in_sched/not_ready: {}/{}/{}/{}/{}, cost: {} ms",
LOG.info("finished to check tablets. unhealth/total/added/in_sched/not_ready/exceed_limit: {}/{}/{}/{}/{}/{}, "
+ "cost: {} ms",
counter.unhealthyTabletNum, counter.totalTabletNum, counter.addToSchedulerTabletNum,
counter.tabletInScheduler, counter.tabletNotReady, cost);
counter.tabletInScheduler, counter.tabletNotReady, counter.tabletExceedLimit, cost);
}

private GlobalColocateStatistic buildGlobalColocateStatistic() {
Expand Down
12 changes: 9 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ public class TabletChecker extends MasterDaemon {
put("added", new AtomicLong(0L));
put("in_sched", new AtomicLong(0L));
put("not_ready", new AtomicLong(0L));
put("exceed_limit", new AtomicLong(0L));
}
};

Expand Down Expand Up @@ -224,6 +225,7 @@ public static class CheckerCounter {
public long addToSchedulerTabletNum = 0;
public long tabletInScheduler = 0;
public long tabletNotReady = 0;
public long tabletExceedLimit = 0;
}

private enum LoopControlStatus {
Expand Down Expand Up @@ -344,10 +346,12 @@ private void checkTablets() {
tabletCountByStatus.get("added").set(counter.addToSchedulerTabletNum);
tabletCountByStatus.get("in_sched").set(counter.tabletInScheduler);
tabletCountByStatus.get("not_ready").set(counter.tabletNotReady);
tabletCountByStatus.get("exceed_limit").set(counter.tabletExceedLimit);

LOG.info("finished to check tablets. unhealth/total/added/in_sched/not_ready: {}/{}/{}/{}/{}, cost: {} ms",
LOG.info("finished to check tablets. unhealth/total/added/in_sched/not_ready/exceed_limit: {}/{}/{}/{}/{}/{},"
+ "cost: {} ms",
counter.unhealthyTabletNum, counter.totalTabletNum, counter.addToSchedulerTabletNum,
counter.tabletInScheduler, counter.tabletNotReady, cost);
counter.tabletInScheduler, counter.tabletNotReady, counter.tabletExceedLimit, cost);
}

private LoopControlStatus handlePartitionTablet(Database db, OlapTable tbl, Partition partition, boolean isInPrios,
Expand Down Expand Up @@ -404,11 +408,13 @@ private LoopControlStatus handlePartitionTablet(Database db, OlapTable tbl, Part
tabletCtx.setIsUniqKeyMergeOnWrite(isUniqKeyMergeOnWrite);

AddResult res = tabletScheduler.addTablet(tabletCtx, false /* not force */);
if (res == AddResult.LIMIT_EXCEED || res == AddResult.DISABLED) {
if (res == AddResult.DISABLED) {
LOG.info("tablet scheduler return: {}. stop tablet checker", res.name());
return LoopControlStatus.BREAK_OUT;
} else if (res == AddResult.ADDED) {
counter.addToSchedulerTabletNum++;
} else if (res == AddResult.REPLACE_ADDED || res == AddResult.LIMIT_EXCEED) {
counter.tabletExceedLimit++;
}
}
} // indices
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1334,13 +1334,13 @@ private long getCompareValue() {
if (tabletHealth.aliveAndVersionCompleteNum < (replicaNum / 2 + 1)) {
value -= 3 * baseTime;
if (tabletHealth.hasRecentLoadFailed) {
value -= 3 * baseTime;
value -= 4 * baseTime;
}
}
if (tabletHealth.hasAliveAndVersionIncomplete) {
value -= 1 * baseTime;
if (isUniqKeyMergeOnWrite) {
value -= 1 * baseTime;
value -= 2 * baseTime;
}
}
}
Expand Down
33 changes: 25 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.MinMaxPriorityQueue;
import com.google.common.collect.Sets;
import com.google.common.collect.Table;
import org.apache.logging.log4j.LogManager;
Expand All @@ -81,7 +82,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Set;

Expand Down Expand Up @@ -120,7 +120,7 @@ public class TabletScheduler extends MasterDaemon {
*
* pendingTablets, allTabletTypes, runningTablets and schedHistory are protected by 'synchronized'
*/
private PriorityQueue<TabletSchedCtx> pendingTablets = new PriorityQueue<>();
private MinMaxPriorityQueue<TabletSchedCtx> pendingTablets = MinMaxPriorityQueue.create();
private Map<Long, TabletSchedCtx.Type> allTabletTypes = Maps.newHashMap();
// contains all tabletCtxs which state are RUNNING
private Map<Long, TabletSchedCtx> runningTablets = Maps.newHashMap();
Expand Down Expand Up @@ -149,6 +149,7 @@ public enum AddResult {
ADDED, // success to add
ALREADY_IN, // already added, skip
LIMIT_EXCEED, // number of pending tablets exceed the limit
REPLACE_ADDED, // succ to add, and envit a lowest task
DISABLED // scheduler has been disabled.
}

Expand Down Expand Up @@ -274,12 +275,22 @@ public synchronized AddResult addTablet(TabletSchedCtx tablet, boolean force) {
return AddResult.ALREADY_IN;
}

AddResult addResult = AddResult.ADDED;
// if this is not a force add,
// and number of scheduling tablets exceed the limit,
// refuse to add.
if (!force && (pendingTablets.size() > Config.max_scheduling_tablets
|| runningTablets.size() > Config.max_scheduling_tablets)) {
return AddResult.LIMIT_EXCEED;
if (!force && (pendingTablets.size() >= Config.max_scheduling_tablets
|| runningTablets.size() >= Config.max_scheduling_tablets)) {
// For a sched tablet, if its compare value is bigger, it will be more close to queue's tail position,
// and its priority is lower.
TabletSchedCtx lowestPriorityTablet = pendingTablets.peekLast();
if (lowestPriorityTablet == null || lowestPriorityTablet.compareTo(tablet) <= 0) {
Copy link
Contributor

@deardeng deardeng Sep 23, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add comment

Copy link
Contributor Author

@yujun777 yujun777 Sep 23, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added

return AddResult.LIMIT_EXCEED;
}
addResult = AddResult.REPLACE_ADDED;
pendingTablets.pollLast();
finalizeTabletCtx(lowestPriorityTablet, TabletSchedCtx.State.CANCELLED, Status.UNRECOVERABLE,
"envit lower priority sched tablet because pending queue is full");
}

if (!contains || tablet.getType() == TabletSchedCtx.Type.REPAIR) {
Expand All @@ -291,7 +302,7 @@ public synchronized AddResult addTablet(TabletSchedCtx tablet, boolean force) {
LOG.info("Add tablet to pending queue, {}", tablet);
}

return AddResult.ADDED;
return addResult;
}


Expand All @@ -312,11 +323,12 @@ public synchronized void cancelRebalanceDisk(AdminCancelRebalanceDiskStmt stmt)
* Iterate current tablets, change their priority to VERY_HIGH if necessary.
*/
public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long tblId, List<Long> partitionIds) {
PriorityQueue<TabletSchedCtx> newPendingTablets = new PriorityQueue<>();
MinMaxPriorityQueue<TabletSchedCtx> newPendingTablets = MinMaxPriorityQueue.create();
for (TabletSchedCtx tabletCtx : pendingTablets) {
if (tabletCtx.getDbId() == dbId && tabletCtx.getTblId() == tblId
&& partitionIds.contains(tabletCtx.getPartitionId())) {
tabletCtx.setPriority(Priority.VERY_HIGH);
tabletCtx.setLastVisitedTime(1L);
}
newPendingTablets.add(tabletCtx);
}
Expand Down Expand Up @@ -1750,7 +1762,7 @@ private synchronized List<TabletSchedCtx> getNextTabletCtxBatch() {
slotNum = 1;
}
while (list.size() < Config.schedule_batch_size && slotNum > 0) {
TabletSchedCtx tablet = pendingTablets.poll();
TabletSchedCtx tablet = pendingTablets.pollFirst();
if (tablet == null) {
// no more tablets
break;
Expand Down Expand Up @@ -1952,6 +1964,11 @@ public void handleRunningTablets() {
});
}

// only use for fe ut
public MinMaxPriorityQueue<TabletSchedCtx> getPendingTabletQueue() {
return pendingTablets;
}

public List<List<String>> getPendingTabletsInfo(int limit) {
return collectTabletCtx(getPendingTablets(limit));
}
Expand Down
36 changes: 34 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/clone/TabletSchedCtxTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,57 @@

package org.apache.doris.clone;

import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.clone.TabletSchedCtx.Priority;
import org.apache.doris.clone.TabletSchedCtx.Type;
import org.apache.doris.common.Config;

import com.google.common.collect.Lists;
import com.google.common.collect.MinMaxPriorityQueue;
import org.junit.Assert;
import org.junit.Test;

import java.util.Collections;
import java.util.List;
import java.util.PriorityQueue;

public class TabletSchedCtxTest {

@Test
public void testAddTablet() {
List<TabletSchedCtx> tablets = Lists.newArrayList();
ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
for (long i = 0; i < 20; i++) {
tablets.add(new TabletSchedCtx(Type.REPAIR, 1, 2, 3, 4,
i, replicaAlloc, i));
tablets.add(new TabletSchedCtx(Type.BALANCE, 1, 2, 3, 4,
1000 + i, replicaAlloc, i));
}
Collections.shuffle(tablets);
Config.max_scheduling_tablets = 5;
TabletScheduler scheduler = Env.getCurrentEnv().getTabletScheduler();
for (TabletSchedCtx tablet : tablets) {
scheduler.addTablet(tablet, false);
}

MinMaxPriorityQueue<TabletSchedCtx> queue = scheduler.getPendingTabletQueue();
List<TabletSchedCtx> gotTablets = Lists.newArrayList();
while (!queue.isEmpty()) {
gotTablets.add(queue.pollFirst());
}
Assert.assertEquals(Config.max_scheduling_tablets, gotTablets.size());
for (int i = 0; i < gotTablets.size(); i++) {
TabletSchedCtx tablet = gotTablets.get(i);
Assert.assertEquals(Type.REPAIR, tablet.getType());
Assert.assertEquals((long) i, tablet.getCreateTime());
}
}

@Test
public void testPriorityCompare() {
// equal priority, but info3's last visit time is earlier than info2 and info1, so info1 should ranks ahead
PriorityQueue<TabletSchedCtx> pendingTablets = new PriorityQueue<>();
MinMaxPriorityQueue<TabletSchedCtx> pendingTablets = MinMaxPriorityQueue.create();
ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
TabletSchedCtx ctx1 = new TabletSchedCtx(Type.REPAIR,
1, 2, 3, 4, 1000, replicaAlloc, System.currentTimeMillis());
Expand Down