Skip to content

[fix](DatabaseTransactionMgr) fix clean label bug which may cause inconsitent editlog operation #29198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
caiconghui merged 1 commit into from
Dec 28, 2023
Merged

[fix](DatabaseTransactionMgr) fix clean label bug which may cause inconsitent editlog operation #29198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 3 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,22 +786,16 @@ private void cleanLabelInternal(long dbId, String label, boolean isReplay) {
} finally {
writeUnlock();
}
LOG.info("clean {} labels on db {} with label '{}' in load mgr.", counter, dbId, label);

// 2. Remove from DatabaseTransactionMgr
try {
Env.getCurrentGlobalTransactionMgr().cleanLabel(dbId, label);
Env.getCurrentGlobalTransactionMgr().cleanLabel(dbId, label, isReplay);
} catch (AnalysisException e) {
// just ignore, because we don't want to throw any exception here.
LOG.warn("Exception:", e);
}

// 3. Log
if (!isReplay) {
CleanLabelOperationLog log = new CleanLabelOperationLog(dbId, label);
Env.getCurrentEnv().getEditLog().logCleanLabel(log);
}
LOG.info("finished to clean label on db {} with label {}. is replay: {}", dbId, label, isReplay);
LOG.info("finished to clean {} labels on db {} with label '{}' in load mgr. is replay: {}",
counter, dbId, label, isReplay);
}

private void readLock() {
Expand Down
10 changes: 7 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.BatchRemoveTransactionsOperationV2;
import org.apache.doris.persist.CleanLabelOperationLog;
import org.apache.doris.persist.EditLog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.AnalysisManager;
Expand Down Expand Up @@ -2072,7 +2073,7 @@ public void unprotectWriteAllTransactionStates(DataOutput out) throws IOExceptio
}
}

protected void cleanLabel(String label) {
protected void cleanLabel(String label, boolean isReplay) {
Set<Long> removedTxnIds = Sets.newHashSet();
writeLock();
try {
Expand Down Expand Up @@ -2113,11 +2114,14 @@ protected void cleanLabel(String label) {
// So that we can keep consistency in meta image
finalStatusTransactionStateDequeShort.removeIf(txn -> removedTxnIds.contains(txn.getTransactionId()));
finalStatusTransactionStateDequeLong.removeIf(txn -> removedTxnIds.contains(txn.getTransactionId()));

if (!isReplay) {
CleanLabelOperationLog log = new CleanLabelOperationLog(dbId, label);
Env.getCurrentEnv().getEditLog().logCleanLabel(log);
}
} finally {
writeUnlock();
}
LOG.info("clean {} labels on db {} with label '{}' in database transaction mgr.", removedTxnIds.size(), dbId,
label);
}

public long getTxnNumByStatus(TransactionStatus status) {
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,8 +754,8 @@ public List<TransactionState> getPreCommittedTxnList(Long dbId) throws AnalysisE
return getDatabaseTransactionMgr(dbId).getPreCommittedTxnList();
}

public void cleanLabel(Long dbId, String label) throws AnalysisException {
getDatabaseTransactionMgr(dbId).cleanLabel(label);
public void cleanLabel(Long dbId, String label, boolean isReplay) throws AnalysisException {
getDatabaseTransactionMgr(dbId).cleanLabel(label, isReplay);
}

public Long getTransactionIdByLabel(Long dbId, String label, List<TransactionStatus> statusList)
Expand Down