Skip to content

[enhance](catalog)Allow parallel running of insert overwrite on the external table #41575

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 2 commits into from
Oct 11, 2024
Merged

[enhance](catalog)Allow parallel running of insert overwrite on the external table #41575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d7a437f
1
zddr Oct 9, 2024
a303fa1
1
zddr Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,14 @@ private boolean rollback(long taskId) {
* @param table Run the table for insert overwrite
*/
public void recordRunningTableOrException(DatabaseIf db, TableIf table) {
// The logic of OlapTable executing insert overwrite is to create temporary partitions,
// replace partitions, etc.
// If executed in parallel, it may cause problems such as not being able to find temporary partitions.
// But in terms of external table, we don't care the internal logic of execution,
// so there's no need to keep records
if (!(table instanceof OlapTable)) {
return;
}
long dbId = db.getId();
long tableId = table.getId();
runningLock.writeLock().lock();
Expand Down
34 changes: 25 additions & 9 deletions fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,27 @@
package org.apache.doris.insertoverwrite;

import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.hive.HMSExternalTable;

import mockit.Expectations;
import mockit.Mocked;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;

public class InsertOverwriteManagerTest {
@Mocked
private DatabaseIf db;

@Mocked
private TableIf table;
private OlapTable table;

@Mocked
private HMSExternalTable hmsExternalTable;

@Before
public void setUp()
Expand All @@ -57,6 +61,14 @@ public void setUp()
table.getName();
minTimes = 0;
result = "table1";

hmsExternalTable.getId();
minTimes = 0;
result = 3L;

hmsExternalTable.getName();
minTimes = 0;
result = "hmsTable";
}
};
}
Expand All @@ -65,13 +77,17 @@ public void setUp()
public void testParallel() {
InsertOverwriteManager manager = new InsertOverwriteManager();
manager.recordRunningTableOrException(db, table);
try {
manager.recordRunningTableOrException(db, table);
} catch (Exception e) {
Assert.assertTrue(e.getMessage().contains("Not allowed"));
}
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
() -> manager.recordRunningTableOrException(db, table));
manager.dropRunningRecord(db.getId(), table.getId());
manager.recordRunningTableOrException(db, table);
Assertions.assertDoesNotThrow(() -> manager.recordRunningTableOrException(db, table));
}

@Test
public void testHmsTableParallel() {
InsertOverwriteManager manager = new InsertOverwriteManager();
manager.recordRunningTableOrException(db, hmsExternalTable);
Assertions.assertDoesNotThrow(() -> manager.recordRunningTableOrException(db, hmsExternalTable));
manager.dropRunningRecord(db.getId(), hmsExternalTable.getId());
}
}