Skip to content

Support Alter Table Clause For External Table #4699

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 7 commits into from
Oct 25, 2020
Merged

Support Alter Table Clause For External Table #4699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1959144
Support Alter Table Clause For External Table
HappenLee Oct 6, 2020
95b0943
fix some problem
HappenLee Oct 7, 2020
fb59c88
change some code
HappenLee Oct 9, 2020
609e85d
change OP_REFRESH_EXTERNAL_TABLE_SCHEMA to OP_ALTER_EXTERNAL_TABLE_SC…
HappenLee Oct 10, 2020
673ab75
Added a forbidden of delete the last column
HappenLee Oct 12, 2020
8b6daa1
fix some mistake
HappenLee Oct 18, 2020
5a3ee73
use GJSON to do Serialize
HappenLee Oct 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 110 additions & 68 deletions fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,92 @@ public void processDropMaterializedView(DropMaterializedViewStmt stmt) throws Dd
}
}

private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, List<AlterClause> alterClauses,
final String clusterName, Database db) throws UserException {
stmt.rewriteAlterClause(olapTable);

// check conflict alter ops first
alterClauses.addAll(stmt.getOps());
AlterOperations currentAlterOps = new AlterOperations();
currentAlterOps.checkConflict(alterClauses);

// check cluster capacity and db quota, only need to check once.
if (currentAlterOps.needCheckCapacity()) {
Catalog.getCurrentSystemInfo().checkClusterCapacity(clusterName);
db.checkQuota();
}

if (olapTable.getState() != OlapTableState.NORMAL) {
throw new DdlException(
"Table[" + olapTable.getName() + "]'s state is not NORMAL. Do not allow doing ALTER ops");
}

boolean needProcessOutsideDatabaseLock = false;
if (currentAlterOps.hasSchemaChangeOp()) {
// if modify storage type to v2, do schema change to convert all related tablets to segment v2 format
schemaChangeHandler.process(alterClauses, clusterName, db, olapTable);
} else if (currentAlterOps.hasRollupOp()) {
materializedViewHandler.process(alterClauses, clusterName, db, olapTable);
} else if (currentAlterOps.hasPartitionOp()) {
Preconditions.checkState(alterClauses.size() == 1);
AlterClause alterClause = alterClauses.get(0);
if (alterClause instanceof DropPartitionClause) {
if (!((DropPartitionClause) alterClause).isTempPartition()) {
DynamicPartitionUtil.checkAlterAllowed((OlapTable) db.getTable(olapTable.getName()));
}
Catalog.getCurrentCatalog().dropPartition(db, olapTable, ((DropPartitionClause) alterClause));
} else if (alterClause instanceof ReplacePartitionClause) {
Catalog.getCurrentCatalog().replaceTempPartition(db, olapTable.getName(), (ReplacePartitionClause) alterClause);
} else if (alterClause instanceof ModifyPartitionClause) {
ModifyPartitionClause clause = ((ModifyPartitionClause) alterClause);
// expand the partition names if it is 'Modify Partition(*)'
if (clause.isNeedExpand()) {
List<String> partitionNames = clause.getPartitionNames();
partitionNames.clear();
for (Partition partition : olapTable.getPartitions()) {
partitionNames.add(partition.getName());
}
}
Map<String, String> properties = clause.getProperties();
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY)) {
needProcessOutsideDatabaseLock = true;
} else {
List<String> partitionNames = clause.getPartitionNames();
modifyPartitionsProperty(db, olapTable, partitionNames, properties);
}
} else if (alterClause instanceof AddPartitionClause) {
needProcessOutsideDatabaseLock = true;
} else {
throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
}
} else if (currentAlterOps.hasRenameOp()) {
processRename(db, olapTable, alterClauses);
} else if (currentAlterOps.hasReplaceTableOp()) {
processReplaceTable(db, olapTable, alterClauses);
} else if (currentAlterOps.contains(AlterOpType.MODIFY_TABLE_PROPERTY_SYNC)) {
needProcessOutsideDatabaseLock = true;
} else {
throw new DdlException("Invalid alter operations: " + currentAlterOps);
}

return needProcessOutsideDatabaseLock;
}

private void processAlterExternalTable(AlterTableStmt stmt, Table externalTable, Database db) throws UserException {
stmt.rewriteAlterClause(externalTable);

// check conflict alter ops first
List<AlterClause> alterClauses = stmt.getOps();
AlterOperations currentAlterOps = new AlterOperations();
currentAlterOps.checkConflict(alterClauses);

if (currentAlterOps.hasRenameOp()) {
processRename(db, externalTable, alterClauses);
} else if (currentAlterOps.hasSchemaChangeOp()) {
schemaChangeHandler.processExternalTable(alterClauses, db, externalTable);
}
}

public void processAlterTable(AlterTableStmt stmt) throws UserException {
TableName dbTableName = stmt.getTbl();
String dbName = dbTableName.getDb();
Expand All @@ -163,7 +249,7 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException {
if (db == null) {
ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName);
}
List<AlterClause> alterClauses;
List<AlterClause> alterClauses = Lists.newArrayList();

// some operations will take long time to process, need to be done outside the database lock
boolean needProcessOutsideDatabaseLock = false;
Expand All @@ -175,73 +261,18 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException {
ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName);
}

if (table.getType() != TableType.OLAP) {
throw new DdlException("Do not support alter non-OLAP table[" + tableName + "]");
}
OlapTable olapTable = (OlapTable) table;
stmt.rewriteAlterClause(olapTable);

// check conflict alter ops first
alterClauses = stmt.getOps();
AlterOperations currentAlterOps = new AlterOperations();
currentAlterOps.checkConflict(alterClauses);

// check cluster capacity and db quota, only need to check once.
if (currentAlterOps.needCheckCapacity()) {
Catalog.getCurrentSystemInfo().checkClusterCapacity(clusterName);
db.checkQuota();
}

if (olapTable.getState() != OlapTableState.NORMAL) {
throw new DdlException(
"Table[" + table.getName() + "]'s state is not NORMAL. Do not allow doing ALTER ops");
}

if (currentAlterOps.hasSchemaChangeOp()) {
// if modify storage type to v2, do schema change to convert all related tablets to segment v2 format
schemaChangeHandler.process(alterClauses, clusterName, db, olapTable);
} else if (currentAlterOps.hasRollupOp()) {
materializedViewHandler.process(alterClauses, clusterName, db, olapTable);
} else if (currentAlterOps.hasPartitionOp()) {
Preconditions.checkState(alterClauses.size() == 1);
AlterClause alterClause = alterClauses.get(0);
if (alterClause instanceof DropPartitionClause) {
if (!((DropPartitionClause) alterClause).isTempPartition()) {
DynamicPartitionUtil.checkAlterAllowed((OlapTable) db.getTable(tableName));
}
Catalog.getCurrentCatalog().dropPartition(db, olapTable, ((DropPartitionClause) alterClause));
} else if (alterClause instanceof ReplacePartitionClause) {
Catalog.getCurrentCatalog().replaceTempPartition(db, tableName, (ReplacePartitionClause) alterClause);
} else if (alterClause instanceof ModifyPartitionClause) {
ModifyPartitionClause clause = ((ModifyPartitionClause) alterClause);
// expand the partition names if it is 'Modify Partition(*)'
if (clause.isNeedExpand()) {
List<String> partitionNames = clause.getPartitionNames();
partitionNames.clear();
for (Partition partition : olapTable.getPartitions()) {
partitionNames.add(partition.getName());
}
}
Map<String, String> properties = clause.getProperties();
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY)) {
needProcessOutsideDatabaseLock = true;
} else {
List<String> partitionNames = clause.getPartitionNames();
modifyPartitionsProperty(db, olapTable, partitionNames, properties);
}
} else if (alterClause instanceof AddPartitionClause) {
needProcessOutsideDatabaseLock = true;
} else {
throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
}
} else if (currentAlterOps.hasRenameOp()) {
processRename(db, olapTable, alterClauses);
} else if (currentAlterOps.hasReplaceTableOp()) {
processReplaceTable(db, olapTable, alterClauses);
} else if (currentAlterOps.contains(AlterOpType.MODIFY_TABLE_PROPERTY_SYNC)) {
needProcessOutsideDatabaseLock = true;
} else {
throw new DdlException("Invalid alter operations: " + currentAlterOps);
switch (table.getType()) {
case OLAP:
OlapTable olapTable = (OlapTable) table;
needProcessOutsideDatabaseLock = processAlterOlapTable(stmt, olapTable, alterClauses, clusterName, db);
break;
case ODBC:
case MYSQL:
case ELASTICSEARCH:
processAlterExternalTable(stmt, table, db);
return;
default:
throw new DdlException("Do not support alter " + table.getType().toString() + " table[" + tableName + "]");
}
} finally {
db.writeUnlock();
Expand Down Expand Up @@ -462,6 +493,17 @@ private void processRename(Database db, OlapTable table, List<AlterClause> alter
}
}

private void processRename(Database db, Table table, List<AlterClause> alterClauses) throws DdlException {
Copy link
Member

@wuyunfeng wuyunfeng Oct 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
private void processRename(Database db, Table table, List<AlterClause> alterClauses) throws DdlException {
private void processRenameAlterOperation(Database db, Table table, List<AlterClause> alterClauses) throws DdlException {

Copy link
Contributor Author

@HappenLee HappenLee Oct 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@wuyunfeng hi，yunfeng. I just want keep the function name same to the olaptable alter function name.

for (AlterClause alterClause : alterClauses) {
if (alterClause instanceof TableRenameClause) {
Catalog.getCurrentCatalog().renameTable(db, table, (TableRenameClause) alterClause);
break;
} else {
Preconditions.checkState(false);
}
}
}

/**
* Batch update partitions' properties
* caller should hold the db lock
Expand Down
7 changes: 7 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
Expand Down Expand Up @@ -380,6 +381,12 @@ public void start() {
public abstract void process(List<AlterClause> alterClauses, String clusterName, Database db, OlapTable olapTable)
throws UserException;

/*
* entry function. handle alter ops for external table
*/
public void processExternalTable(List<AlterClause> alterClauses, Database db, Table externalTable)
throws UserException {};

/*
* cancel alter ops
*/
Expand Down
Loading