From 079731654f620808a4a3dddf3fdf62776ab2841c Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 9 Jul 2019 14:19:24 +0800 Subject: [PATCH 01/79] first commit --- .../org/apache/doris/alter/AlterHandler.java | 23 +- .../org/apache/doris/alter/AlterJobV2.java | 197 ++++++ .../org/apache/doris/alter/RollupHandler.java | 230 +++---- .../org/apache/doris/alter/RollupJobV2.java | 643 ++++++++++++++++++ .../doris/analysis/AddRollupClause.java | 10 + .../doris/catalog/MaterializedIndex.java | 3 +- .../org/apache/doris/catalog/Partition.java | 2 + .../doris/common/util/PropertyAnalyzer.java | 16 + .../apache/doris/journal/JournalEntity.java | 6 + .../org/apache/doris/persist/EditLog.java | 27 +- .../apache/doris/persist/OperationType.java | 1 + .../org/apache/doris/task/AgentBatchTask.java | 33 + .../java/org/apache/doris/task/AgentTask.java | 12 + .../org/apache/doris/task/AgentTaskQueue.java | 14 + .../apache/doris/task/CreateReplicaTask.java | 17 +- .../apache/doris/task/CreateRollupTaskV2.java | 80 +++ .../org/apache/doris/alter/RollupJobTest.java | 6 +- 17 files changed, 1176 insertions(+), 144 deletions(-) create mode 100644 fe/src/main/java/org/apache/doris/alter/AlterJobV2.java create mode 100644 fe/src/main/java/org/apache/doris/alter/RollupJobV2.java create mode 100644 fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 06849f41e6ef4b..e8af5df79eebab 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -45,10 +45,15 @@ public abstract class AlterHandler extends Daemon { private static final Logger LOG = LogManager.getLogger(AlterHandler.class); // tableId -> AlterJob + @Deprecated protected ConcurrentHashMap alterJobs = new ConcurrentHashMap(); - + @Deprecated protected ConcurrentLinkedQueue finishedOrCancelledAlterJobs = new ConcurrentLinkedQueue(); + // jobId -> AlterJobV2 + protected ConcurrentHashMap alterJobsV2 = new ConcurrentHashMap(); + protected ConcurrentLinkedQueue finishedOrCancelledAlterJobsV2 = new ConcurrentLinkedQueue(); + /* * lock to perform atomic operations. * eg. @@ -70,19 +75,28 @@ public AlterHandler(String name) { super(name, 10000); } + protected void addAlterJobV2(AlterJobV2 alterJob) { + this.alterJobsV2.put(alterJob.getJobId(), alterJob); + LOG.info("add {} job {}", alterJob.getType(), alterJob.getJobId()); + } + + @Deprecated protected void addAlterJob(AlterJob alterJob) { this.alterJobs.put(alterJob.getTableId(), alterJob); LOG.info("add {} job[{}]", alterJob.getType(), alterJob.getTableId()); } + @Deprecated public AlterJob getAlterJob(long tableId) { return this.alterJobs.get(tableId); } + @Deprecated public boolean hasUnfinishedAlterJob(long tableId) { return this.alterJobs.containsKey(tableId); } + @Deprecated public int getAlterJobNum(JobState state, long dbId) { int jobNum = 0; if (state == JobState.PENDING || state == JobState.RUNNING || state == JobState.FINISHING) { @@ -121,24 +135,29 @@ public int getAlterJobNum(JobState state, long dbId) { return jobNum; } + @Deprecated public Map unprotectedGetAlterJobs() { return this.alterJobs; } + @Deprecated public ConcurrentLinkedQueue unprotectedGetFinishedOrCancelledAlterJobs() { return this.finishedOrCancelledAlterJobs; } + @Deprecated public void addFinishedOrCancelledAlterJob(AlterJob alterJob) { alterJob.clear(); LOG.info("add {} job[{}] to finished or cancel list", alterJob.getType(), alterJob.getTableId()); this.finishedOrCancelledAlterJobs.add(alterJob); } + @Deprecated protected AlterJob removeAlterJob(long tableId) { return this.alterJobs.remove(tableId); } + @Deprecated public void removeDbAlterJob(long dbId) { Iterator> iterator = alterJobs.entrySet().iterator(); while (iterator.hasNext()) { @@ -154,6 +173,7 @@ public void removeDbAlterJob(long dbId) { * handle task report * reportVersion is used in schema change job. */ + @Deprecated public void handleFinishedReplica(AgentTask task, TTabletInfo finishTabletInfo, long reportVersion) throws MetaNotFoundException { long tableId = task.getTableId(); @@ -289,6 +309,7 @@ public abstract void process(List alterClauses, String clusterName, */ public abstract void cancel(CancelStmt stmt) throws DdlException; + @Deprecated public Integer getAlterJobNumByState(JobState state) { int jobNum = 0; for (AlterJob alterJob : alterJobs.values()) { diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java new file mode 100644 index 00000000000000..d90ec75d078292 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; + +import com.google.common.base.Preconditions; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/* + * Author: Chenmingyu + * Date: Jul 8, 2019 + */ + +/* + * Version 2 of AlterJob, for replacing the old version of AlterJob. + * This base class of RollupJob and SchemaChangeJob + */ +public class AlterJobV2 implements Writable { + private static final Logger LOG = LogManager.getLogger(AlterJobV2.class); + + public enum JobState { + PENDING, // Job is created + WAITING_TXN, // New replicas are created and Shadow catalog object is visible for incoming txns, + // waiting for previous txns to be finished + RUNNING, // alter tasks are sent to BE, and waiting for them finished. + FINISHED, // job is done + CANCELLED; // job is cancelled(failed or be cancelled by user) + + public boolean isFinalState() { + return this == JobState.FINISHED || this == JobState.CANCELLED; + } + } + + public enum JobType { + ROLLUP, SCHEMA_CHANGE + } + + protected JobType type; + protected long jobId; + protected JobState jobState; + + protected long dbId; + protected long tableId; + + protected String errMsg = ""; + protected long createTimeMs = -1; + protected long finishedTimeMs = -1; + protected long timeoutMs = -1; + + public AlterJobV2(long jobId, JobType jobType, long dbId, long tableId, long timeoutMs) { + this.jobId = jobId; + this.type = jobType; + this.dbId = dbId; + this.tableId = tableId; + this.timeoutMs = timeoutMs; + + this.createTimeMs = System.currentTimeMillis(); + this.jobState = JobState.PENDING; + } + + protected AlterJobV2(JobType type) { + this.type = type; + } + + public long getJobId() { + return jobId; + } + + public JobState getJobState() { + return jobState; + } + + public JobType getType() { + return type; + } + + public long getDbId() { + return dbId; + } + + public long getTableId() { + return tableId; + } + + protected void setCancelled(String errMsg) { + jobState = JobState.CANCELLED; + this.errMsg = errMsg; + LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); + // edit log will be wrote later + } + + private boolean isTimeout() { + return System.currentTimeMillis() - createTimeMs > timeoutMs; + } + + public void run() { + if (isTimeout()) { + cancel("Timeout"); + } + + switch (jobState) { + case PENDING: + runPendingJob(); + break; + case WAITING_TXN: + runWaitingTxnJob(); + break; + case RUNNING: + runRunningJob(); + break; + default: + break; + } + } + + protected void runPendingJob() { + throw new NotImplementedException(); + } + + protected void runWaitingTxnJob() { + throw new NotImplementedException(); + } + + protected void runRunningJob() { + throw new NotImplementedException(); + } + + protected void cancel(String errMsg) { + throw new NotImplementedException(); + } + + public static AlterJobV2 read(DataInput in) throws IOException { + JobType type = JobType.valueOf(Text.readString(in)); + switch (type) { + case ROLLUP: + return RollupJobV2.read(in); + default: + Preconditions.checkState(false); + return null; + } + } + + @Override + public synchronized void write(DataOutput out) throws IOException { + Text.writeString(out, type.name()); + Text.writeString(out, jobState.name()); + + out.writeLong(jobId); + out.writeLong(dbId); + out.writeLong(tableId); + + Text.writeString(out, errMsg); + out.writeLong(createTimeMs); + out.writeLong(finishedTimeMs); + out.writeLong(timeoutMs); + } + + @Override + public synchronized void readFields(DataInput in) throws IOException { + // read common members as write in AlterJobV2.write(). + // except 'type' member, which is read in AlterJobV2.read() + jobState = JobState.valueOf(Text.readString(in)); + + jobId = in.readLong(); + dbId = in.readLong(); + tableId = in.readLong(); + + errMsg = Text.readString(in); + createTimeMs = in.readLong(); + finishedTimeMs = in.readLong(); + timeoutMs = in.readLong(); + } +} diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index ca4900da06daec..f000cc28cf3808 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -33,7 +33,6 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Table; @@ -45,18 +44,13 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.util.ListComparator; -import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.EditLog; -import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.DropReplicaTask; -import org.apache.doris.thrift.TKeysType; -import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TStorageMedium; -import org.apache.doris.thrift.TStorageType; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -68,6 +62,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -80,17 +75,15 @@ public RollupHandler() { super("rollup"); } - private void processAddRollup(AddRollupClause alterClause, Database db, OlapTable olapTable, boolean isRestore) + private void processAddRollup(AddRollupClause alterClause, Database db, OlapTable olapTable) throws DdlException { - if (!isRestore) { - // table is under rollup or has a finishing alter job - if (olapTable.getState() == OlapTableState.ROLLUP || this.hasUnfinishedAlterJob(olapTable.getId())) { - throw new DdlException("Table[" + olapTable.getName() + "]'s is under ROLLUP"); - } - // up to here, table's state can only be NORMAL - Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); + // table is under rollup or has a finishing alter job + if (olapTable.getState() == OlapTableState.ROLLUP || this.hasUnfinishedAlterJob(olapTable.getId())) { + throw new DdlException("Table[" + olapTable.getName() + "]'s is under ROLLUP"); } + // up to here, table's state can only be NORMAL + Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); String rollupIndexName = alterClause.getRollupName(); String baseIndexName = alterClause.getBaseRollupName(); @@ -120,7 +113,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl Preconditions.checkState(baseIndex.getState() == IndexState.NORMAL, baseIndex.getState().name()); } - // 3 check if rollup columns are valid + // 3. check if rollup columns are valid // a. all columns should exist in base rollup schema // b. value after key // c. if rollup contains REPLACE column, all keys on base index should be included. @@ -184,7 +177,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl if (alterClause.getDupKeys() == null || alterClause.getDupKeys().isEmpty()) { // user does not specify duplicate key for rollup, // use base table's duplicate key. - // so we should check if rollup column contains all base table's duplicate key. + // so we should check if rollup columns contains all base table's duplicate key. List baseIdxCols = olapTable.getSchemaByIndexId(baseIndexId); Set baseIdxKeyColNames = Sets.newHashSet(); for (Column baseCol : baseIdxCols) { @@ -210,6 +203,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl } } + // check (a)(b) for (String columnName : rollupColumnNames) { Column oneColumn = olapTable.getColumn(columnName); if (oneColumn == null) { @@ -230,10 +224,10 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl throw new DdlException("No key column is found"); } } else { - // rollup have different dup keys with base table + // user specify the duplicate keys for rollup index List dupKeys = alterClause.getDupKeys(); if (dupKeys.size() > rollupColumnNames.size()) { - throw new DdlException("Duplicate key should be the prefix of rollup columns. Exceeded"); + throw new DdlException("Num of duplicate keys should less than or equal to num of rollup columns."); } for (int i = 0; i < rollupColumnNames.size(); i++) { @@ -242,7 +236,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl if (i < dupKeys.size()) { String dupKeyName = dupKeys.get(i); if (!rollupColName.equalsIgnoreCase(dupKeyName)) { - throw new DdlException("Duplicate key should be the prefix of rollup columns"); + throw new DdlException("Duplicate keys should be the prefix of rollup columns"); } isKey = true; } @@ -270,82 +264,42 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl } } - // 4. do create things - // 4.1 get storage type. default is COLUMN - - TKeysType rollupKeysType; - if (keysType == KeysType.DUP_KEYS) { - rollupKeysType = TKeysType.DUP_KEYS; - } else if (keysType == KeysType.UNIQUE_KEYS) { - rollupKeysType = TKeysType.UNIQUE_KEYS; - } else { - rollupKeysType = TKeysType.AGG_KEYS; - } + // assign rollup index's key type, same as base index's + KeysType rollupKeysType = keysType; - Map properties = alterClause.getProperties(); - TStorageType rollupStorageType = null; - try { - rollupStorageType = PropertyAnalyzer.analyzeStorageType(properties); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - // check storage type if has null column - boolean hasNullColumn = false; - for (Column column : rollupSchema) { - if (column.isAllowNull()) { - hasNullColumn = true; - break; - } - } - if (hasNullColumn && rollupStorageType != TStorageType.COLUMN) { - throw new DdlException("Only column rollup support null columns"); - } - - // 4.2 get rollup schema hash - int schemaVersion = 0; - try { - schemaVersion = PropertyAnalyzer.analyzeSchemaVersion(properties); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - int rollupSchemaHash = Util.schemaHash(schemaVersion, rollupSchema, olapTable.getCopiedBfColumns(), + // get rollup schema hash + int rollupSchemaHash = Util.schemaHash(0 /* init schema version */, rollupSchema, olapTable.getCopiedBfColumns(), olapTable.getBfFpp()); - // 4.3 get short key column count + // get short key column count + Map properties = alterClause.getProperties(); short rollupShortKeyColumnCount = Catalog.calcShortKeyColumnCount(rollupSchema, properties); + + // get timeout + long timeoutMs = alterClause.getTimeoutSecond() * 1000; - // 4.4 get user resource info - TResourceInfo resourceInfo = null; - if (ConnectContext.get() != null) { - resourceInfo = ConnectContext.get().toResourceCtx(); - } - - // 4.5 create rollup job + // 4. create rollup job long dbId = db.getId(); long tableId = olapTable.getId(); int baseSchemaHash = olapTable.getSchemaHashByIndexId(baseIndexId); - Catalog catalog = Catalog.getInstance(); + Catalog catalog = Catalog.getCurrentCatalog(); + long jobId = catalog.getNextId(); long rollupIndexId = catalog.getNextId(); - long transactionId = Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId(); - RollupJob rollupJob = new RollupJob(dbId, tableId, baseIndexId, rollupIndexId, - baseIndexName, rollupIndexName, rollupSchema, - baseSchemaHash, rollupSchemaHash, rollupStorageType, - rollupShortKeyColumnCount, resourceInfo, rollupKeysType, transactionId); + + RollupJobV2 rollupJob = new RollupJobV2(jobId, dbId, tableId, timeoutMs, + baseIndexId, rollupIndexId, baseIndexName, rollupIndexName, + rollupSchema, baseSchemaHash, rollupSchemaHash, + rollupKeysType, rollupShortKeyColumnCount); for (Partition partition : olapTable.getPartitions()) { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); - MaterializedIndex rollupIndex = new MaterializedIndex(rollupIndexId, IndexState.ROLLUP); - if (isRestore) { - rollupIndex.setState(IndexState.NORMAL); - } + MaterializedIndex rollupIndex = new MaterializedIndex(rollupIndexId, IndexState.SHADOW); MaterializedIndex baseIndex = partition.getIndex(baseIndexId); - TabletMeta rollupTabletMeta = new TabletMeta(dbId, tableId, partitionId, rollupIndexId, - rollupSchemaHash, medium); - short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); + TabletMeta rollupTabletMeta = new TabletMeta(dbId, tableId, partitionId, rollupIndexId, rollupSchemaHash, medium); for (Tablet baseTablet : baseIndex.getTablets()) { long baseTabletId = baseTablet.getId(); long rollupTabletId = catalog.getNextId(); @@ -353,10 +307,9 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl Tablet newTablet = new Tablet(rollupTabletId); rollupIndex.addTablet(newTablet, rollupTabletMeta); - rollupJob.setTabletIdMap(partitionId, rollupTabletId, baseTabletId); + rollupJob.addTabletIdMap(partitionId, rollupTabletId, baseTabletId); List baseReplicas = baseTablet.getReplicas(); - int replicaNum = 0; for (Replica baseReplica : baseReplicas) { long rollupReplicaId = catalog.getNextId(); long backendId = baseReplica.getBackendId(); @@ -367,60 +320,25 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl continue; } Preconditions.checkState(baseReplica.getState() == ReplicaState.NORMAL); - ++replicaNum; - // the new replica's init version is -1 until finished history rollup - Replica rollupReplica = new Replica(rollupReplicaId, backendId, rollupSchemaHash, - ReplicaState.ROLLUP); - // new replica's last failed version should be set to the partition's next version - 1, - // if all go well, the last failed version will be overwritten when rollup task finished and update - // replica version info. - // If not set, there is no other way to know that this replica has failed version. - rollupReplica.updateVersionInfo(rollupReplica.getVersion(), rollupReplica.getVersionHash(), - partition.getCommittedVersion(), partition.getCommittedVersionHash(), - rollupReplica.getLastSuccessVersion(), rollupReplica.getLastSuccessVersionHash()); - if (isRestore) { - rollupReplica.setState(ReplicaState.NORMAL); - } - // yiguolei: the rollup tablet's replica num maybe less than base tablet's replica num + Replica rollupReplica = new Replica(rollupReplicaId, backendId, rollupSchemaHash, ReplicaState.NORMAL); newTablet.addReplica(rollupReplica); } // end for baseReplica - - if (replicaNum < replicationNum / 2 + 1) { - String errMsg = "Tablet[" + baseTabletId + "] does not have enough replica. [" - + replicaNum + "/" + replicationNum + "]"; - LOG.warn(errMsg); - throw new DdlException(errMsg); - } } // end for baseTablets - if (isRestore) { - partition.createRollupIndex(rollupIndex); - } else { - rollupJob.addRollupIndex(partitionId, rollupIndex); - } + rollupJob.addRollupIndex(partitionId, rollupIndex); - LOG.debug("create rollup index[{}] based on index[{}] in partition[{}], restore: {}", - rollupIndexId, baseIndexId, partitionId, isRestore); + LOG.debug("create rollup index {} based on index {} in partition {}", + rollupIndexId, baseIndexId, partitionId); } // end for partitions - if (isRestore) { - olapTable.setIndexSchemaInfo(rollupIndexId, rollupIndexName, rollupSchema, 0, - rollupSchemaHash, rollupShortKeyColumnCount); - olapTable.setStorageTypeToIndex(rollupIndexId, rollupStorageType); - } else { - // update partition and table state - for (Partition partition : olapTable.getPartitions()) { - partition.setState(PartitionState.ROLLUP); - } - olapTable.setState(OlapTableState.ROLLUP); + // update table state + olapTable.setState(OlapTableState.ROLLUP); - addAlterJob(rollupJob); + addAlterJobV2(rollupJob); - // log rollup operation - EditLog editLog = catalog.getEditLog(); - editLog.logStartRollup(rollupJob); - LOG.debug("sync start create rollup index[{}] in table[{}]", rollupIndexId, tableId); - } + // log rollup operation + catalog.getEditLog().logAlterJob(rollupJob); + LOG.info("finished to create rollup job: {}", rollupJob.getJobId()); } public void processDropRollup(DropRollupClause alterClause, Database db, OlapTable olapTable) @@ -565,6 +483,26 @@ private AlterJob checkIfAnyRollupBasedOn(long tableId, String baseIndexName) { @Override protected void runOneCycle() { super.runOneCycle(); + runOldAlterJob(); + runAlterJobV2(); + } + + private void runAlterJobV2() { + Iterator> iter = alterJobsV2.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + AlterJobV2 alterJob = entry.getValue(); + alterJob.run(); + if (alterJob.getJobState().isFinalState()) { + iter.remove(); + finishedOrCancelledAlterJobsV2.offer(alterJob); + Catalog.getCurrentCatalog().getEditLog().logAlterJob(alterJob); + } + } + } + + @Deprecated + private void runOldAlterJob() { List cancelledJobs = Lists.newArrayList(); List finishedJobs = Lists.newArrayList(); @@ -719,14 +657,9 @@ public List> getAlterJobInfosByDb(Database db) { @Override public void process(List alterClauses, String clusterName, Database db, OlapTable olapTable) throws DdlException { - process(alterClauses, db, olapTable, false); - } - - public void process(List alterClauses, Database db, OlapTable olapTable, boolean isRestore) - throws DdlException { for (AlterClause alterClause : alterClauses) { if (alterClause instanceof AddRollupClause) { - processAddRollup((AddRollupClause) alterClause, db, olapTable, isRestore); + processAddRollup((AddRollupClause) alterClause, db, olapTable); } else if (alterClause instanceof DropRollupClause) { processDropRollup((DropRollupClause) alterClause, db, olapTable); } else { @@ -779,4 +712,39 @@ public void cancel(CancelStmt stmt) throws DdlException { jobDone(rollupJob); } + + public void replayRollupJobV2(RollupJobV2 rollupJob) { + switch (rollupJob.getJobState()) { + case PENDING: + // PENDING job SHOULD NOT be in alterJobsV2 before + Preconditions.checkState(!alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); + rollupJob.replayPending(); + addAlterJobV2(rollupJob); + break; + case WAITING_TXN: + // WAITING_TXN job SHOULD be in alterJobsV2 before + Preconditions.checkState(alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); + rollupJob.replayWaitingTxn(); + // this add operation will replace the old rollup job with the new one + addAlterJobV2(rollupJob); + break; + case CANCELLED: + // CANCELLED job SHOULD be in alterJobsV2 before + Preconditions.checkState(alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); + rollupJob.replayCancelled(); + break; + case FINISHED: + // FINISHED job SHOULD be in alterJobsV2 before + Preconditions.checkState(alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); + rollupJob.replayFinished(); + break; + default: + break; + } + + if (rollupJob.getJobState().isFinalState()) { + alterJobsV2.remove(rollupJob.getJobId()); + finishedOrCancelledAlterJobsV2.offer(rollupJob); + } + } } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java new file mode 100644 index 00000000000000..8474297b600464 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -0,0 +1,643 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Partition.PartitionState; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.TabletInvertedIndex; +import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.common.Config; +import org.apache.doris.common.MarkedCountDownLatch; +import org.apache.doris.common.io.Text; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.CreateReplicaTask; +import org.apache.doris.task.CreateRollupTaskV2; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TStorageType; +import org.apache.doris.thrift.TTaskType; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +/* + * Author: Chenmingyu + * Date: Jul 8, 2019 + */ + +/* + * Version 2 of RollupJob. + * This is for replacing the old RollupJob + * https://github.com/apache/incubator-doris/issues/1429 + */ +public class RollupJobV2 extends AlterJobV2 { + private static final Logger LOG = LogManager.getLogger(RollupJobV2.class); + + // partition id -> (rollup tablet id -> base tablet id) + private Map> partitionIdToBaseRollupTabletIdMap = Maps.newHashMap(); + private Map partitionIdToRollupIndex = Maps.newHashMap(); + + // rollup and base schema info + private long baseIndexId; + private long rollupIndexId; + private String baseIndexName; + private String rollupIndexName; + + private List rollupSchema = Lists.newArrayList(); + private int baseSchemaHash; + private int rollupSchemaHash; + + private KeysType rollupKeysType; + private short rollupShortKeyColumnCount; + + // The rollup job will wait all transactions before this txn id finished, then send the rollup tasks. + protected long watershedTxnId = -1; + + // save all create rollup tasks + private AgentBatchTask rollupBatchTask = new AgentBatchTask(); + + public RollupJobV2(long jobId, long dbId, long tableId, long timeoutMs, + long baseIndexId, long rollupIndexId, String baseIndexName, String rollupIndexName, + List rollupSchema, int baseSchemaHash, int rollupSchemaHash, + KeysType rollupKeysType, short rollupShortKeyColumnCount) { + super(jobId, JobType.ROLLUP, dbId, tableId, timeoutMs); + + this.baseIndexId = baseIndexId; + this.rollupIndexId = rollupIndexId; + this.baseIndexName = baseIndexName; + this.rollupIndexName = rollupIndexName; + + this.rollupSchema = rollupSchema; + this.baseSchemaHash = baseSchemaHash; + this.rollupSchemaHash = rollupSchemaHash; + this.rollupKeysType = rollupKeysType; + this.rollupShortKeyColumnCount = rollupShortKeyColumnCount; + } + + private RollupJobV2() { + super(JobType.ROLLUP); + } + + public void addTabletIdMap(long partitionId, long rollupTabletId, long baseTabletId) { + Map tabletIdMap = partitionIdToBaseRollupTabletIdMap.get(partitionId); + if (tabletIdMap == null) { + tabletIdMap = Maps.newHashMap(); + partitionIdToBaseRollupTabletIdMap.put(partitionId, tabletIdMap); + } + tabletIdMap.put(rollupTabletId, baseTabletId); + } + + public void addRollupIndex(long partitionId, MaterializedIndex rollupIndex) { + this.partitionIdToRollupIndex.put(partitionId, rollupIndex); + } + + /* + * runPendingJob(): + * 1. Create all rollup replicas and wait them finished. + * 2. After creating done, set the rollup index state as SHADOW, add it to catalog, user can not see this + * rollup, but internal load process will generate data for this rollup index. + * 3. get a new transaction id, then set job's state to WAITING_TXN + */ + @Override + protected void runPendingJob() { + Preconditions.checkState(jobState == JobState.PENDING, jobState); + + LOG.info("begin to send create rollup replica tasks. job: {}", jobId); + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancel("Databasee " + dbId + " does not exist"); + return; + } + + // 1. create rollup replicas + AgentBatchTask batchTask = new AgentBatchTask(); + // count total replica num + int totalReplicaNum = 0; + for (MaterializedIndex rollupIdx : partitionIdToRollupIndex.values()) { + for (Tablet tablet : rollupIdx.getTablets()) { + totalReplicaNum += tablet.getReplicas().size(); + } + } + MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalReplicaNum); + db.readLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); + + for (Map.Entry entry : this.partitionIdToRollupIndex.entrySet()) { + long partitionId = entry.getKey(); + Partition partition = tbl.getPartition(partitionId); + if (partition == null) { + continue; + } + TStorageMedium storageMedium = tbl.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + MaterializedIndex rollupIndex = entry.getValue(); + + Map tabletIdMap = this.partitionIdToBaseRollupTabletIdMap.get(partitionId); + for (Tablet rollupTablet : rollupIndex.getTablets()) { + long rollupTabletId = rollupTablet.getId(); + List rollupReplicas = rollupTablet.getReplicas(); + for (Replica rollupReplica : rollupReplicas) { + long backendId = rollupReplica.getBackendId(); + Preconditions.checkNotNull(tabletIdMap.get(rollupTabletId)); // baseTabletId + + CreateReplicaTask createReplicaTask = new CreateReplicaTask( + backendId, dbId, tableId, partitionId, rollupIndexId, rollupTabletId, + rollupShortKeyColumnCount, rollupSchemaHash, + Partition.PARTITION_INIT_VERSION, Partition.PARTITION_INIT_VERSION_HASH, + rollupKeysType, TStorageType.COLUMN, storageMedium, + rollupSchema, tbl.getCopiedBfColumns(), tbl.getBfFpp(), countDownLatch); + createReplicaTask.setBaseTabletId(tabletIdMap.get(rollupTabletId)); + + batchTask.addTask(createReplicaTask); + } // end for rollupReplicas + } // end for rollupTablets + } + } finally { + db.readUnlock(); + } + + // send all tasks and wait them finished + AgentTaskQueue.addBatchTask(batchTask); + AgentTaskExecutor.submit(batchTask); + // max timeout is 30 seconds + long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, 30000); + boolean ok = false; + try { + ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOG.warn("InterruptedException: ", e); + ok = false; + } + + if (!ok) { + // create rollup replicas failed. just cancel the job + // clear tasks and show the failed replicas to user + AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 10 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 10)); + String idStr = Joiner.on(", ").join(subList); + LOG.warn("failed to create rollup replicas for job: {}, {}", jobId, idStr); + cancel("Create rollup replicas failed. Error replicas: " + idStr); + return; + } + + // create all rollup replicas success. + // set rollup index’s state to SHADOW and add it to catalog + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); + addRollupIndexToCatalog(tbl); + } finally { + db.writeUnlock(); + } + + this.watershedTxnId = Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId(); + this.jobState = JobState.WAITING_TXN; + + // write edit log + Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + LOG.info("transfer rollup job {} state to {}, watershed txn id: {}", jobId, this.jobState, watershedTxnId); + } + + private void addRollupIndexToCatalog(OlapTable tbl) { + for (Partition partition : tbl.getPartitions()) { + long partitionId = partition.getId(); + MaterializedIndex rollupIndex = this.partitionIdToRollupIndex.get(partitionId); + Preconditions.checkNotNull(rollupIndex); + partition.createRollupIndex(rollupIndex); + } + + tbl.setIndexSchemaInfo(rollupIndexId, rollupIndexName, rollupSchema, 0, + rollupSchemaHash, rollupShortKeyColumnCount); + tbl.setStorageTypeToIndex(rollupIndexId, TStorageType.COLUMN); + } + + /* + * runWaitingTxnJob(): + * 1. Wait the previous transactions to be finished. + * 2. If all previous transactions finished, send create rollup tasks to BE + * 3. Change job state to RUNNING + */ + @Override + protected void runWaitingTxnJob() { + Preconditions.checkState(jobState == JobState.WAITING_TXN, jobState); + + if (!isPreviousLoadFinished()) { + LOG.info("wait transactions before {} to be finished, rollup job: {}", watershedTxnId, jobId); + return; + } + + LOG.info("previous transactions are all finished, begin to send rollup tasks. job: {}", jobId); + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancel("Databasee " + dbId + " does not exist"); + return; + } + + db.readLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); + for (Map.Entry entry : this.partitionIdToRollupIndex.entrySet()) { + long partitionId = entry.getKey(); + Partition partition = tbl.getPartition(partitionId); + if (partition == null) { + continue; + } + + // the rollup task will transform the data before committed version. + // DO NOT use visible version because we need to handle the committed but not published version + // on BE. + long committedVersion = partition.getCommittedVersion(); + long committedVersionHash = partition.getCommittedVersionHash(); + + MaterializedIndex rollupIndex = entry.getValue(); + Map tabletIdMap = this.partitionIdToBaseRollupTabletIdMap.get(partitionId); + for (Tablet rollupTablet : rollupIndex.getTablets()) { + long rollupTabletId = rollupTablet.getId(); + long baseTabletId = tabletIdMap.get(rollupTabletId); + + List rollupReplicas = rollupTablet.getReplicas(); + for (Replica rollupReplica : rollupReplicas) { + CreateRollupTaskV2 rollupTask = new CreateRollupTaskV2( + rollupReplica.getBackendId(), dbId, tableId, partitionId, + rollupIndexId, baseIndexId, + rollupTabletId, baseTabletId, rollupReplica.getId(), + rollupSchemaHash, baseSchemaHash, + committedVersion, committedVersionHash); + rollupBatchTask.addTask(rollupTask); + } + } + } + } finally { + db.readUnlock(); + } + + AgentTaskQueue.addBatchTask(rollupBatchTask); + AgentTaskExecutor.submit(rollupBatchTask); + this.jobState = JobState.RUNNING; + + // do not write edit log here, tasks will be send again if FE restart or master changed. + LOG.info("transfer rollup job {} state to {}", jobId, this.jobState); + } + + @Override + protected void runRunningJob() { + Preconditions.checkState(jobState == JobState.RUNNING, jobState); + if (!rollupBatchTask.isFinished()) { + LOG.info("rollup tasks not finished. job: {}", jobId); + return; + } + + /* + * all tasks are finished. check the integrity. + * we just check whether all rollup replicas are healthy. + */ + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancel("Databasee " + dbId + " does not exist"); + return; + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); + for (Map.Entry entry : this.partitionIdToRollupIndex.entrySet()) { + long partitionId = entry.getKey(); + Partition partition = tbl.getPartition(partitionId); + if (partition == null) { + continue; + } + + long visiableVersion = partition.getVisibleVersion(); + long visiableVersionHash = partition.getVisibleVersionHash(); + short expectReplicationNum = tbl.getPartitionInfo().getReplicationNum(partition.getId()); + + MaterializedIndex rollupIndex = entry.getValue(); + for (Tablet rollupTablet : rollupIndex.getTablets()) { + List replicas = rollupTablet.getReplicas(); + int healthyReplicaNum = 0; + for (Replica replica : replicas) { + if (replica.getLastFailedVersion() < 0 + && replica.checkVersionCatchUp(visiableVersion, visiableVersionHash)) { + healthyReplicaNum++; + } + } + + if (healthyReplicaNum < expectReplicationNum / 2 + 1) { + LOG.warn("rollup tablet {} has few healthy replicas: {}, rollup job: {}", + rollupTablet.getId(), replicas, jobId); + cancel("rollup tablet " + rollupTablet.getId() + " has few healthy replicas"); + return; + } + } // end for tablets + } // end for partitions + + // all partitions are good + // set table and rollup index state to NORMAL, then finish the job + for (Partition partition : tbl.getPartitions()) { + // get index from catalog, not from 'partitionIdToRollupIndex'. + // because if this alter job is recovered from edit log, rollup index in 'partitionIdToRollupIndex' + // is not the same object in catalog. So modification on that index can not reflect to the index + // in catalog. + MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); + Preconditions.checkNotNull(rollupIndex, rollupIndex); + rollupIndex.setState(IndexState.NORMAL); + } + + tbl.setState(OlapTableState.NORMAL); + } finally { + db.writeUnlock(); + } + + this.jobState = JobState.FINISHED; + this.finishedTimeMs = System.currentTimeMillis(); + + Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + LOG.info("rollup job finished: {}", jobId); + } + + @Override + protected void cancel(String errMsg) { + cancelInternal(); + setCancelled(errMsg); + } + + private void cancelInternal() { + // clear tasks if have + AgentTaskQueue.removeBatchTask(rollupBatchTask, TTaskType.ROLLUP); + // remove all rollup indexes, and set state to NORMAL + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db != null) { + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl != null) { + for (Long partitionId : partitionIdToRollupIndex.keySet()) { + MaterializedIndex rollupIndex = partitionIdToRollupIndex.get(partitionId); + for (Tablet rollupTablet : rollupIndex.getTablets()) { + invertedIndex.deleteTablet(rollupTablet.getId()); + } + Partition partition = tbl.getPartition(partitionId); + partition.deleteRollupIndex(rollupIndexId); + } + tbl.setState(OlapTableState.NORMAL); + } + } finally { + db.writeUnlock(); + } + } + } + + protected boolean isPreviousLoadFinished() { + return Catalog.getCurrentGlobalTransactionMgr().isPreviousTransactionsFinished(watershedTxnId, dbId); + } + + public static RollupJobV2 read(DataInput in) throws IOException { + RollupJobV2 rollupJob = new RollupJobV2(); + rollupJob.readFields(in); + return rollupJob; + } + + @Override + public synchronized void write(DataOutput out) throws IOException { + super.write(out); + + out.writeInt(partitionIdToRollupIndex.size()); + for (long partitionId : partitionIdToRollupIndex.keySet()) { + out.writeLong(partitionId); + + out.writeInt(partitionIdToBaseRollupTabletIdMap.get(partitionId).size()); + for (Map.Entry entry : partitionIdToBaseRollupTabletIdMap.get(partitionId).entrySet()) { + out.writeLong(entry.getKey()); + out.writeLong(entry.getValue()); + } + + MaterializedIndex rollupIndex = partitionIdToRollupIndex.get(partitionId); + rollupIndex.write(out); + } + + out.writeLong(baseIndexId); + out.writeLong(rollupIndexId); + Text.writeString(out, baseIndexName); + Text.writeString(out, rollupIndexName); + + // rollup schema + out.writeInt(rollupSchema.size()); + for (Column column : rollupSchema) { + column.write(out); + } + out.writeInt(baseSchemaHash); + out.writeInt(rollupSchemaHash); + + Text.writeString(out, rollupKeysType.name()); + out.writeShort(rollupShortKeyColumnCount); + + out.writeLong(watershedTxnId); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + int size = in.readInt(); + for (int i = 0; i < size; i++) { + long partitionId = in.readLong(); + int size2 = in.readInt(); + Map tabletIdMap = partitionIdToBaseRollupTabletIdMap.get(partitionId); + if (tabletIdMap == null) { + tabletIdMap = Maps.newHashMap(); + partitionIdToBaseRollupTabletIdMap.put(partitionId, tabletIdMap); + } + for (int j = 0; j < size2; j++) { + long rollupTabletId = in.readLong(); + long baseTabletId = in.readLong(); + tabletIdMap.put(rollupTabletId, baseTabletId); + } + + partitionIdToRollupIndex.put(partitionId, MaterializedIndex.read(in)); + } + + baseIndexId = in.readLong(); + rollupIndexId = in.readLong(); + baseIndexName = Text.readString(in); + rollupIndexName = Text.readString(in); + + size = in.readInt(); + for (int i = 0; i < size; i++) { + Column column = Column.read(in); + rollupSchema.add(column); + } + baseSchemaHash = in.readInt(); + rollupSchemaHash = in.readInt(); + + rollupKeysType = KeysType.valueOf(Text.readString(in)); + rollupShortKeyColumnCount = in.readShort(); + + watershedTxnId = in.readLong(); + } + + /* + * Replay job in pending state. + * Should replay all changes before this job's state transfer to PENDING. + * These changes should be same as changes in RollupHander.processAddRollup() + */ + public void replayPending() { + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + // database may be dropped before replaying this log. just return + return; + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + // table may be dropped before replaying this log. just return + return; + } + + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + for (Long partitionId : partitionIdToRollupIndex.keySet()) { + Partition partition = tbl.getPartition(partitionId); + MaterializedIndex rollupIndex = partitionIdToRollupIndex.get(partitionId); + TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + TabletMeta rollupTabletMeta = new TabletMeta(dbId, tableId, partitionId, rollupIndexId, + rollupSchemaHash, medium); + + for (Tablet rollupTablet : rollupIndex.getTablets()) { + invertedIndex.addTablet(rollupTablet.getId(), rollupTabletMeta); + for (Replica rollupReplica : rollupTablet.getReplicas()) { + invertedIndex.addReplica(rollupTablet.getId(), rollupReplica); + } + } + + partition.setState(PartitionState.ROLLUP); + } + tbl.setState(OlapTableState.ROLLUP); + } finally { + db.writeUnlock(); + } + LOG.info("replay pending rollup job: {}", jobId); + } + + /* + * Replay job in WAITING_TXN state. + * Should replay all changes in runPendingJob() + */ + public void replayWaitingTxn() { + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + // database may be dropped before replaying this log. just return + return; + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + // table may be dropped before replaying this log. just return + return; + } + addRollupIndexToCatalog(tbl); + } finally { + db.writeUnlock(); + } + LOG.info("replay waiting txn rollup job: {}", jobId); + } + + /* + * Replay job in FINISHED state. + * Should replay all changes in runRuningJob() + */ + public void replayFinished() { + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db != null) { + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl != null) { + Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); + for (Partition partition : tbl.getPartitions()) { + MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); + Preconditions.checkNotNull(rollupIndex, rollupIndex); + rollupIndex.setState(IndexState.NORMAL); + } + tbl.setState(OlapTableState.NORMAL); + } + } finally { + db.writeUnlock(); + } + } + LOG.info("replay finished rollup job: {}", jobId); + } + + /* + * Replay job in CANCELLED state. + */ + public void replayCancelled() { + cancelInternal(); + LOG.info("replay cancelled rollup job: {}", jobId); + } + +} diff --git a/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java b/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java index e674d32df0c4bb..454cf31cb72d1d 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java +++ b/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java @@ -18,10 +18,12 @@ package org.apache.doris.analysis; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.io.Text; +import org.apache.doris.common.util.PropertyAnalyzer; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -43,6 +45,8 @@ public class AddRollupClause extends AlterClause { private List columnNames; private String baseRollupName; private List dupKeys; + private long timeoutSecond; + private Map properties; public AddRollupClause() { @@ -66,6 +70,10 @@ public String getBaseRollupName() { return baseRollupName; } + public long getTimeoutSecond() { + return timeoutSecond; + } + public AddRollupClause(String rollupName, List columnNames, List dupKeys, String baseRollupName, Map properties) { @@ -93,6 +101,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException { } } baseRollupName = Strings.emptyToNull(baseRollupName); + + timeoutSecond = PropertyAnalyzer.analyzeTimeout(properties, Config.alter_table_timeout_second); } @Override diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 61234be1593890..5ddae4144c6502 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -38,7 +38,8 @@ public class MaterializedIndex extends MetaObject implements Writable { public enum IndexState { NORMAL, ROLLUP, - SCHEMA_CHANGE + SCHEMA_CHANGE, + SHADOW // index in SHADOW state is visible to load process, but invisible to query } private long id; diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 4dec9661808b64..013ab752eb67d4 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -47,7 +47,9 @@ public class Partition extends MetaObject implements Writable { public enum PartitionState { NORMAL, + @Deprecated ROLLUP, + @Deprecated SCHEMA_CHANGE } diff --git a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index caa8e6d705bf2a..4dcb15798baa50 100644 --- a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -68,6 +68,8 @@ public class PropertyAnalyzer { public static final String PROPERTIES_COLOCATE_WITH = "colocate_with"; + public static final String PROPERTIES_TIMEOUT = "timeout"; + public static DataProperty analyzeDataProperty(Map properties, DataProperty oldDataProperty) throws AnalysisException { DataProperty dataProperty = oldDataProperty; @@ -353,4 +355,18 @@ public static String analyzeColocate(Map properties) throws Anal } return colocateGroup; } + + public static long analyzeTimeout(Map properties, long defaultTimeout) throws AnalysisException { + long timeout = defaultTimeout; + if (properties != null && properties.containsKey(PROPERTIES_TIMEOUT)) { + String timeoutStr = properties.get(PROPERTIES_TIMEOUT); + try { + timeout = Long.valueOf(timeoutStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid timeout format: " + timeoutStr); + } + properties.remove(PROPERTIES_TIMEOUT); + } + return timeout; + } } diff --git a/fe/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/src/main/java/org/apache/doris/journal/JournalEntity.java index a28ebfb7b38ab6..06a8515b4f5861 100644 --- a/fe/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -18,6 +18,7 @@ package org.apache.doris.journal; import org.apache.doris.alter.AlterJob; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; @@ -411,6 +412,11 @@ public void readFields(DataInput in) throws IOException { needRead = false; break; } + case OperationType.OP_ALTER_JOB_V2: { + data = AlterJobV2.read(in); + needRead = false; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/src/main/java/org/apache/doris/persist/EditLog.java b/fe/src/main/java/org/apache/doris/persist/EditLog.java index 3a115ad939900e..9581829e627176 100644 --- a/fe/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/src/main/java/org/apache/doris/persist/EditLog.java @@ -17,8 +17,10 @@ package org.apache.doris.persist; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.alter.DecommissionBackendJob; import org.apache.doris.alter.RollupJob; +import org.apache.doris.alter.RollupJobV2; import org.apache.doris.alter.SchemaChangeJob; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupJob; @@ -654,28 +656,39 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { } case OperationType.OP_REMOVE_ROUTINE_LOAD_JOB: { RoutineLoadOperation operation = (RoutineLoadOperation) journal.getData(); - Catalog.getCurrentCatalog().getRoutineLoadManager().replayRemoveOldRoutineLoad(operation); + catalog.getRoutineLoadManager().replayRemoveOldRoutineLoad(operation); break; } case OperationType.OP_CREATE_LOAD_JOB: { org.apache.doris.load.loadv2.LoadJob loadJob = (org.apache.doris.load.loadv2.LoadJob) journal.getData(); - Catalog.getCurrentCatalog().getLoadManager().replayCreateLoadJob(loadJob); + catalog.getLoadManager().replayCreateLoadJob(loadJob); break; } case OperationType.OP_END_LOAD_JOB: { LoadJobFinalOperation operation = (LoadJobFinalOperation) journal.getData(); - Catalog.getCurrentCatalog().getLoadManager().replayEndLoadJob(operation); + catalog.getLoadManager().replayEndLoadJob(operation); break; } case OperationType.OP_CREATE_SMALL_FILE: { SmallFile smallFile = (SmallFile) journal.getData(); - Catalog.getCurrentCatalog().getSmallFileMgr().replayCreateFile(smallFile); + catalog.getSmallFileMgr().replayCreateFile(smallFile); break; } case OperationType.OP_DROP_SMALL_FILE: { SmallFile smallFile = (SmallFile) journal.getData(); - Catalog.getCurrentCatalog().getSmallFileMgr().replayRemoveFile(smallFile); + catalog.getSmallFileMgr().replayRemoveFile(smallFile); + break; + } + case OperationType.OP_ALTER_JOB_V2: { + AlterJobV2 alterJob = (AlterJobV2) journal.getData(); + switch (alterJob.getType()) { + case ROLLUP: + catalog.getRollupHandler().replayRollupJobV2((RollupJobV2) alterJob); + break; + default: + break; + } break; } default: { @@ -1167,4 +1180,8 @@ public void logCreateSmallFile(SmallFile info) { public void logDropSmallFile(SmallFile info) { logEdit(OperationType.OP_DROP_SMALL_FILE, info); } + + public void logAlterJob(AlterJobV2 alterJob) { + logEdit(OperationType.OP_ALTER_JOB_V2, alterJob); + } } diff --git a/fe/src/main/java/org/apache/doris/persist/OperationType.java b/fe/src/main/java/org/apache/doris/persist/OperationType.java index 1eb8108b0e9c4d..5041cbc793ab28 100644 --- a/fe/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/src/main/java/org/apache/doris/persist/OperationType.java @@ -54,6 +54,7 @@ public class OperationType { public static final short OP_CLEAR_ROLLUP_INFO = 28; public static final short OP_FINISH_CONSISTENCY_CHECK = 29; public static final short OP_RENAME_ROLLUP = 120; + public static final short OP_ALTER_JOB_V2 = 121; // 30~39 130~139 230~239 ... // load job for only hadoop load diff --git a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java index 6f09c3016f4099..71d16ae1a28414 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java +++ b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.common.ClientPool; +import org.apache.doris.common.Pair; import org.apache.doris.system.Backend; import org.apache.doris.thrift.BackendService; import org.apache.doris.thrift.TAgentServiceVersion; @@ -44,6 +45,8 @@ import org.apache.doris.thrift.TUpdateTabletMetaInfoReq; import org.apache.doris.thrift.TUploadReq; +import com.google.common.collect.Lists; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -101,6 +104,36 @@ public int getTaskNum() { return num; } + // return true only if all tasks are finished. + // NOTICE that even if AgentTask.isFinished() return false, it does not mean that task is not finished. + // this depends on caller's logic. See comments on 'isFinished' member. + public boolean isFinished() { + for (List tasks : this.backendIdToTasks.values()) { + for (AgentTask agentTask : tasks) { + if (!agentTask.isFinished()) { + return false; + } + } + } + return true; + } + + // return the limit number of unfinished task. + // backend id -> signature + public List> getUnfinishedTasks(int limit) { + List> res = Lists.newArrayList(); + for (List tasks : this.backendIdToTasks.values()) { + for (AgentTask agentTask : tasks) { + if (!agentTask.isFinished()) { + if (res.size() < limit) { + res.add(Pair.create(agentTask.getBackendId(), agentTask.getSignature())); + } + } + } + } + return res; + } + @Override public void run() { for (Long backendId : this.backendIdToTasks.keySet()) { diff --git a/fe/src/main/java/org/apache/doris/task/AgentTask.java b/fe/src/main/java/org/apache/doris/task/AgentTask.java index 807d859c557d3e..60486e73791d77 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentTask.java +++ b/fe/src/main/java/org/apache/doris/task/AgentTask.java @@ -34,6 +34,10 @@ public abstract class AgentTask { protected TResourceInfo resourceInfo; protected int failedTimes; + // some of process may use this member to check if the task is finished. + // some of are not. + // so whether the task is finished depends on caller's logic, not the value of this member. + protected boolean isFinished = false; public AgentTask(TResourceInfo resourceInfo, long backendId, TTaskType taskType, long dbId, long tableId, long partitionId, long indexId, long tabletId, long signature) { @@ -101,6 +105,14 @@ public int getFailedTimes() { return this.failedTimes; } + public void setFinished(boolean isFinished) { + this.isFinished = isFinished; + } + + public boolean isFinished() { + return isFinished; + } + @Override public String toString() { return "[" + taskType + "], signature: " + signature + ", backendId: " + backendId + ", tablet id: " + tabletId; diff --git a/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java b/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java index 38e4fdfab3a8b9..48e02054ebd6bf 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java +++ b/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java @@ -44,6 +44,12 @@ public class AgentTaskQueue { // backend id -> (task type -> (signature -> agent task)) private static Table> tasks = HashBasedTable.create(); private static int taskNum = 0; + + public static synchronized void addBatchTask(AgentBatchTask batchTask) { + for (AgentTask task : batchTask.getAllTasks()) { + addTask(task); + } + } public static synchronized boolean addTask(AgentTask task) { long backendId = task.getBackendId(); @@ -70,6 +76,14 @@ public static synchronized boolean addTask(AgentTask task) { return true; } + // remove all task in AgentBatchTask. + // the caller should make sure all tasks in AgentBatchTask is type of 'type' + public static synchronized void removeBatchTask(AgentBatchTask batchTask, TTaskType type) { + for (AgentTask task : batchTask.getAllTasks()) { + removeTask(task.getBackendId(), type, task.getSignature()); + } + } + public static synchronized void removeTask(long backendId, TTaskType type, long signature) { if (!tasks.contains(backendId, type)) { return; diff --git a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 4cb938d698f9db..ce05f9e76547c5 100644 --- a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -54,15 +54,18 @@ public class CreateReplicaTask extends AgentTask { private double bfFpp; // used for synchronous process - private MarkedCountDownLatch latch; + private MarkedCountDownLatch latch; private boolean inRestoreMode = false; + // if base tablet id is set, BE will create the replica on same disk as this base tablet + private long baseTabletId = -1; + public CreateReplicaTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, short shortKeyColumnCount, int schemaHash, long version, long versionHash, KeysType keysType, TStorageType storageType, TStorageMedium storageMedium, List columns, - Set bfColumns, double bfFpp, MarkedCountDownLatch latch) { + Set bfColumns, double bfFpp, MarkedCountDownLatch latch) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.shortKeyColumnCount = shortKeyColumnCount; @@ -92,7 +95,7 @@ public void countDownLatch(long backendId, long tabletId) { } } - public void setLatch(MarkedCountDownLatch latch) { + public void setLatch(MarkedCountDownLatch latch) { this.latch = latch; } @@ -100,6 +103,10 @@ public void setInRestoreMode(boolean inRestoreMode) { this.inRestoreMode = inRestoreMode; } + public void setBaseTabletId(long baseTabletId) { + this.baseTabletId = baseTabletId; + } + public TCreateTabletReq toThrift() { TCreateTabletReq createTabletReq = new TCreateTabletReq(); createTabletReq.setTablet_id(tabletId); @@ -136,6 +143,10 @@ public TCreateTabletReq toThrift() { createTabletReq.setTable_id(tableId); createTabletReq.setPartition_id(partitionId); + if (baseTabletId != -1) { + createTabletReq.setBase_tablet_id(baseTabletId); + } + return createTabletReq; } } diff --git a/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java b/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java new file mode 100644 index 00000000000000..9a76d9e3f85ea4 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.task; + +import org.apache.doris.thrift.TAlterTabletReq; +import org.apache.doris.thrift.TTaskType; + +/* + * This is to replace the old CreateRollupTask. + * This task only do data transformation from base tablet to rollup tablet. + * The rollup tablet has already been created before. + */ +public class CreateRollupTaskV2 extends AgentTask { + + private long baseTabletId; + private long rollupReplicaId; + private int baseSchemaHash; + private int rollupSchemaHash; + private long version; + private long versionHash; + + public CreateRollupTaskV2(long backendId, long dbId, long tableId, + long partitionId, long rollupIndexId, long baseIndexId, long rollupTabletId, + long baseTabletId, long rollupReplicaId, int rollupSchemaHash, int baseSchemaHash, + long version, long versionHash) { + super(null, backendId, TTaskType.ROLLUP, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); + + this.baseTabletId = baseTabletId; + this.rollupReplicaId = rollupReplicaId; + + this.rollupSchemaHash = rollupSchemaHash; + this.baseSchemaHash = baseSchemaHash; + + this.version = version; + this.versionHash = versionHash; + } + + public TAlterTabletReq toThrift() { + return null; + } + + public long getBaseTabletId() { + return baseTabletId; + } + + public long getRollupReplicaId() { + return rollupReplicaId; + } + + public int getRollupSchemaHash() { + return rollupSchemaHash; + } + + public int getBaseSchemaHash() { + return baseSchemaHash; + } + + public long getVersion() { + return version; + } + + public long getVersionHash() { + return versionHash; + } +} diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java index 87b49af93f8ac2..284046f4c952c9 100644 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java @@ -116,7 +116,7 @@ public void testAddRollup() throws Exception { alterClauses.add(clause); Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - rollupHandler.process(alterClauses, db, olapTable, false); + rollupHandler.process(alterClauses, db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); @@ -134,7 +134,7 @@ public void testRollup1() throws Exception { Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db, olapTable, false); + rollupHandler.process(alterClauses, db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); @@ -226,7 +226,7 @@ public void testRollup2() throws Exception { Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db, olapTable, false); + rollupHandler.process(alterClauses, db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); From 5cd0bb76f7ca5dfcd6e4cf069cf18ca445c3d617 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 12 Jul 2019 11:20:11 +0800 Subject: [PATCH 02/79] second commit --- .../java/org/apache/doris/alter/Alter.java | 9 +- .../org/apache/doris/alter/AlterHandler.java | 17 ++- .../org/apache/doris/alter/AlterJobV2.java | 21 +-- .../org/apache/doris/alter/RollupHandler.java | 131 +++++------------- .../org/apache/doris/alter/RollupJobV2.java | 45 ++++-- .../apache/doris/task/CreateReplicaTask.java | 5 +- 6 files changed, 99 insertions(+), 129 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/Alter.java b/fe/src/main/java/org/apache/doris/alter/Alter.java index 2a3e7278e7f864..2100b3a25f1658 100644 --- a/fe/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/src/main/java/org/apache/doris/alter/Alter.java @@ -171,13 +171,8 @@ public void processAlterTable(AlterTableStmt stmt) throws DdlException { throw new DdlException("table with empty parition cannot do schema change. [" + tableName + "]"); } - if (olapTable.getState() == OlapTableState.SCHEMA_CHANGE - || olapTable.getState() == OlapTableState.RESTORE) { - throw new DdlException("Table[" + table.getName() + "]'s state[" + olapTable.getState() - + "] does not allow doing ALTER ops"); - // here we pass NORMAL and ROLLUP - // NORMAL: ok to do any alter ops - // ROLLUP: we allow user DROP a rollup index when it's under ROLLUP + if (olapTable.getState() != OlapTableState.NORMAL) { + throw new DdlException("Table[" + table.getName() + "]'s state is not NORMAL. Do not allow doing ALTER ops"); } if (hasSchemaChange || hasModifyProp || hasRollup) { diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index e8af5df79eebab..f813b9e429a928 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -31,6 +31,8 @@ import org.apache.doris.task.AgentTask; import org.apache.doris.thrift.TTabletInfo; +import com.google.common.collect.Maps; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -39,6 +41,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; public abstract class AlterHandler extends Daemon { @@ -50,9 +53,8 @@ public abstract class AlterHandler extends Daemon { @Deprecated protected ConcurrentLinkedQueue finishedOrCancelledAlterJobs = new ConcurrentLinkedQueue(); - // jobId -> AlterJobV2 - protected ConcurrentHashMap alterJobsV2 = new ConcurrentHashMap(); - protected ConcurrentLinkedQueue finishedOrCancelledAlterJobsV2 = new ConcurrentLinkedQueue(); + // queue of alter job v2 + protected ConcurrentMap alterJobsV2 = Maps.newConcurrentMap(); /* * lock to perform atomic operations. @@ -80,6 +82,15 @@ protected void addAlterJobV2(AlterJobV2 alterJob) { LOG.info("add {} job {}", alterJob.getType(), alterJob.getJobId()); } + public AlterJobV2 getAlterJobV2(long tblId) { + for (AlterJobV2 alterJob : alterJobsV2.values()) { + if (alterJob.getTableId() == tblId) { + return alterJob; + } + } + return null; + } + @Deprecated protected void addAlterJob(AlterJob alterJob) { this.alterJobs.put(alterJob.getTableId(), alterJob); diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index d90ec75d078292..574511e3b65127 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -106,18 +106,21 @@ public long getTableId() { return tableId; } - protected void setCancelled(String errMsg) { - jobState = JobState.CANCELLED; - this.errMsg = errMsg; - LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); - // edit log will be wrote later - } - private boolean isTimeout() { return System.currentTimeMillis() - createTimeMs > timeoutMs; } - public void run() { + public boolean isDone() { + return jobState.isFinalState(); + } + + /* + * The keyword 'synchronized' only protects 2 method: + * run() and cancel() + * Only these 2 methods can be visited by different thread(internal working thread and user connection thread) + * So using 'synchronized' to make sure only one thread can run the job at one time. + */ + public synchronized void run() { if (isTimeout()) { cancel("Timeout"); } @@ -149,7 +152,7 @@ protected void runRunningJob() { throw new NotImplementedException(); } - protected void cancel(String errMsg) { + public synchronized void cancel(String errMsg) { throw new NotImplementedException(); } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index f000cc28cf3808..e2db70492b2e11 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -47,9 +47,6 @@ import org.apache.doris.common.util.Util; import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.EditLog; -import org.apache.doris.task.AgentBatchTask; -import org.apache.doris.task.AgentTaskExecutor; -import org.apache.doris.task.DropReplicaTask; import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Preconditions; @@ -297,6 +294,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl for (Partition partition : olapTable.getPartitions()) { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + // index state is SHADOW MaterializedIndex rollupIndex = new MaterializedIndex(rollupIndexId, IndexState.SHADOW); MaterializedIndex baseIndex = partition.getIndex(baseIndexId); TabletMeta rollupTabletMeta = new TabletMeta(dbId, tableId, partitionId, rollupIndexId, rollupSchemaHash, medium); @@ -343,10 +341,9 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl public void processDropRollup(DropRollupClause alterClause, Database db, OlapTable olapTable) throws DdlException { - // make sure we got db write lock here - // up to here, table's state can be NORMAL or ROLLUP - Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL - || olapTable.getState() == OlapTableState.ROLLUP, olapTable.getState().name()); + // make sure we got db write lock here. + // up to here, table's state can only be NORMAL. + Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); String rollupIndexName = alterClause.getRollupName(); if (rollupIndexName.equals(olapTable.getName())) { @@ -356,74 +353,37 @@ public void processDropRollup(DropRollupClause alterClause, Database db, OlapTab long dbId = db.getId(); long tableId = olapTable.getId(); if (!olapTable.hasMaterializedIndex(rollupIndexName)) { - // when rollup job is unfinished, rollup index is not added to the table - AlterJob alterJob = getAlterJob(tableId); - if (alterJob == null || !((RollupJob) alterJob).getRollupIndexName().equals(rollupIndexName)) { - throw new DdlException("Rollup index[" + rollupIndexName + "] does not exist in table[" - + olapTable.getName() + "]"); - } - - // cancel rollup job - cancelInternal(alterJob, olapTable, "rollup index is dropped"); - return; + throw new DdlException("Rollup index[" + rollupIndexName + "] does not exist in table[" + + olapTable.getName() + "]"); } - - // 1. check if any rollup job is based on this index - AlterJob alterJob = null; - if ((alterJob = checkIfAnyRollupBasedOn(tableId, rollupIndexName)) != null) { - throw new DdlException("Rollup index[" + ((RollupJob) alterJob).getRollupIndexName() - + "] is doing rollup based on this index[" + rollupIndexName + "] and not finished yet."); - } - - // if the index is a during rollup and in finishing state, then it could not be dropped - // because the finishing state could not be roll back, it is very difficult - alterJob = getAlterJob(tableId); - if (alterJob != null && ((RollupJob) alterJob).getRollupIndexName().equals(rollupIndexName) - && alterJob.getState() == JobState.FINISHING) { - throw new DdlException("Rollup index[" + rollupIndexName + "] in table[" - + olapTable.getName() + "] is in finishing state, waiting it to finish"); - } - // drop rollup for each partition long rollupIndexId = olapTable.getIndexIdByName(rollupIndexName); int rollupSchemaHash = olapTable.getSchemaHashByIndexId(rollupIndexId); Preconditions.checkState(rollupSchemaHash != -1); - Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL); + // drop rollup for each partition. + // also remove tablets from inverted index. TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - AgentBatchTask batchTask = new AgentBatchTask(); for (Partition partition : olapTable.getPartitions()) { MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); Preconditions.checkNotNull(rollupIndex); - // 2. delete rollup index + // delete rollup index partition.deleteRollupIndex(rollupIndexId); - // 3. send DropReplicaTask + // remove tablets from inverted index for (Tablet tablet : rollupIndex.getTablets()) { long tabletId = tablet.getId(); - List replicas = tablet.getReplicas(); - for (Replica replica : replicas) { - long backendId = replica.getBackendId(); - DropReplicaTask dropTask = new DropReplicaTask(backendId, tabletId, rollupSchemaHash); - batchTask.addTask(dropTask); - } // end for replicas - - // remove from inverted index invertedIndex.deleteTablet(tabletId); - } // end for tablets - } // end for partitions + } + } olapTable.deleteIndexInfo(rollupIndexName); - AgentTaskExecutor.submit(batchTask); - // 5. log drop rollup operation + // log drop rollup operation EditLog editLog = Catalog.getInstance().getEditLog(); DropInfo dropInfo = new DropInfo(dbId, tableId, rollupIndexId); editLog.logDropRollup(dropInfo); - LOG.debug("log drop rollup index[{}] finished in table[{}]", dropInfo.getIndexId(), - dropInfo.getTableId()); - LOG.info("finished drop rollup index[{}] in table[{}]", rollupIndexName, olapTable.getName()); } @@ -492,12 +452,10 @@ private void runAlterJobV2() { while (iter.hasNext()) { Map.Entry entry = iter.next(); AlterJobV2 alterJob = entry.getValue(); - alterJob.run(); - if (alterJob.getJobState().isFinalState()) { - iter.remove(); - finishedOrCancelledAlterJobsV2.offer(alterJob); - Catalog.getCurrentCatalog().getEditLog().logAlterJob(alterJob); + if (alterJob.isDone()) { + continue; } + alterJob.run(); } } @@ -683,6 +641,7 @@ public void cancel(CancelStmt stmt) throws DdlException { } AlterJob rollupJob = null; + AlterJobV2 rollupJobV2 = null; db.writeLock(); try { Table table = db.getTable(tableName); @@ -698,53 +657,29 @@ public void cancel(CancelStmt stmt) throws DdlException { + "Use 'ALTER TABLE DROP ROLLUP' if you want to."); } - rollupJob = getAlterJob(olapTable.getId()); - Preconditions.checkNotNull(rollupJob); - - if (rollupJob.getState() == JobState.FINISHED || rollupJob.getState() == JobState.CANCELLED) { - throw new DdlException("job is already " + rollupJob.getState().name() + ", can not cancel it"); + // find from new alter jobs first + rollupJobV2 = getAlterJobV2(olapTable.getId()); + if (rollupJobV2 != null) { + rollupJobV2.cancel("user cancelled"); + } else { + rollupJob = getAlterJob(olapTable.getId()); + Preconditions.checkNotNull(rollupJob, olapTable.getId()); + if (rollupJob.getState() == JobState.FINISHED || rollupJob.getState() == JobState.CANCELLED) { + throw new DdlException("job is already " + rollupJob.getState().name() + ", can not cancel it"); + } + rollupJob.cancel(olapTable, "user cancelled"); } - - rollupJob.cancel(olapTable, "user cancelled"); } finally { db.writeUnlock(); } - jobDone(rollupJob); + if (rollupJob != null) { + jobDone(rollupJob); + } } + // replay the rollup job by job state public void replayRollupJobV2(RollupJobV2 rollupJob) { - switch (rollupJob.getJobState()) { - case PENDING: - // PENDING job SHOULD NOT be in alterJobsV2 before - Preconditions.checkState(!alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); - rollupJob.replayPending(); - addAlterJobV2(rollupJob); - break; - case WAITING_TXN: - // WAITING_TXN job SHOULD be in alterJobsV2 before - Preconditions.checkState(alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); - rollupJob.replayWaitingTxn(); - // this add operation will replace the old rollup job with the new one - addAlterJobV2(rollupJob); - break; - case CANCELLED: - // CANCELLED job SHOULD be in alterJobsV2 before - Preconditions.checkState(alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); - rollupJob.replayCancelled(); - break; - case FINISHED: - // FINISHED job SHOULD be in alterJobsV2 before - Preconditions.checkState(alterJobsV2.containsKey(rollupJob.getJobId()), rollupJob.getJobId()); - rollupJob.replayFinished(); - break; - default: - break; - } - - if (rollupJob.getJobState().isFinalState()) { - alterJobsV2.remove(rollupJob.getJobId()); - finishedOrCancelledAlterJobsV2.offer(rollupJob); - } + rollupJob.replay(); } } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 8474297b600464..71c232b7f50242 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -26,7 +26,6 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; @@ -190,7 +189,7 @@ protected void runPendingJob() { Partition.PARTITION_INIT_VERSION, Partition.PARTITION_INIT_VERSION_HASH, rollupKeysType, TStorageType.COLUMN, storageMedium, rollupSchema, tbl.getCopiedBfColumns(), tbl.getBfFpp(), countDownLatch); - createReplicaTask.setBaseTabletId(tabletIdMap.get(rollupTabletId)); + createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); batchTask.addTask(createReplicaTask); } // end for rollupReplicas @@ -417,9 +416,18 @@ protected void runRunningJob() { } @Override - protected void cancel(String errMsg) { + public synchronized void cancel(String errMsg) { + if (jobState.isFinalState()) { + return; + } + cancelInternal(); - setCancelled(errMsg); + + jobState = JobState.CANCELLED; + this.errMsg = errMsg; + this.finishedTimeMs = System.currentTimeMillis(); + LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); + Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); } private void cancelInternal() { @@ -542,7 +550,7 @@ public void readFields(DataInput in) throws IOException { * Should replay all changes before this job's state transfer to PENDING. * These changes should be same as changes in RollupHander.processAddRollup() */ - public void replayPending() { + private void replayPending() { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return @@ -559,7 +567,6 @@ public void replayPending() { TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); for (Long partitionId : partitionIdToRollupIndex.keySet()) { - Partition partition = tbl.getPartition(partitionId); MaterializedIndex rollupIndex = partitionIdToRollupIndex.get(partitionId); TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); TabletMeta rollupTabletMeta = new TabletMeta(dbId, tableId, partitionId, rollupIndexId, @@ -571,8 +578,6 @@ public void replayPending() { invertedIndex.addReplica(rollupTablet.getId(), rollupReplica); } } - - partition.setState(PartitionState.ROLLUP); } tbl.setState(OlapTableState.ROLLUP); } finally { @@ -585,7 +590,7 @@ public void replayPending() { * Replay job in WAITING_TXN state. * Should replay all changes in runPendingJob() */ - public void replayWaitingTxn() { + private void replayWaitingTxn() { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return @@ -610,7 +615,7 @@ public void replayWaitingTxn() { * Replay job in FINISHED state. * Should replay all changes in runRuningJob() */ - public void replayFinished() { + private void replayFinished() { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db != null) { db.writeLock(); @@ -635,9 +640,27 @@ public void replayFinished() { /* * Replay job in CANCELLED state. */ - public void replayCancelled() { + private void replayCancelled() { cancelInternal(); LOG.info("replay cancelled rollup job: {}", jobId); } + public void replay() { + switch (jobState) { + case PENDING: + replayPending(); + break; + case WAITING_TXN: + replayWaitingTxn(); + break; + case FINISHED: + replayFinished(); + break; + case CANCELLED: + replayCancelled(); + break; + default: + break; + } + } } diff --git a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java index ce05f9e76547c5..c2cf514fd57f58 100644 --- a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -60,6 +60,7 @@ public class CreateReplicaTask extends AgentTask { // if base tablet id is set, BE will create the replica on same disk as this base tablet private long baseTabletId = -1; + private int baseSchemaHash = -1; public CreateReplicaTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, short shortKeyColumnCount, int schemaHash, long version, long versionHash, @@ -103,8 +104,9 @@ public void setInRestoreMode(boolean inRestoreMode) { this.inRestoreMode = inRestoreMode; } - public void setBaseTabletId(long baseTabletId) { + public void setBaseTablet(long baseTabletId, int baseSchemaHash) { this.baseTabletId = baseTabletId; + this.baseSchemaHash = baseSchemaHash; } public TCreateTabletReq toThrift() { @@ -145,6 +147,7 @@ public TCreateTabletReq toThrift() { if (baseTabletId != -1) { createTabletReq.setBase_tablet_id(baseTabletId); + createTabletReq.setBase_schema_hash(baseSchemaHash); } return createTabletReq; From 1e2ed4db0ef6b3a773a4ee038bef38c56a8b54b0 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 12 Jul 2019 17:59:54 +0800 Subject: [PATCH 03/79] third commit --- .../org/apache/doris/alter/AlterJobV2.java | 15 +++- .../org/apache/doris/alter/RollupHandler.java | 72 +++++++++---------- .../org/apache/doris/alter/RollupJob.java | 2 + .../org/apache/doris/alter/RollupJobV2.java | 67 +++++++++++++---- .../org/apache/doris/catalog/Catalog.java | 12 ---- .../doris/common/proc/RollupJobProcDir.java | 40 +++-------- .../doris/common/proc/RollupProcDir.java | 20 +++--- .../common/proc/RollupTabletsProcNode.java | 62 ---------------- .../org/apache/doris/master/MasterImpl.java | 9 +++ .../apache/doris/master/ReportHandler.java | 5 +- .../org/apache/doris/task/AgentBatchTask.java | 31 ++++++-- .../apache/doris/task/CreateRollupTaskV2.java | 23 ++++-- .../org/apache/doris/alter/RollupJobTest.java | 8 +-- gensrc/thrift/Types.thrift | 3 +- 14 files changed, 178 insertions(+), 191 deletions(-) delete mode 100644 fe/src/main/java/org/apache/doris/common/proc/RollupTabletsProcNode.java diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 574511e3b65127..ceed8fe110a12a 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -29,6 +29,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.List; /* * Author: Chenmingyu @@ -65,17 +66,19 @@ public enum JobType { protected long dbId; protected long tableId; + protected String tableName; protected String errMsg = ""; protected long createTimeMs = -1; protected long finishedTimeMs = -1; protected long timeoutMs = -1; - public AlterJobV2(long jobId, JobType jobType, long dbId, long tableId, long timeoutMs) { + public AlterJobV2(long jobId, JobType jobType, long dbId, long tableId, String tableName, long timeoutMs) { this.jobId = jobId; this.type = jobType; this.dbId = dbId; this.tableId = tableId; + this.tableName = tableName; this.timeoutMs = timeoutMs; this.createTimeMs = System.currentTimeMillis(); @@ -106,6 +109,10 @@ public long getTableId() { return tableId; } + public String getTableName() { + return tableName; + } + private boolean isTimeout() { return System.currentTimeMillis() - createTimeMs > timeoutMs; } @@ -115,7 +122,7 @@ public boolean isDone() { } /* - * The keyword 'synchronized' only protects 2 method: + * The keyword 'synchronized' only protects 2 methods: * run() and cancel() * Only these 2 methods can be visited by different thread(internal working thread and user connection thread) * So using 'synchronized' to make sure only one thread can run the job at one time. @@ -156,6 +163,10 @@ public synchronized void cancel(String errMsg) { throw new NotImplementedException(); } + protected void getInfo(List info) { + throw new NotImplementedException(); + } + public static AlterJobV2 read(DataInput in) throws IOException { JobType type = JobType.valueOf(Text.readString(in)); switch (type) { diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index e2db70492b2e11..24374f44405a0b 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -39,7 +39,6 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -65,6 +64,9 @@ import java.util.Map; import java.util.Set; +/* + * RollupHandler is responsible for ADD/DROP rollup. + */ public class RollupHandler extends AlterHandler { private static final Logger LOG = LogManager.getLogger(RollupHandler.class); @@ -72,6 +74,14 @@ public RollupHandler() { super("rollup"); } + /* + * Handle the Add Rollup request. + * 3 main steps: + * 1. Validate the request. + * 2. Create RollupJob with rollup index + * All replicas of the rollup index will be created in meta and added to TabletInvertedIndex + * 3. Set table's state to ROLLUP. + */ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTable olapTable) throws DdlException { @@ -285,8 +295,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl long jobId = catalog.getNextId(); long rollupIndexId = catalog.getNextId(); - - RollupJobV2 rollupJob = new RollupJobV2(jobId, dbId, tableId, timeoutMs, + RollupJobV2 rollupJob = new RollupJobV2(jobId, dbId, tableId, olapTable.getName(), timeoutMs, baseIndexId, rollupIndexId, baseIndexName, rollupIndexName, rollupSchema, baseSchemaHash, rollupSchemaHash, rollupKeysType, rollupShortKeyColumnCount); @@ -412,32 +421,7 @@ public void replayDropRollup(DropInfo dropInfo, Catalog catalog) { } finally { db.writeUnlock(); } - } - - public void removeReplicaRelatedTask(long tableId, long partitionId, long indexId, long tabletId, long backendId) { - // make sure to get db writeLock - AlterJob alterJob = checkIfAnyRollupBasedOn(tableId, indexId); - if (alterJob != null) { - alterJob.removeReplicaRelatedTask(partitionId, tabletId, -1L, backendId); - } - } - - // this is for handle delete replica op - private AlterJob checkIfAnyRollupBasedOn(long tableId, long baseIndexId) { - AlterJob alterJob = this.alterJobs.get(tableId); - if (alterJob != null && ((RollupJob) alterJob).getBaseIndexId() == baseIndexId) { - return alterJob; - } - return null; - } - - // this is for drop rollup op - private AlterJob checkIfAnyRollupBasedOn(long tableId, String baseIndexName) { - AlterJob alterJob = this.alterJobs.get(tableId); - if (alterJob != null && ((RollupJob) alterJob).getBaseIndexName().equals(baseIndexName)) { - return alterJob; - } - return null; + LOG.info("replay drop rollup {}", dropInfo.getIndexId()); } @Override @@ -570,8 +554,29 @@ private void runOldAlterJob() { @Override public List> getAlterJobInfosByDb(Database db) { List> rollupJobInfos = new LinkedList>(); - List jobs = Lists.newArrayList(); + getOldAlterJobInfos(db, rollupJobInfos); + getAlterJobV2Infos(rollupJobInfos); + + // sort by + // "JobId", "TableName", "CreateTime", "FinishedTime", "BaseIndexName", "RollupIndexName" + ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4, 5); + Collections.sort(rollupJobInfos, comparator); + + return rollupJobInfos; + } + + private void getAlterJobV2Infos(List> rollupJobInfos) { + for (AlterJobV2 alterJob : alterJobsV2.values()) { + List info = Lists.newArrayList(); + alterJob.getInfo(info); + rollupJobInfos.add(info); + } + } + + @Deprecated + private void getOldAlterJobInfos(Database db, List> rollupJobInfos) { + List jobs = Lists.newArrayList(); // lock to perform atomically lock(); try { @@ -603,13 +608,6 @@ public List> getAlterJobInfosByDb(Database db) { } finally { db.readUnlock(); } - - // sort by - // "JobId", "TableName", "CreateTime", "FinishedTime", "BaseIndexName", "RollupIndexName" - ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4, 5); - Collections.sort(rollupJobInfos, comparator); - - return rollupJobInfos; } @Override diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJob.java b/fe/src/main/java/org/apache/doris/alter/RollupJob.java index b135eea2ebe226..f674a111ee7e1d 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJob.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJob.java @@ -31,6 +31,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.common.Config; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.io.Text; @@ -1006,6 +1007,7 @@ public void getJobInfo(List> jobInfos, OlapTable tbl) { } else { jobInfo.add("N/A"); } + jobInfo.add(Config.alter_table_timeout_second); jobInfos.add(jobInfo); } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 71c232b7f50242..51d246881f6b5a 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -33,7 +33,9 @@ import org.apache.doris.common.Config; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.io.Text; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.CreateReplicaTask; @@ -94,11 +96,11 @@ public class RollupJobV2 extends AlterJobV2 { // save all create rollup tasks private AgentBatchTask rollupBatchTask = new AgentBatchTask(); - public RollupJobV2(long jobId, long dbId, long tableId, long timeoutMs, + public RollupJobV2(long jobId, long dbId, long tableId, String tableName, long timeoutMs, long baseIndexId, long rollupIndexId, String baseIndexName, String rollupIndexName, List rollupSchema, int baseSchemaHash, int rollupSchemaHash, KeysType rollupKeysType, short rollupShortKeyColumnCount) { - super(jobId, JobType.ROLLUP, dbId, tableId, timeoutMs); + super(jobId, JobType.ROLLUP, dbId, tableId, tableName, timeoutMs); this.baseIndexId = baseIndexId; this.rollupIndexId = rollupIndexId; @@ -134,7 +136,7 @@ public void addRollupIndex(long partitionId, MaterializedIndex rollupIndex) { * 1. Create all rollup replicas and wait them finished. * 2. After creating done, set the rollup index state as SHADOW, add it to catalog, user can not see this * rollup, but internal load process will generate data for this rollup index. - * 3. get a new transaction id, then set job's state to WAITING_TXN + * 3. Get a new transaction id, then set job's state to WAITING_TXN */ @Override protected void runPendingJob() { @@ -256,16 +258,16 @@ private void addRollupIndexToCatalog(OlapTable tbl) { partition.createRollupIndex(rollupIndex); } - tbl.setIndexSchemaInfo(rollupIndexId, rollupIndexName, rollupSchema, 0, + tbl.setIndexSchemaInfo(rollupIndexId, rollupIndexName, rollupSchema, 0 /* init schema version */, rollupSchemaHash, rollupShortKeyColumnCount); tbl.setStorageTypeToIndex(rollupIndexId, TStorageType.COLUMN); } /* * runWaitingTxnJob(): - * 1. Wait the previous transactions to be finished. - * 2. If all previous transactions finished, send create rollup tasks to BE - * 3. Change job state to RUNNING + * 1. Wait the transactions before the watershedTxnId to be finished. + * 2. If all previous transactions finished, send create rollup tasks to BE. + * 3. Change job state to RUNNING. */ @Override protected void runWaitingTxnJob() { @@ -294,13 +296,10 @@ protected void runWaitingTxnJob() { for (Map.Entry entry : this.partitionIdToRollupIndex.entrySet()) { long partitionId = entry.getKey(); Partition partition = tbl.getPartition(partitionId); - if (partition == null) { - continue; - } + Preconditions.checkNotNull(partition, partitionId); // the rollup task will transform the data before committed version. - // DO NOT use visible version because we need to handle the committed but not published version - // on BE. + // DO NOT use visible version because we need to handle the committed but not published version on BE. long committedVersion = partition.getCommittedVersion(); long committedVersionHash = partition.getCommittedVersionHash(); @@ -317,7 +316,7 @@ protected void runWaitingTxnJob() { rollupIndexId, baseIndexId, rollupTabletId, baseTabletId, rollupReplica.getId(), rollupSchemaHash, baseSchemaHash, - committedVersion, committedVersionHash); + committedVersion, committedVersionHash, jobId); rollupBatchTask.addTask(rollupTask); } } @@ -330,7 +329,7 @@ protected void runWaitingTxnJob() { AgentTaskExecutor.submit(rollupBatchTask); this.jobState = JobState.RUNNING; - // do not write edit log here, tasks will be send again if FE restart or master changed. + // DO NOT write edit log here, tasks will be send again if FE restart or master changed. LOG.info("transfer rollup job {} state to {}", jobId, this.jobState); } @@ -546,7 +545,7 @@ public void readFields(DataInput in) throws IOException { } /* - * Replay job in pending state. + * Replay job in PENDING state. * Should replay all changes before this job's state transfer to PENDING. * These changes should be same as changes in RollupHander.processAddRollup() */ @@ -565,6 +564,7 @@ private void replayPending() { return; } + // add all rollup replicas to tablet inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); for (Long partitionId : partitionIdToRollupIndex.keySet()) { MaterializedIndex rollupIndex = partitionIdToRollupIndex.get(partitionId); @@ -663,4 +663,41 @@ public void replay() { break; } } + + @Override + protected void getInfo(List info) { + info.add(jobId); + info.add(tableName); + info.add(TimeUtils.longToTimeString(createTimeMs)); + info.add(TimeUtils.longToTimeString(finishedTimeMs)); + info.add(baseIndexName); + info.add(rollupIndexName); + info.add(rollupIndexId); + info.add(watershedTxnId); + info.add(jobState.name()); + info.add(errMsg); + // progress + if (jobState == JobState.RUNNING && rollupBatchTask.getTaskNum() > 0) { + info.add(rollupBatchTask.getFinishedTaskNum() + "/" + rollupBatchTask.getTaskNum()); + } else { + info.add("N/A"); + } + info.add(timeoutMs / 1000); + } + + public List> getUnfinishedTasks(int limit) { + List> taskInfos = Lists.newArrayList(); + if (jobState == JobState.RUNNING) { + List tasks = rollupBatchTask.getUnfinishedTasks(limit); + for (AgentTask agentTask : tasks) { + CreateRollupTaskV2 rollupTask = (CreateRollupTaskV2)agentTask; + List info = Lists.newArrayList(); + info.add(String.valueOf(rollupTask.getBackendId())); + info.add(String.valueOf(rollupTask.getBaseTabletId())); + info.add(String.valueOf(rollupTask.getSignature())); + taskInfos.add(info); + } + } + return taskInfos; + } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 26330385e36b6f..8601d42a1a6245 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -4181,18 +4181,6 @@ public void replayRecoverTable(RecoverInfo info) { } } - public void handleJobsWhenDeleteReplica(long tableId, long partitionId, long indexId, long tabletId, long replicaId, - long backendId) { - // rollup - getRollupHandler().removeReplicaRelatedTask(tableId, partitionId, indexId, tabletId, backendId); - - // schema change - getSchemaChangeHandler().removeReplicaRelatedTask(tableId, tabletId, replicaId, backendId); - - // task - AgentTaskQueue.removeReplicaRelatedTasks(backendId, tabletId); - } - private void unprotectAddReplica(ReplicaPersistInfo info) { LOG.debug("replay add a replica {}", info); Database db = getDb(info.getDbId()); diff --git a/fe/src/main/java/org/apache/doris/common/proc/RollupJobProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/RollupJobProcDir.java index 8e2096367d7bf1..741e0038a45d6f 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/RollupJobProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/RollupJobProcDir.java @@ -17,23 +17,23 @@ package org.apache.doris.common.proc; -import org.apache.doris.alter.RollupJob; +import org.apache.doris.alter.RollupJobV2; import org.apache.doris.common.AnalysisException; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; -public class RollupJobProcDir implements ProcDirInterface { +// Show unfinished rollup tasks of rollup job v2 +public class RollupJobProcDir implements ProcNodeInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("PartitionId").add("RollupIndexId").add("IndexState") + .add("BackendId").add("BaseTabletId").add("RollupTabletId") .build(); - private RollupJob rollupJob; + private RollupJobV2 rollupJob; - public RollupJobProcDir(RollupJob rollupJob) { + public RollupJobProcDir(RollupJobV2 rollupJob) { this.rollupJob = rollupJob; } @@ -44,32 +44,8 @@ public ProcResult fetchResult() throws AnalysisException { BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); - List> rollupJobInfos = rollupJob.getInfos(); - for (List infoStr : rollupJobInfos) { - List oneInfo = new ArrayList(TITLE_NAMES.size()); - for (Comparable element : infoStr) { - oneInfo.add(element.toString()); - } - result.addRow(oneInfo); - } + List> unfinishedRollupTasks = rollupJob.getUnfinishedTasks(2000); + result.setRows(unfinishedRollupTasks); return result; } - - @Override - public boolean register(String name, ProcNodeInterface node) { - return false; - } - - @Override - public ProcNodeInterface lookup(String partitionIdStr) throws AnalysisException { - long partitionId; - try { - partitionId = Long.valueOf(partitionIdStr); - } catch (NumberFormatException e) { - throw new AnalysisException("Invalid table id format: " + partitionIdStr); - } - - return new RollupTabletsProcNode(rollupJob, partitionId); - } - } diff --git a/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java index 42c31523c65819..c290f55f0010ca 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java @@ -17,9 +17,9 @@ package org.apache.doris.common.proc; -import org.apache.doris.alter.AlterJob; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.alter.RollupHandler; -import org.apache.doris.alter.RollupJob; +import org.apache.doris.alter.RollupJobV2; import org.apache.doris.catalog.Database; import org.apache.doris.common.AnalysisException; @@ -34,7 +34,7 @@ public class RollupProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("JobId").add("TableName").add("CreateTime").add("FinishedTime") .add("BaseIndexName").add("RollupIndexName").add("RollupId").add("TransactionId") - .add("State").add("Msg") .add("Progress") + .add("State").add("Msg").add("Progress").add("Timeout") .build(); private RollupHandler rollupHandler; @@ -70,25 +70,25 @@ public boolean register(String name, ProcNodeInterface node) { } @Override - public ProcNodeInterface lookup(String tableIdStr) throws AnalysisException { - if (Strings.isNullOrEmpty(tableIdStr)) { + public ProcNodeInterface lookup(String jobIdStr) throws AnalysisException { + if (Strings.isNullOrEmpty(jobIdStr)) { throw new AnalysisException("Table id is null"); } - long tableId = -1L; + long jobId = -1L; try { - tableId = Long.valueOf(tableIdStr); + jobId = Long.valueOf(jobIdStr); } catch (Exception e) { throw new AnalysisException("Table id is invalid"); } - Preconditions.checkState(tableId != -1L); - AlterJob job = rollupHandler.getAlterJob(tableId); + Preconditions.checkState(jobId != -1L); + AlterJobV2 job = rollupHandler.getAlterJobV2(jobId); if (job == null) { return null; } - return new RollupJobProcDir((RollupJob) job); + return new RollupJobProcDir((RollupJobV2) job); } } diff --git a/fe/src/main/java/org/apache/doris/common/proc/RollupTabletsProcNode.java b/fe/src/main/java/org/apache/doris/common/proc/RollupTabletsProcNode.java deleted file mode 100644 index 08ad1ff195bd42..00000000000000 --- a/fe/src/main/java/org/apache/doris/common/proc/RollupTabletsProcNode.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.proc; - -import org.apache.doris.alter.RollupJob; -import org.apache.doris.common.AnalysisException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -public class RollupTabletsProcNode implements ProcNodeInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("TabletId").add("ReplicaId").add("BackendId").add("Version") - .add("VersionHash").add("DataSize").add("RowCount").add("State") - .build(); - - private RollupJob rollupJob; - private long partitionId; - - public RollupTabletsProcNode(RollupJob rollupJob, long partitionId) { - this.rollupJob = rollupJob; - this.partitionId = partitionId; - } - - @Override - public ProcResult fetchResult() throws AnalysisException { - Preconditions.checkNotNull(rollupJob); - - BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); - - List> tabletInfos = rollupJob.getRollupIndexInfo(partitionId); - for (int i = 0; i < tabletInfos.size(); i++) { - List info = tabletInfos.get(i); - List row = new ArrayList(info.size()); - for (int j = 0; j < info.size(); j++) { - row.add(info.get(j).toString()); - } - result.addRow(row); - } - return result; - } - -} diff --git a/fe/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/src/main/java/org/apache/doris/master/MasterImpl.java index a94b4e22485c82..770059dffa2812 100644 --- a/fe/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/src/main/java/org/apache/doris/master/MasterImpl.java @@ -46,6 +46,7 @@ import org.apache.doris.task.CloneTask; import org.apache.doris.task.CreateReplicaTask; import org.apache.doris.task.CreateRollupTask; +import org.apache.doris.task.CreateRollupTaskV2; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; import org.apache.doris.task.PublishVersionTask; @@ -199,6 +200,9 @@ public TMasterResult finishTask(TFinishTaskRequest request) throws TException { case RECOVER_TABLET: finishRecoverTablet(task); break; + case ALTER: + finishAlterTask(task); + break; default: break; } @@ -766,4 +770,9 @@ public TFetchResourceResult fetchResource() { return Catalog.getInstance().getAuth().toResourceThrift(); } + private void finishAlterTask(AgentTask task) { + CreateRollupTaskV2 createRollupTaskV2 = (CreateRollupTaskV2) task; + createRollupTaskV2.setFinished(true); + AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); + } } diff --git a/fe/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/src/main/java/org/apache/doris/master/ReportHandler.java index 462ddde216144e..4d05660c9f16c0 100644 --- a/fe/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/src/main/java/org/apache/doris/master/ReportHandler.java @@ -591,9 +591,8 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta tablet.deleteReplicaByBackendId(backendId); ++deleteCounter; - // handle related task - Catalog.getInstance().handleJobsWhenDeleteReplica(tableId, partitionId, indexId, tabletId, - replica.getId(), backendId); + // remove replica related tasks + AgentTaskQueue.removeReplicaRelatedTasks(backendId, tabletId); // write edit log ReplicaPersistInfo info = ReplicaPersistInfo.createForDelete(dbId, tableId, partitionId, diff --git a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java index 71d16ae1a28414..0a34403332bb97 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java +++ b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java @@ -19,12 +19,13 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.common.ClientPool; -import org.apache.doris.common.Pair; import org.apache.doris.system.Backend; import org.apache.doris.thrift.BackendService; import org.apache.doris.thrift.TAgentServiceVersion; import org.apache.doris.thrift.TAgentTaskRequest; import org.apache.doris.thrift.TAlterTabletReq; +import org.apache.doris.thrift.TAlterTabletReqV2; +import org.apache.doris.thrift.TCancelDeleteDataReq; import org.apache.doris.thrift.TCheckConsistencyReq; import org.apache.doris.thrift.TClearAlterTaskRequest; import org.apache.doris.thrift.TClearTransactionTaskRequest; @@ -118,15 +119,14 @@ public boolean isFinished() { return true; } - // return the limit number of unfinished task. - // backend id -> signature - public List> getUnfinishedTasks(int limit) { - List> res = Lists.newArrayList(); + // return the limit number of unfinished tasks. + public List getUnfinishedTasks(int limit) { + List res = Lists.newArrayList(); for (List tasks : this.backendIdToTasks.values()) { for (AgentTask agentTask : tasks) { if (!agentTask.isFinished()) { if (res.size() < limit) { - res.add(Pair.create(agentTask.getBackendId(), agentTask.getSignature())); + res.add(agentTask); } } } @@ -134,6 +134,18 @@ public List> getUnfinishedTasks(int limit) { return res; } + public int getFinishedTaskNum() { + int count = 0; + for (List tasks : this.backendIdToTasks.values()) { + for (AgentTask agentTask : tasks) { + if (agentTask.isFinished()) { + count++; + } + } + } + return count; + } + @Override public void run() { for (Long backendId : this.backendIdToTasks.keySet()) { @@ -348,6 +360,13 @@ private TAgentTaskRequest toAgentTaskRequest(AgentTask task) { LOG.debug(request.toString()); } tAgentTaskRequest.setUpdate_tablet_meta_info_req(request); + case ALTER: { + CreateRollupTaskV2 createRollupTask = (CreateRollupTaskV2) task; + TAlterTabletReqV2 request = createRollupTask.toThrift(); + if (LOG.isDebugEnabled()) { + LOG.debug(request.toString()); + } + tAgentTaskRequest.setAlter_tablet_req_v2(request); return tAgentTaskRequest; } default: diff --git a/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java b/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java index 9a76d9e3f85ea4..86be00a1035cda 100644 --- a/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java +++ b/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java @@ -17,7 +17,7 @@ package org.apache.doris.task; -import org.apache.doris.thrift.TAlterTabletReq; +import org.apache.doris.thrift.TAlterTabletReqV2; import org.apache.doris.thrift.TTaskType; /* @@ -33,12 +33,13 @@ public class CreateRollupTaskV2 extends AgentTask { private int rollupSchemaHash; private long version; private long versionHash; + private long jobId; public CreateRollupTaskV2(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId, long baseIndexId, long rollupTabletId, long baseTabletId, long rollupReplicaId, int rollupSchemaHash, int baseSchemaHash, - long version, long versionHash) { - super(null, backendId, TTaskType.ROLLUP, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); + long version, long versionHash, long jobId) { + super(null, backendId, TTaskType.ALTER, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); this.baseTabletId = baseTabletId; this.rollupReplicaId = rollupReplicaId; @@ -48,10 +49,7 @@ public CreateRollupTaskV2(long backendId, long dbId, long tableId, this.version = version; this.versionHash = versionHash; - } - - public TAlterTabletReq toThrift() { - return null; + this.jobId = jobId; } public long getBaseTabletId() { @@ -77,4 +75,15 @@ public long getVersion() { public long getVersionHash() { return versionHash; } + + public long getJobId() { + return jobId; + } + + public TAlterTabletReqV2 toThrift() { + TAlterTabletReqV2 req = new TAlterTabletReqV2(baseTabletId, signature, baseSchemaHash, rollupSchemaHash); + req.setAlter_version(version); + req.setAlter_version_hash(versionHash); + return req; + } } diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java index 284046f4c952c9..d635d38fb0d2b5 100644 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java @@ -116,7 +116,7 @@ public void testAddRollup() throws Exception { alterClauses.add(clause); Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - rollupHandler.process(alterClauses, db, olapTable); + rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); @@ -134,7 +134,7 @@ public void testRollup1() throws Exception { Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db, olapTable); + rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); @@ -226,7 +226,7 @@ public void testRollup2() throws Exception { Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db, olapTable); + rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); @@ -284,7 +284,7 @@ public void testRollup3() throws Exception { Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db, olapTable, false); + rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index c354e89390b725..b88a9de3814868 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -162,7 +162,8 @@ enum TTaskType { RECOVER_TABLET, STREAM_LOAD, UPDATE_TABLET_META_INFO, - ALTER_TASK + // this type of task will replace both ROLLUP and SCHEMA_CHANGE + ALTER } enum TStmtType { From 305605e93ef7e256e4aad32799fb6845c6606304 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 16 Jul 2019 08:40:57 +0800 Subject: [PATCH 04/79] schema change first commit --- .../org/apache/doris/alter/RollupJobV2.java | 14 +- .../doris/alter/SchemaChangeHandler.java | 333 +++++++++--------- .../org/apache/doris/alter/SystemHandler.java | 2 +- .../java/org/apache/doris/catalog/Column.java | 10 +- .../apache/doris/master/ReportHandler.java | 6 + 5 files changed, 185 insertions(+), 180 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 51d246881f6b5a..eda193caf135e0 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -333,6 +333,13 @@ protected void runWaitingTxnJob() { LOG.info("transfer rollup job {} state to {}", jobId, this.jobState); } + /* + * runRunningJob() + * 1. Wait all create rollup tasks to be finished. + * 2. Check the integrity of the newly created rollup index. + * 3. Set rollup index's state to NORMAL to let it visible to query. + * 4. Set job'state as FINISHED. + */ @Override protected void runRunningJob() { Preconditions.checkState(jobState == JobState.RUNNING, jobState); @@ -414,6 +421,10 @@ protected void runRunningJob() { LOG.info("rollup job finished: {}", jobId); } + /* + * cancel() can be called any time any place. + * We need to clean any possible residual of this job. + */ @Override public synchronized void cancel(String errMsg) { if (jobState.isFinalState()) { @@ -430,7 +441,7 @@ public synchronized void cancel(String errMsg) { } private void cancelInternal() { - // clear tasks if have + // clear tasks if has AgentTaskQueue.removeBatchTask(rollupBatchTask, TTaskType.ROLLUP); // remove all rollup indexes, and set state to NORMAL TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); @@ -456,6 +467,7 @@ private void cancelInternal() { } } + // Check whether transactions of the given database which txnId is less than 'watershedTxnId' are finished. protected boolean isPreviousLoadFinished() { return Catalog.getCurrentGlobalTransactionMgr().isPreviousTransactionsFinished(watershedTxnId, dbId); } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 074ae8cd76f31c..3fa4ce995028e3 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -41,13 +41,13 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -55,8 +55,7 @@ import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TResourceInfo; +import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import com.google.common.base.Preconditions; @@ -92,10 +91,6 @@ private void processAddColumn(AddColumnClause alterClause, OlapTable olapTable, String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); - if (column.isKey()) { - checkKeyModificationIfInRandomDistributedTable(olapTable); - } - String baseIndexName = olapTable.getName(); checkAssignedTargetIndexName(baseIndexName, targetIndexName); @@ -118,9 +113,6 @@ private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable Set newColNameSet = Sets.newHashSet(); for (Column column : columns) { - if (column.isKey()) { - checkKeyModificationIfInRandomDistributedTable(olapTable); - } newColNameSet.add(column.getName()); } @@ -145,27 +137,28 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); - Column dropColumn = olapTable.getColumn(dropColName); - if (dropColumn != null && dropColumn.isKey()) { - checkKeyModificationIfInRandomDistributedTable(olapTable); - } - String baseIndexName = olapTable.getName(); checkAssignedTargetIndexName(baseIndexName, targetIndexName); + /* + * UNIQUE: + * Can not drop any key column. + * AGGREGATION: + * Can not drp any key column is has value with REPLACE method + */ if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { long baseIndexId = olapTable.getBaseIndexId(); List baseSchema = indexSchemaMap.get(baseIndexId); boolean isKey = false; for (Column column : baseSchema) { - if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { + if (column.isKey() && column.getName().equals(dropColName)) { isKey = true; break; } } if (isKey) { - throw new DdlException("key column of unique key table cannot be droped"); + throw new DdlException("Can not drop key column in Unique data model table"); } } else if (KeysType.AGG_KEYS == olapTable.getKeysType()) { @@ -176,38 +169,38 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable boolean isKey = false; boolean hasReplaceColumn = false; for (Column column : baseSchema) { - if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { + if (column.isKey() && column.getName().equals(dropColName)) { isKey = true; } else if (AggregateType.REPLACE == column.getAggregationType()) { hasReplaceColumn = true; } } if (isKey && hasReplaceColumn) { - throw new DdlException("key column of table with replace aggregation method cannot be droped"); + throw new DdlException("Can not drop key column when table has value column with REPLACE aggregation method"); } } else { - // drop column in rollup and basetable + // drop column in rollup and base index long targetIndexId = olapTable.getIndexIdByName(targetIndexName); // find column List targetIndexSchema = indexSchemaMap.get(targetIndexId); boolean isKey = false; boolean hasReplaceColumn = false; for (Column column : targetIndexSchema) { - if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { + if (column.isKey() && column.getName().equals(dropColName)) { isKey = true; } else if (AggregateType.REPLACE == column.getAggregationType()) { hasReplaceColumn = true; } } if (isKey && hasReplaceColumn) { - throw new DdlException("key column of table with replace aggregation method cannot be droped"); + throw new DdlException("Can not drop key column when rollup has value column with REPLACE aggregation metho"); } } } long baseIndexId = olapTable.getBaseIndexId(); if (targetIndexName == null) { - // drop base index and all rollup indices's column + // if not specify rollup index, column should be dropped from both base and rollup indexes. List indexIds = new ArrayList(); indexIds.add(baseIndexId); for (long indexId : olapTable.getIndexIdToSchema().keySet()) { @@ -223,14 +216,14 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable Iterator baseIter = baseSchema.iterator(); while (baseIter.hasNext()) { Column column = baseIter.next(); - if (column.getName().equalsIgnoreCase(dropColName)) { + if (column.getName().equals(dropColName)) { baseIter.remove(); found = true; break; } } if (!found) { - throw new DdlException("Column[" + dropColName + "] does not exists"); + throw new DdlException("Column does not exists: " + dropColName); } // remove column in rollup index if exists (i = 1 to skip base index) @@ -239,14 +232,14 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable Iterator iter = rollupSchema.iterator(); while (iter.hasNext()) { Column column = iter.next(); - if (column.getName().equalsIgnoreCase(dropColName)) { + if (column.getName().equals(dropColName)) { iter.remove(); break; } } } // end for index names } else { - // only drop column from specified rollup index + // if specify rollup index, only drop column from specified rollup index long targetIndexId = olapTable.getIndexIdByName(targetIndexName); // find column List targetIndexSchema = indexSchemaMap.get(targetIndexId); @@ -254,40 +247,42 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable Iterator iter = targetIndexSchema.iterator(); while (iter.hasNext()) { Column column = iter.next(); - if (column.getName().equalsIgnoreCase(dropColName)) { + if (column.getName().equals(dropColName)) { iter.remove(); found = true; break; } } if (!found) { - throw new DdlException("Column[" + dropColName + "] does not exists"); + throw new DdlException("Column does not exists: " + dropColName); } } } + // User can modify column type and column position private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapTable, Map> indexSchemaMap) throws DdlException { Column modColumn = alterClause.getColumn(); if (KeysType.AGG_KEYS == olapTable.getKeysType()) { if (modColumn.isKey() && null != modColumn.getAggregationType()) { - throw new DdlException("key column of aggregate key table cannot use aggregation method"); + throw new DdlException("Can not assign aggregation method on key column: " + modColumn.getName()); } else if (null == modColumn.getAggregationType()) { - // in aggregate key table, no aggreation method indicate key column + // in aggregate key table, no aggregation method indicate key column modColumn.setIsKey(true); } } else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { if (null != modColumn.getAggregationType()) { - throw new DdlException("column of unique key table cannot use aggregation method"); + throw new DdlException("Can not assign aggregation method on column in Unique data model table: " + modColumn.getName()); } if (false == modColumn.isKey()) { modColumn.setAggregationType(AggregateType.REPLACE, true); } } else { + // DUPLICATE table also has key and value column. if (null != modColumn.getAggregationType()) { - throw new DdlException("column of duplicate key table cannot use aggregation method"); + throw new DdlException("Can not assign aggregation method on column in Duplicate data model table: " + modColumn.getName()); } - if (false == modColumn.isKey()) { + if (!modColumn.isKey()) { modColumn.setAggregationType(AggregateType.NONE, true); } } @@ -295,10 +290,6 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); - if (modColumn.isKey()) { - checkKeyModificationIfInRandomDistributedTable(olapTable); - } - String baseIndexName = olapTable.getName(); checkAssignedTargetIndexName(baseIndexName, targetIndexName); @@ -322,12 +313,12 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT int lastColIndex = -1; for (int i = 0; i < schemaForFinding.size(); i++) { Column col = schemaForFinding.get(i); - if (col.getName().equalsIgnoreCase(newColName)) { + if (col.getName().equals(newColName)) { modColIndex = i; found = true; } if (hasColPos) { - if (col.getName().equalsIgnoreCase(columnPos.getLastCol())) { + if (col.getName().equals(columnPos.getLastCol())) { lastColIndex = i; } } else { @@ -372,8 +363,6 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT } else { schemaForFinding.set(modColIndex, modColumn); } - int temp = modColIndex; - Column tempCol = schemaForFinding.get(temp); // check if column being mod if (!modColumn.equals(oriColumn)) { @@ -396,8 +385,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT } } - if (KeysType.AGG_KEYS == olapTable.getKeysType() - || KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { + if (KeysType.AGG_KEYS == olapTable.getKeysType() || KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { for (Long otherIndexId : otherIndexIds) { List otherIndexSchema = indexSchemaMap.get(otherIndexId); modColIndex = -1; @@ -412,6 +400,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT otherIndexSchema.set(modColIndex, modColumn); } // end for other indices } else { + // DUPLICATE data model has a little for (Long otherIndexId : otherIndexIds) { List otherIndexSchema = indexSchemaMap.get(otherIndexId); modColIndex = -1; @@ -434,7 +423,6 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT } otherIndexSchema.set(modColIndex, otherCol); } - tempCol = schemaForFinding.get(temp); } } // end for handling other indices } @@ -445,13 +433,6 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); - for (String colName : orderedColNames) { - Column reorderdCol = olapTable.getColumn(colName); - if (reorderdCol != null && reorderdCol.isKey()) { - checkKeyModificationIfInRandomDistributedTable(olapTable); - } - } - String baseIndexName = olapTable.getName(); checkAssignedTargetIndexName(baseIndexName, targetIndexName); @@ -469,7 +450,7 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol for (String colName : orderedColNames) { Column oneCol = null; for (Column column : targetIndexSchema) { - if (column.getName().equalsIgnoreCase(colName)) { + if (column.getName().equals(colName)) { oneCol = column; break; } @@ -491,27 +472,34 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol indexSchemaMap.put(targetIndexId, newSchema); } + /* + * Add 'newColumn' to specified index. + * Modified schema will be saved in 'indexSchemaMap' + */ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos, long targetIndexId, long baseIndexId, String baseIndexName, Map> indexSchemaMap, Set newColNameSet) throws DdlException { + String newColName = newColumn.getName(); + // check the validation of aggregation method on column. + // also fill the default aggregation method if not specified. if (KeysType.AGG_KEYS == olapTable.getKeysType()) { if (newColumn.isKey() && newColumn.getAggregationType() != null) { - throw new DdlException("key column of aggregate table cannot use aggregation method"); + throw new DdlException("Can not assign aggregation method on key column: " + newColName); } else if (null == newColumn.getAggregationType()) { newColumn.setIsKey(true); } } else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { if (newColumn.getAggregationType() != null) { - throw new DdlException("column of unique table cannot use aggregation method"); + throw new DdlException("Can not assign aggregation method on column in Unique data model table: " + newColName); } if (!newColumn.isKey()) { newColumn.setAggregationType(AggregateType.REPLACE, true); } } else { if (newColumn.getAggregationType() != null) { - throw new DdlException("column of duplicate table cannot use aggregation method"); + throw new DdlException("Can not assign aggregation method on column in Duplicate data model table: " + newColName); } if (!newColumn.isKey()) { newColumn.setAggregationType(AggregateType.NONE, true); @@ -520,34 +508,45 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi // hll must be used in agg_keys if (newColumn.getType().isHllType() && KeysType.AGG_KEYS != olapTable.getKeysType()) { - throw new DdlException("HLL must be used in AGG_KEYS"); + throw new DdlException("HLL type column can only be in Aggregation data model table: " + newColName); } - if (newColumn.getAggregationType() == BITMAP_UNION && KeysType.AGG_KEYS != olapTable.getKeysType()) { + if (newColumn.getAggregationType() == BITMAP_UNION && KeysType.AGG_KEYS != olapTable.getKeysType()) { throw new DdlException("BITMAP_UNION must be used in AGG_KEYS"); } + // check if the new column already exist in base schema. + // do not support adding new column which already exist in base schema. List baseSchema = olapTable.getBaseSchema(); - String newColName = newColumn.getName(); boolean found = false; for (Column column : baseSchema) { - if (column.getName().equalsIgnoreCase(newColName)) { + if (column.getName().equals(newColName)) { found = true; break; } } if (found) { - throw new DdlException("Column[" + newColName + "] already exists in base index[" + baseIndexName + "]"); + throw new DdlException("Can not add column which already exists in base table: " + newColName); } + /* + * add new column to indexes. + * UNIQUE: + * 1. If new column is key, it should be added to all indexes. + * 2. Else, add the new column to base index and specified rollup index. + * DUPLICATE: + * 1. If not specify rollup index, just add it to base index. + * 2. Else, first add it to specify rollup index. Then if the new column is key, add it to base + * index, at the end of all other existing key columns. If new new column is value, add it to + * base index by user specified position. + * AGGREGATION: + * 1. Add it to base index, as well as specified rollup index. + */ if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { - // check if has default value. this should be done in Analyze phase - // 1. add to base index first List modIndexSchema; if (newColumn.isKey()) { - // add key column to unique key table, should add to all rollups - // Column column = olapTable.getColumn(columnPos.getLastCol()); - // add to all table including base and rollup + // add key column to unique key table + // add to all indexes including base and rollup for (Map.Entry> entry : indexSchemaMap.entrySet()) { modIndexSchema = entry.getValue(); boolean isBaseIdex = entry.getKey() == baseIndexId; @@ -560,28 +559,26 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi if (targetIndexId == -1L) { return; } - // 2. add to rollup modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); } } else if (KeysType.DUP_KEYS == olapTable.getKeysType()) { if (targetIndexId == -1L) { - // check if has default value. this should be done in Analyze phase - // 1. add to base index first + // add to base index List modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true); // no specified target index. return return; } else { - // 2. add to rollup index + // add to rollup index List modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); if (newColumn.isKey()) { /* * if add column in rollup is key, - * then put the column in base table as end key + * then put the column in base table as the last key column */ modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, null, newColNameSet, true); @@ -607,6 +604,15 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi } } + /* + * add new column to specified index schema('modIndexSchema'). + * if 'isBaseIndex' is true, which means 'modIndexSchema' is base index's schema. + * so we will not check repeat adding of column. + * For example, user want to add column k1 to both rollup1 and rollup2 in one alter stmt: + * ADD COLUMN k1 int to rollup1, + * ADD COLUMN k1 int to rollup2 + * So that k1 will be added to base index 'twice', and we just ignore this repeat adding. + */ private void checkAndAddColumn(List modIndexSchema, Column newColumn, ColumnPosition columnPos, Set newColNameSet, boolean isBaseIndex) throws DdlException { int posIndex = -1; @@ -614,16 +620,16 @@ private void checkAndAddColumn(List modIndexSchema, Column newColumn, Co boolean hasPos = (columnPos != null && !columnPos.isFirst()); for (int i = 0; i < modIndexSchema.size(); i++) { Column col = modIndexSchema.get(i); - if (col.getName().equalsIgnoreCase(newColName)) { + if (col.getName().equals(newColName)) { if (!isBaseIndex || !newColNameSet.contains(newColName)) { // if this is not a base index, we should check if user repeatedly add columns - throw new DdlException("Repeatedly add column[" + newColName + "]"); + throw new DdlException("Repeatedly add column: " + newColName); } // this is a base index, and the column we check here is added by previous 'add column clause' // in same ALTER stmt. // so here we will check if the 2 columns is exactly same. if not, throw exception if (!col.equals(newColumn)) { - throw new DdlException("Repeatedly add same column[" + newColName + "] with different definition"); + throw new DdlException("Repeatedly add same column with different definition: " + newColName); } // column already exist, return @@ -632,7 +638,7 @@ private void checkAndAddColumn(List modIndexSchema, Column newColumn, Co if (hasPos) { // after the field - if (col.getName().equalsIgnoreCase(columnPos.getLastCol())) { + if (col.getName().equals(columnPos.getLastCol())) { posIndex = i; } } else { @@ -669,16 +675,7 @@ private void checkAndAddColumn(List modIndexSchema, Column newColumn, Co checkRowLength(modIndexSchema); } - private void checkKeyModificationIfInRandomDistributedTable(OlapTable olapTable) throws DdlException { - for (Partition partition : olapTable.getPartitions()) { - DistributionInfo distributionInfo = partition.getDistributionInfo(); - if (distributionInfo.getType() == DistributionInfoType.RANDOM) { - throw new DdlException("Cannot add/del/reorder/modify key column " - + "in table which is distributed by random"); - } - } - } - + // row length can not large than limit private void checkRowLength(List modIndexSchema) throws DdlException { int rowLengthBytes = 0; for (Column column : modIndexSchema) { @@ -798,15 +795,9 @@ private void createJob(long dbId, OlapTable olapTable, Map 0) { - // just skip it (replica cloned from old schema will be deleted) - continue; - } - ++replicaNum; - } // end for replicas - - if (replicaNum < replicationNum / 2 + 1) { - String errMsg = "Tablet[" + tablet.getId() + "] does not have enough replicas. [" - + replicaNum + "/" + replicationNum + "]"; - LOG.warn(errMsg); - throw new DdlException(errMsg); - } - } // end for tablets - } // end for partitions - - // 6. calc short key + // 5. calc short key short newShortKeyColumnCount = Catalog.calcShortKeyColumnCount(alterSchema, indexIdToProperties.get(alterIndexId)); LOG.debug("alter index[{}] short key column count: {}", alterIndexId, newShortKeyColumnCount); indexIdToShortKeyColumnCount.put(alterIndexId, newShortKeyColumnCount); - // 7. check storage type if has null column - TStorageType storageType = olapTable.getStorageTypeByIndexId(alterIndexId); - boolean hasNullColumn = false; - for (Column column : alterSchema) { - if (column.isAllowNull()) { - hasNullColumn = true; - break; - } - } - if (hasNullColumn && storageType != TStorageType.COLUMN) { - throw new DdlException("Only column rollup support null columns"); - } - - // 8. store the changed columns for edit log + // 6. store the changed columns for edit log schemaChangeJob.putToChangedIndexSchemaMap(alterIndexId, alterSchema); LOG.debug("schema change[{}-{}-{}] check pass.", dbId, tableId, alterIndexId); @@ -1031,28 +973,54 @@ private void createJob(long dbId, OlapTable olapTable, Map> entry : schemaChangeJob.getChangedIndexToSchema().entrySet()) { - long indexId = entry.getKey(); - MaterializedIndex alterIndex = onePartition.getIndex(indexId); - Preconditions.checkState(alterIndex.getState() == IndexState.NORMAL, alterIndex.getState()); - - // set new schema - int currentSchemaVersion = olapTable.getSchemaVersionByIndexId(indexId); - int newSchemaVersion = currentSchemaVersion + 1; - List alterColumns = entry.getValue(); - // int newSchemaHash = Util.schemaHash(newSchemaVersion, alterColumns, bfColumns, bfFpp); - // new schema hash should only be generate one time, or the schema hash will differenent from each other in different partitions - if (newSchemaHash == -1) { - newSchemaHash = Util.generateSchemaHash(); - int currentSchemaHash = olapTable.getSchemaHashByIndexId(indexId); - // has to generate a new schema hash not equal to current schema hash - while (currentSchemaHash == newSchemaHash) { - newSchemaHash = Util.generateSchemaHash(); + /* + * Create schema change job + * 1. For each index which has been changed, create a SHADOW index, and save the mapping of origin index to SHADOW index. + * 2. Create all tablets and replicas of all SHADOW index, add them to tablet inverted index. + * 3. Change table's state as SCHEMA_CHANGE + */ + Catalog catalog = Catalog.getCurrentCatalog(); + for (Map.Entry> entry : schemaChangeJob.getChangedIndexToSchema().entrySet()) { + long originIndexId = entry.getKey(); + // 1. get new schema version/schema version hash, short key column count + int currentSchemaVersion = olapTable.getSchemaVersionByIndexId(originIndexId); + int newSchemaVersion = currentSchemaVersion + 1; + // generate schema hash for new index has to generate a new schema hash not equal to current schema hash + int currentSchemaHash = olapTable.getSchemaHashByIndexId(originIndexId); + int newSchemaHash = Util.generateSchemaHash(); + while (currentSchemaHash == newSchemaHash) { + newSchemaHash = Util.generateSchemaHash(); + } + short newShortKeyColumnCount = indexIdToShortKeyColumnCount.get(originIndexId); + long shadowIndexId = catalog.getNextId(); + + // create SHADOW index for each partition + for (Partition partition : olapTable.getPartitions()) { + long partitionId = partition.getId(); + TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + // index state is SHADOW + MaterializedIndex shadowIndex = new MaterializedIndex(shadowIndexId, IndexState.SHADOW); + MaterializedIndex originIndex = partition.getIndex(originIndexId); + TabletMeta shadowTabletMeta = new TabletMeta(dbId, tableId, partitionId, shadowIndexId, newSchemaHash, medium); + for (Tablet originTablet : originIndex.getTablets()) { + long originTabletId = originTablet.getId(); + long shadowTabletId = catalog.getNextId(); + + Tablet shadowTablet = new Tablet(shadowTabletId); + shadowIndex.addTablet(shadowTablet, shadowTabletMeta); + + schemaChangeJob.addTabletIdMap(partitionId, shadowIndexId, shadowTabletId, originTabletId); + List originReplicas = originTablet.getReplicas(); + + for (Replica originReplica : originReplicas) { + long shadowReplicaId = catalog.getNextId(); + long backendId = originReplica.getBackendId(); + Preconditions.checkState(originReplica.getState() == ReplicaState.NORMAL); + Replica rollupReplica = new Replica(shadowReplicaId, backendId, newSchemaHash, ReplicaState.NORMAL); + shadowTablet.addReplica(rollupReplica); } } +<<<<<<< HEAD short newShortKeyColumnCount = indexIdToShortKeyColumnCount.get(indexId); schemaChangeJob.setNewSchemaInfo(indexId, newSchemaVersion, newSchemaHash, newShortKeyColumnCount); @@ -1073,17 +1041,15 @@ private void createJob(long dbId, OlapTable olapTable, Map>>>>>> schema change first commit - alterIndex.setState(IndexState.SCHEMA_CHANGE); - } // end for indices - - onePartition.setState(PartitionState.SCHEMA_CHANGE); - } // end for partitions - - olapTable.setState(OlapTableState.SCHEMA_CHANGE); + schemaChangeJob.addShadowIndex(partitionId, originIndexId, shadowIndex); + } // end for partition + } // end for index // 2. add schemaChangeJob - addAlterJob(schemaChangeJob); + addAlterJobV2(schemaChangeJob); // 3. log schema change start operation Catalog.getInstance().getEditLog().logStartSchemaChange(schemaChangeJob); @@ -1116,6 +1082,24 @@ public void removeReplicaRelatedTask(long tableId, long tabletId, long replicaId @Override protected void runOneCycle() { super.runOneCycle(); + runOldAlterJob(); + runAlterJobV2(); + } + + private void runAlterJobV2() { + Iterator> iter = alterJobsV2.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + AlterJobV2 alterJob = entry.getValue(); + if (alterJob.isDone()) { + continue; + } + alterJob.run(); + } + } + + @Deprecated + private void runOldAlterJob() { List cancelledJobs = Lists.newArrayList(); List finishedJobs = Lists.newArrayList(); @@ -1284,6 +1268,9 @@ public void process(List alterClauses, String clusterName, Database throw new DdlException("reduplicated PROPERTIES"); } + // modification of colocate property is handle alone. + // And because there should be only one colocate property modification clause in stmt, + // so just return after finished handling. if (properties.containsKey(PropertyAnalyzer.PROPERTIES_COLOCATE_WITH)) { String colocateGroup = properties.get(PropertyAnalyzer.PROPERTIES_COLOCATE_WITH); Catalog.getInstance().modifyTableColocate(db, olapTable, colocateGroup, false, null); diff --git a/fe/src/main/java/org/apache/doris/alter/SystemHandler.java b/fe/src/main/java/org/apache/doris/alter/SystemHandler.java index 470488e628a79b..ba237ec510c4a7 100644 --- a/fe/src/main/java/org/apache/doris/alter/SystemHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SystemHandler.java @@ -38,8 +38,8 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.common.Config; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; diff --git a/fe/src/main/java/org/apache/doris/catalog/Column.java b/fe/src/main/java/org/apache/doris/catalog/Column.java index 52067b9434fded..ed46953f69725c 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/src/main/java/org/apache/doris/catalog/Column.java @@ -214,24 +214,24 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException { } if (!ColumnType.isSchemaChangeAllowed(type, other.type)) { - throw new DdlException("Cannot change " + getDataType() + " to " + other.getDataType()); + throw new DdlException("Can not change " + getDataType() + " to " + other.getDataType()); } if (this.aggregationType != other.aggregationType) { - throw new DdlException("Cannot change aggregation type"); + throw new DdlException("Can not change aggregation type"); } if (this.isAllowNull && !other.isAllowNull) { - throw new DdlException("Cannot change from null to not null"); + throw new DdlException("Can not change from nullable to non-nullable"); } if (this.getDefaultValue() == null) { if (other.getDefaultValue() != null) { - throw new DdlException("Cannot change default value"); + throw new DdlException("Can not change default value"); } } else { if (!this.getDefaultValue().equals(other.getDefaultValue())) { - throw new DdlException("Cannot change default value"); + throw new DdlException("Can not change default value"); } } diff --git a/fe/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/src/main/java/org/apache/doris/master/ReportHandler.java index 4d05660c9f16c0..5225f54df5f749 100644 --- a/fe/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/src/main/java/org/apache/doris/master/ReportHandler.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -521,6 +522,11 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta if (index == null) { continue; } + if (index.getState() == IndexState.SHADOW) { + // This index is under schema change or rollup, tablet may not be created on BE. + // ignore it. + continue; + } Tablet tablet = index.getTablet(tabletId); if (tablet == null) { From 29a5847b5f058bc79f908489666208441930a4b6 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 17 Jul 2019 20:02:15 +0800 Subject: [PATCH 05/79] schema change second commit --- .../org/apache/doris/alter/AlterHandler.java | 3 +- .../org/apache/doris/alter/RollupJobV2.java | 13 +- .../doris/alter/SchemaChangeHandler.java | 116 ++- .../apache/doris/alter/SchemaChangeJobV2.java | 715 ++++++++++++++++++ .../org/apache/doris/catalog/OlapTable.java | 6 + .../org/apache/doris/catalog/Partition.java | 5 + 6 files changed, 780 insertions(+), 78 deletions(-) create mode 100644 fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index f813b9e429a928..7d264014c8f1cc 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -26,6 +26,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.Daemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.task.AgentTask; @@ -313,7 +314,7 @@ public void start() { * entry function. handle alter ops */ public abstract void process(List alterClauses, String clusterName, Database db, OlapTable olapTable) - throws DdlException; + throws UserException; /* * cancel alter ops diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index eda193caf135e0..a1d722ac2f58dc 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -134,7 +134,7 @@ public void addRollupIndex(long partitionId, MaterializedIndex rollupIndex) { /* * runPendingJob(): * 1. Create all rollup replicas and wait them finished. - * 2. After creating done, set the rollup index state as SHADOW, add it to catalog, user can not see this + * 2. After creating done, add this shadow rollup index to catalog, user can not see this * rollup, but internal load process will generate data for this rollup index. * 3. Get a new transaction id, then set job's state to WAITING_TXN */ @@ -228,7 +228,7 @@ protected void runPendingJob() { } // create all rollup replicas success. - // set rollup index’s state to SHADOW and add it to catalog + // add rollup index to catalog db.writeLock(); try { OlapTable tbl = (OlapTable) db.getTable(tableId); @@ -298,10 +298,9 @@ protected void runWaitingTxnJob() { Partition partition = tbl.getPartition(partitionId); Preconditions.checkNotNull(partition, partitionId); - // the rollup task will transform the data before committed version. - // DO NOT use visible version because we need to handle the committed but not published version on BE. - long committedVersion = partition.getCommittedVersion(); - long committedVersionHash = partition.getCommittedVersionHash(); + // the rollup task will transform the data before visible version(included). + long visibleVersion = partition.getVisibleVersion(); + long visibleVersionHash = partition.getVisibleVersionHash(); MaterializedIndex rollupIndex = entry.getValue(); Map tabletIdMap = this.partitionIdToBaseRollupTabletIdMap.get(partitionId); @@ -316,7 +315,7 @@ protected void runWaitingTxnJob() { rollupIndexId, baseIndexId, rollupTabletId, baseTabletId, rollupReplica.getId(), rollupSchemaHash, baseSchemaHash, - committedVersion, committedVersionHash, jobId); + visibleVersion, visibleVersionHash, jobId); rollupBatchTask.addTask(rollupTask); } } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 3fa4ce995028e3..54ca76b208bb95 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -52,15 +52,16 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.thrift.TStorageMedium; -import org.apache.doris.thrift.TStorageType; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; @@ -80,6 +81,9 @@ public class SchemaChangeHandler extends AlterHandler { private static final Logger LOG = LogManager.getLogger(SchemaChangeHandler.class); + // all shadow indexes should have this prefix in name + public static final String SHADOW_INDEX_NAME_PRFIX = "__doris_shadow_"; + public SchemaChangeHandler() { super("schema change"); } @@ -688,8 +692,24 @@ private void checkRowLength(List modIndexSchema) throws DdlException { } } + private void checkIndexExists(OlapTable olapTable, String targetIndexName) throws DdlException { + if (targetIndexName != null && !olapTable.hasMaterializedIndex(targetIndexName)) { + throw new DdlException("Index[" + targetIndexName + "] does not exist in table[" + olapTable.getName() + + "]"); + } + } + + private void checkAssignedTargetIndexName(String baseIndexName, String targetIndexName) throws DdlException { + // user cannot assign base index to do schema change + if (targetIndexName != null) { + if (targetIndexName.equals(baseIndexName)) { + throw new DdlException("Do not need to assign base index[" + baseIndexName + "] to do schema change"); + } + } + } + private void createJob(long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap) throws DdlException { + Map propertyMap) throws UserException { if (olapTable.getState() == OlapTableState.ROLLUP) { throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); } @@ -790,20 +810,20 @@ private void createJob(long dbId, OlapTable olapTable, Map indexIdToShortKeyColumnCount = new HashMap(); + Map indexIdToShortKeyColumnCount = Maps.newHashMap(); + Map> changedIndexIdToSchema = Maps.newHashMap(); for (Long alterIndexId : indexSchemaMap.keySet()) { List originSchema = olapTable.getSchemaByIndexId(alterIndexId); List alterSchema = indexSchemaMap.get(alterIndexId); @@ -958,18 +978,15 @@ private void createJob(long dbId, OlapTable olapTable, Map> entry : schemaChangeJob.getChangedIndexToSchema().entrySet()) { + for (Map.Entry> entry : changedIndexIdToSchema.entrySet()) { long originIndexId = entry.getKey(); // 1. get new schema version/schema version hash, short key column count int currentSchemaVersion = olapTable.getSchemaVersionByIndexId(originIndexId); @@ -991,6 +1007,7 @@ private void createJob(long dbId, OlapTable olapTable, Map 0) { - // this should not happen, cause we only allow schema change when table is stable. - LOG.error("replica {} of tablet {} on backend {} is not NORMAL: {}", - replica.getId(), tablet.getId(), replica.getBackendId(), replica); - continue; - } - Preconditions.checkState(replica.getState() == ReplicaState.NORMAL, replica.getState()); - replica.setState(ReplicaState.SCHEMA_CHANGE); - } // end for replicas - } // end for tablets - - Catalog.getCurrentInvertedIndex().setNewSchemaHash(onePartition.getId(), indexId, newSchemaHash); -======= ->>>>>>> schema change first commit - - schemaChangeJob.addShadowIndex(partitionId, originIndexId, shadowIndex); + + schemaChangeJob.addPartitionShadowIndex(partitionId, shadowIndexId, shadowIndex); } // end for partition + schemaChangeJob.addIndexSchema(shadowIndexId, originIndexId, newIndexName, newSchemaVersion, newSchemaHash, newShortKeyColumnCount, entry.getValue()); } // end for index + + // set table state + olapTable.setState(OlapTableState.SCHEMA_CHANGE); // 2. add schemaChangeJob addAlterJobV2(schemaChangeJob); - // 3. log schema change start operation - Catalog.getInstance().getEditLog().logStartSchemaChange(schemaChangeJob); - LOG.info("schema change job created. table[{}]", olapTable.getName()); - } - - private void checkIndexExists(OlapTable olapTable, String targetIndexName) throws DdlException { - if (targetIndexName != null && !olapTable.hasMaterializedIndex(targetIndexName)) { - throw new DdlException("Index[" + targetIndexName + "] does not exist in table[" + olapTable.getName() - + "]"); - } - } - - private void checkAssignedTargetIndexName(String baseIndexName, String targetIndexName) throws DdlException { - // user cannot assign base index to do schema change - if (targetIndexName != null) { - if (targetIndexName.equals(baseIndexName)) { - throw new DdlException("Do not need to assign base index[" + baseIndexName + "] to do schema change"); - } - } - } - - public void removeReplicaRelatedTask(long tableId, long tabletId, long replicaId, long backendId) { - AlterJob job = getAlterJob(tableId); - if (job != null) { - job.removeReplicaRelatedTask(-1L, tabletId, replicaId, backendId); - } + // 3. write edit log + Catalog.getInstance().getEditLog().logAlterJob(schemaChangeJob); + LOG.info("finished to create schema change job: {}", schemaChangeJob.getJobId()); } @Override @@ -1250,7 +1225,8 @@ public List> getAlterJobInfosByDb(Database db) { } @Override - public void process(List alterClauses, String clusterName, Database db, OlapTable olapTable) throws DdlException { + public void process(List alterClauses, String clusterName, Database db, OlapTable olapTable) + throws UserException { // index id -> index schema Map> indexSchemaMap = new HashMap>(); for (Map.Entry> entry : olapTable.getIndexIdToSchema().entrySet()) { diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java new file mode 100644 index 00000000000000..1bcf2dabcdf932 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -0,0 +1,715 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.TabletInvertedIndex; +import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.common.Config; +import org.apache.doris.common.MarkedCountDownLatch; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.CreateReplicaTask; +import org.apache.doris.task.CreateRollupTaskV2; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TStorageType; +import org.apache.doris.thrift.TTaskType; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Table; +import com.google.common.collect.Table.Cell; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/* + * Author: Chenmingyu + * Date: Jul 8, 2019 + */ + +/* + * Version 2 of SchemaChangeJob. + * This is for replacing the old SchemaChangeJob + * https://github.com/apache/incubator-doris/issues/1429 + */ +public class SchemaChangeJobV2 extends AlterJobV2 { + private static final Logger LOG = LogManager.getLogger(SchemaChangeJobV2.class); + + // partition id -> (shadow index id -> (shadow tablet id -> origin tablet id)) + private Table> partitionIndexTabletMap = HashBasedTable.create(); + // partition id -> (shadow index id -> shadow index)) + private Table partitionIndexMap = HashBasedTable.create(); + // shadow index id -> origin index id + private Map indexIdMap = Maps.newHashMap(); + // shadow index id -> origin index name + private Map indexIdToName = Maps.newHashMap(); + // shadow index id -> index schema + private Map> indexSchemaMap = Maps.newHashMap(); + // shadow index id -> (shadow index schema version : schema hash) + private Map> indexSchemaVersionAndHashMap = Maps.newHashMap(); + // shadow index id -> shadow index short key count + private Map indexShortKeyMap = Maps.newHashMap(); + + // bloom filter info + private boolean hasBfChange; + private Set bfColumns; + private double bfFpp; + + // The schema change job will wait all transactions before this txn id finished, then send the schema change tasks. + protected long watershedTxnId = -1; + + // save all schema change tasks + private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); + + public SchemaChangeJobV2(long jobId, long dbId, long tableId, String tableName, long timeoutMs) { + super(jobId, JobType.SCHEMA_CHANGE, dbId, tableId, tableName, timeoutMs); + + } + + private SchemaChangeJobV2() { + super(JobType.SCHEMA_CHANGE); + } + + public void addTabletIdMap(long partitionId, long shadowIdxId, long shadowTabletId, long originTabletId) { + Map tabletMap = partitionIndexTabletMap.get(partitionId, shadowIdxId); + if (tabletMap == null) { + tabletMap = Maps.newHashMap(); + partitionIndexTabletMap.put(partitionId, shadowIdxId, tabletMap); + } + tabletMap.put(shadowTabletId, originTabletId); + } + + public void addPartitionShadowIndex(long partitionId, long shadowIdxId, MaterializedIndex shadowIdx) { + partitionIndexMap.put(partitionId, shadowIdxId, shadowIdx); + } + + public void addIndexSchema(long shadowIdxId, long originIdxId, + String shadowIndexName, int shadowSchemaVersion, int shadowSchemaHash, + short shadowIdxShortKeyCount, List shadowIdxSchema) { + indexIdMap.put(shadowIdxId, originIdxId); + indexIdToName.put(shadowIdxId, shadowIndexName); + indexSchemaVersionAndHashMap.put(shadowIdxId, Pair.create(shadowSchemaVersion, shadowSchemaHash)); + indexShortKeyMap.put(shadowIdxId, shadowIdxShortKeyCount); + indexSchemaMap.put(shadowIdxId, shadowIdxSchema); + } + + public void setBloomFilterInfo(boolean hasBfChange, Set bfColumns, double bfFpp) { + this.hasBfChange = hasBfChange; + this.bfColumns = bfColumns; + this.bfFpp = bfFpp; + } + + /* + * runPendingJob(): + * 1. Create all replicas of all shadow indexes and wait them finished. + * 2. After creating done, add the shadow indexes to catalog, user can not see this + * shadow index, but internal load process will generate data for these indexes. + * 3. Get a new transaction id, then set job's state to WAITING_TXN + */ + @Override + protected void runPendingJob() { + Preconditions.checkState(jobState == JobState.PENDING, jobState); + + LOG.info("begin to send create replica tasks. job: {}", jobId); + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancel("Databasee " + dbId + " does not exist"); + return; + } + + // 1. create replicas + AgentBatchTask batchTask = new AgentBatchTask(); + // count total replica num + int totalReplicaNum = 0; + for (MaterializedIndex shadowIdx : partitionIndexMap.values()) { + for (Tablet tablet : shadowIdx.getTablets()) { + totalReplicaNum += tablet.getReplicas().size(); + } + } + MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalReplicaNum); + db.readLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); + + for (long partitionId : partitionIndexMap.rowKeySet()) { + Partition partition = tbl.getPartition(partitionId); + if (partition == null) { + continue; + } + TStorageMedium storageMedium = tbl.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + + Map shadowIndexMap = partitionIndexMap.row(partitionId); + for (Map.Entry entry : shadowIndexMap.entrySet()) { + long shadowIdxId = entry.getKey(); + MaterializedIndex shadowIdx = entry.getValue(); + + short shadowShortKeyColumnCount = indexShortKeyMap.get(shadowIdxId); + List shadowSchema = indexSchemaMap.get(shadowIdxId); + int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).second; + int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId)); + + for (Tablet shadowTablet : shadowIdx.getTablets()) { + long shadowTabletId = shadowTablet.getId(); + List shadowReplicas = shadowTablet.getReplicas(); + for (Replica shadowReplica : shadowReplicas) { + long backendId = shadowReplica.getBackendId(); + + CreateReplicaTask createReplicaTask = new CreateReplicaTask( + backendId, dbId, tableId, partitionId, shadowIdxId, shadowTabletId, + shadowShortKeyColumnCount, shadowSchemaHash, + Partition.PARTITION_INIT_VERSION, Partition.PARTITION_INIT_VERSION_HASH, + tbl.getKeysType(), TStorageType.COLUMN, storageMedium, + shadowSchema, bfColumns, bfFpp, countDownLatch); + createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdx).get(shadowTabletId), originSchemaHash); + + batchTask.addTask(createReplicaTask); + } // end for rollupReplicas + } // end for rollupTablets + } + } + } finally { + db.readUnlock(); + } + + // send all tasks and wait them finished + AgentTaskQueue.addBatchTask(batchTask); + AgentTaskExecutor.submit(batchTask); + // max timeout is 1 min + long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, 60000); + boolean ok = false; + try { + ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOG.warn("InterruptedException: ", e); + ok = false; + } + + if (!ok) { + // create replicas failed. just cancel the job + // clear tasks and show the failed replicas to user + AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 10 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 10)); + String idStr = Joiner.on(", ").join(subList); + LOG.warn("failed to create replicas for job: {}, {}", jobId, idStr); + cancel("Create replicas failed. Error replicas: " + idStr); + return; + } + + // create all replicas success. + // add all shadow indexes to catalog + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); + addShadowIndexToCatalog(tbl); + } finally { + db.writeUnlock(); + } + + this.watershedTxnId = Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId(); + this.jobState = JobState.WAITING_TXN; + + // write edit log + Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + LOG.info("transfer schema change job {} state to {}, watershed txn id: {}", jobId, this.jobState, watershedTxnId); + } + + private void addShadowIndexToCatalog(OlapTable tbl) { + for (long partitionId : partitionIndexMap.rowKeySet()) { + Partition partition = tbl.getPartition(partitionId); + if (partition == null) { + continue; + } + Map shadowIndexMap = partitionIndexMap.row(partitionId); + for (MaterializedIndex shadowIndex : shadowIndexMap.values()) { + partition.createRollupIndex(shadowIndex); + } + } + + for (long shadowIdxId : indexIdMap.keySet()) { + tbl.setIndexSchemaInfo(shadowIdxId, indexIdToName.get(shadowIdxId), indexSchemaMap.get(shadowIdxId), + indexSchemaVersionAndHashMap.get(shadowIdxId).first, + indexSchemaVersionAndHashMap.get(shadowIdxId).second, + indexShortKeyMap.get(shadowIdxId)); + tbl.setStorageTypeToIndex(shadowIdxId, TStorageType.COLUMN); + } + } + + /* + * runWaitingTxnJob(): + * 1. Wait the transactions before the watershedTxnId to be finished. + * 2. If all previous transactions finished, send schema change tasks to BE. + * 3. Change job state to RUNNING. + */ + @Override + protected void runWaitingTxnJob() { + Preconditions.checkState(jobState == JobState.WAITING_TXN, jobState); + + if (!isPreviousLoadFinished()) { + LOG.info("wait transactions before {} to be finished, schema change job: {}", watershedTxnId, jobId); + return; + } + + LOG.info("previous transactions are all finished, begin to send schema change tasks. job: {}", jobId); + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancel("Databasee " + dbId + " does not exist"); + return; + } + + db.readLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); + + for (long partitionId : partitionIndexMap.rowKeySet()) { + Partition partition = tbl.getPartition(partitionId); + Preconditions.checkNotNull(partition, partitionId); + + // the rollup task will transform the data before visible version(included). + long committedVersion = partition.getCommittedVersion(); + long committedVersionHash = partition.getCommittedVersionHash(); + + Map shadowIndexMap = partitionIndexMap.row(partitionId); + for (Map.Entry entry : shadowIndexMap.entrySet()) { + long shadowIdxId = entry.getKey(); + MaterializedIndex shadowIdx = entry.getValue(); + + long originIdxId = indexIdMap.get(shadowIdxId); + int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).second; + int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId)); + + for (Tablet shadowTablet : shadowIdx.getTablets()) { + long shadowTabletId = shadowTablet.getId(); + long originTabletId = partitionIndexTabletMap.get(partitionId, shadowIdxId).get(shadowTabletId); + List shadowReplicas = shadowTablet.getReplicas(); + for (Replica shadowReplica : shadowReplicas) { + CreateRollupTaskV2 rollupTask = new CreateRollupTaskV2( + shadowReplica.getBackendId(), dbId, tableId, partitionId, + shadowIdxId, originIdxId, + shadowTabletId, originTabletId, shadowReplica.getId(), + shadowSchemaHash, originSchemaHash, + committedVersion, committedVersionHash, jobId); + schemaChangeBatchTask.addTask(rollupTask); + } + } + } + } // end for partitions + } finally { + db.readUnlock(); + } + + AgentTaskQueue.addBatchTask(schemaChangeBatchTask); + AgentTaskExecutor.submit(schemaChangeBatchTask); + this.jobState = JobState.RUNNING; + + // DO NOT write edit log here, tasks will be send again if FE restart or master changed. + LOG.info("transfer schema change job {} state to {}", jobId, this.jobState); + } + + /* + * runRunningJob() + * 1. Wait all schema change tasks to be finished. + * 2. Check the integrity of the newly created shadow indexes. + * 3. Replace the origin index with shadow index, and set shadow index's state as NORMAL to be visible to user. + * 4. Set job'state as FINISHED. + */ + @Override + protected void runRunningJob() { + Preconditions.checkState(jobState == JobState.RUNNING, jobState); + if (!schemaChangeBatchTask.isFinished()) { + LOG.info("schema change tasks not finished. job: {}", jobId); + return; + } + + /* + * all tasks are finished. check the integrity. + * we just check whether all new replicas are healthy. + */ + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancel("Databasee " + dbId + " does not exist"); + return; + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancel("Table " + tableId + " does not exist"); + return; + } + Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); + + for (long partitionId : partitionIndexMap.rowKeySet()) { + Partition partition = tbl.getPartition(partitionId); + Preconditions.checkNotNull(partition, partitionId); + + long visiableVersion = partition.getVisibleVersion(); + long visiableVersionHash = partition.getVisibleVersionHash(); + short expectReplicationNum = tbl.getPartitionInfo().getReplicationNum(partition.getId()); + + Map shadowIndexMap = partitionIndexMap.row(partitionId); + for (Map.Entry entry : shadowIndexMap.entrySet()) { + MaterializedIndex shadowIdx = entry.getValue(); + + for (Tablet shadowTablet : shadowIdx.getTablets()) { + List replicas = shadowTablet.getReplicas(); + int healthyReplicaNum = 0; + for (Replica replica : replicas) { + if (replica.getLastFailedVersion() < 0 + && replica.checkVersionCatchUp(visiableVersion, visiableVersionHash)) { + healthyReplicaNum++; + } + } + + if (healthyReplicaNum < expectReplicationNum / 2 + 1) { + LOG.warn("rollup tablet {} has few healthy replicas: {}, rollup job: {}", + shadowTablet.getId(), replicas, jobId); + cancel("shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); + return; + } + } // end for tablets + } + } // end for partitions + + // all partitions are good + // replace the origin index with shadow index, set index state as NORMAL + for (Partition partition : tbl.getPartitions()) { + // drop the origin index from partitions + for (Map.Entry entry : indexIdMap.entrySet()) { + long shadowIdxId = entry.getKey(); + long originIdxId = entry.getValue(); + // get index from catalog, not from 'partitionIdToRollupIndex'. + // because if this alter job is recovered from edit log, index in 'partitionIndexMap' + // is not the same object in catalog. So modification on that index can not reflect to the index + // in catalog. + MaterializedIndex shadowIdx = partition.getIndex(shadowIdxId); + Preconditions.checkNotNull(shadowIdx, shadowIdxId); + // base index need special handling + if (originIdxId == partition.getBaseIndex().getId()) { + partition.setBaseIndex(shadowIdx); + } + partition.deleteRollupIndex(originIdxId); + shadowIdx.setState(IndexState.NORMAL); + } + } + + // update index schema info in table + for (Map.Entry entry : indexIdMap.entrySet()) { + long shadowIdxId = entry.getKey(); + long originIdxId = entry.getValue(); + String shadowIdxName = tbl.getIndexNameById(shadowIdxId); + String originIdxName = tbl.getIndexNameById(originIdxId); + tbl.deleteIndexInfo(originIdxName); + // the shadow index name is '__doris_shadow_xxx', rename it to origin name 'xxx' + tbl.renameIndexForSchemaChange(shadowIdxName, originIdxName); + + if (originIdxId == tbl.getId()) { + // set base index + tbl.setNewBaseSchema(indexSchemaMap.get(shadowIdxId)); + } + } + + tbl.setState(OlapTableState.NORMAL); + } finally { + db.writeUnlock(); + } + + this.jobState = JobState.FINISHED; + this.finishedTimeMs = System.currentTimeMillis(); + + Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + LOG.info("rollup job finished: {}", jobId); + } + + /* + * cancel() can be called any time any place. + * We need to clean any possible residual of this job. + */ + @Override + public synchronized void cancel(String errMsg) { + if (jobState.isFinalState()) { + return; + } + + cancelInternal(); + + jobState = JobState.CANCELLED; + this.errMsg = errMsg; + this.finishedTimeMs = System.currentTimeMillis(); + LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); + Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + } + + private void cancelInternal() { + // clear tasks if has + AgentTaskQueue.removeBatchTask(schemaChangeBatchTask, TTaskType.ROLLUP); + // remove all rollup indexes, and set state to NORMAL + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db != null) { + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl != null) { + for (long partitionId : partitionIndexMap.rowKeySet()) { + Partition partition = tbl.getPartition(partitionId); + Preconditions.checkNotNull(partition, partitionId); + + Map shadowIndexMap = partitionIndexMap.row(partitionId); + for (Map.Entry entry : shadowIndexMap.entrySet()) { + MaterializedIndex shadowIdx = entry.getValue(); + for (Tablet shadowTablet : shadowIdx.getTablets()) { + invertedIndex.deleteTablet(shadowTablet.getId()); + } + partition.deleteRollupIndex(shadowIdx.getId()); + } + } + + tbl.setState(OlapTableState.NORMAL); + } + } finally { + db.writeUnlock(); + } + } + } + + // Check whether transactions of the given database which txnId is less than 'watershedTxnId' are finished. + protected boolean isPreviousLoadFinished() { + return Catalog.getCurrentGlobalTransactionMgr().isPreviousTransactionsFinished(watershedTxnId, dbId); + } + + public static SchemaChangeJobV2 read(DataInput in) throws IOException { + SchemaChangeJobV2 rollupJob = new SchemaChangeJobV2(); + rollupJob.readFields(in); + return rollupJob; + } + + @Override + public synchronized void write(DataOutput out) throws IOException { + super.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + } + + /* + * Replay job in PENDING state. + * Should replay all changes before this job's state transfer to PENDING. + * These changes should be same as changes in SchemaChangeHandler.createJob() + */ + private void replayPending() { + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + // database may be dropped before replaying this log. just return + return; + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + // table may be dropped before replaying this log. just return + return; + } + + TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); + for (Cell cell : partitionIndexMap.cellSet()) { + long partitionId = cell.getRowKey(); + long shadowIndexId = cell.getColumnKey(); + MaterializedIndex shadowIndex = cell.getValue(); + + TStorageMedium medium = tbl.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); + TabletMeta shadowTabletMeta = new TabletMeta(dbId, tableId, partitionId, shadowIndexId, + indexSchemaVersionAndHashMap.get(shadowIndexId).second, medium); + + for (Tablet shadownTablet : shadowIndex.getTablets()) { + invertedIndex.addTablet(shadownTablet.getId(), shadowTabletMeta); + for (Replica shadowReplica : shadownTablet.getReplicas()) { + invertedIndex.addReplica(shadownTablet.getId(), shadowReplica); + } + } + } + + // set table state + tbl.setState(OlapTableState.SCHEMA_CHANGE); + } finally { + db.writeUnlock(); + } + LOG.info("replay pending schema change job: {}", jobId); + } + + /* + * Replay job in WAITING_TXN state. + * Should replay all changes in runPendingJob() + */ + private void replayWaitingTxn() { + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + // database may be dropped before replaying this log. just return + return; + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + // table may be dropped before replaying this log. just return + return; + } + addShadowIndexToCatalog(tbl); + } finally { + db.writeUnlock(); + } + LOG.info("replay waiting txn schema change job: {}", jobId); + } + + /* + * Replay job in FINISHED state. + * Should replay all changes in runRuningJob() + */ + private void replayFinished() { + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db != null) { + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl != null) { + Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); + for (Partition partition : tbl.getPartitions()) { + MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); + Preconditions.checkNotNull(rollupIndex, rollupIndex); + rollupIndex.setState(IndexState.NORMAL); + } + tbl.setState(OlapTableState.NORMAL); + } + } finally { + db.writeUnlock(); + } + } + LOG.info("replay finished rollup job: {}", jobId); + } + + /* + * Replay job in CANCELLED state. + */ + private void replayCancelled() { + cancelInternal(); + LOG.info("replay cancelled rollup job: {}", jobId); + } + + public void replay() { + switch (jobState) { + case PENDING: + replayPending(); + break; + case WAITING_TXN: + replayWaitingTxn(); + break; + case FINISHED: + replayFinished(); + break; + case CANCELLED: + replayCancelled(); + break; + default: + break; + } + } + + @Override + protected void getInfo(List info) { + info.add(jobId); + info.add(tableName); + info.add(TimeUtils.longToTimeString(createTimeMs)); + info.add(TimeUtils.longToTimeString(finishedTimeMs)); + info.add(baseIndexName); + info.add(rollupIndexName); + info.add(rollupIndexId); + info.add(watershedTxnId); + info.add(jobState.name()); + info.add(errMsg); + // progress + if (jobState == JobState.RUNNING && schemaChangeBatchTask.getTaskNum() > 0) { + info.add(schemaChangeBatchTask.getFinishedTaskNum() + "/" + schemaChangeBatchTask.getTaskNum()); + } else { + info.add("N/A"); + } + info.add(timeoutMs / 1000); + } + + public List> getUnfinishedTasks(int limit) { + List> taskInfos = Lists.newArrayList(); + if (jobState == JobState.RUNNING) { + List tasks = schemaChangeBatchTask.getUnfinishedTasks(limit); + for (AgentTask agentTask : tasks) { + CreateRollupTaskV2 rollupTask = (CreateRollupTaskV2)agentTask; + List info = Lists.newArrayList(); + info.add(String.valueOf(rollupTask.getBackendId())); + info.add(String.valueOf(rollupTask.getBaseTabletId())); + info.add(String.valueOf(rollupTask.getSignature())); + taskInfos.add(info); + } + } + return taskInfos; + } +} diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 3467f0984eff30..a19cb3b3b6dbf7 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -258,6 +258,12 @@ public String getIndexNameById(long indexId) { return null; } + // this is only for schema change. + public void renameIndexForSchemaChange(String name, String newName) { + long idxId = indexNameToId.remove(name); + indexNameToId.put(newName, idxId); + } + public Status resetIdsForRestore(Catalog catalog, Database db, int restoreReplicationNum) { // table id id = catalog.getNextId(); diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 013ab752eb67d4..918bdc27789c96 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -181,6 +181,11 @@ public MaterializedIndex getBaseIndex() { return baseIndex; } + // this is now only for schema change job + public void setBaseIndex(MaterializedIndex baseIndex) { + this.baseIndex = baseIndex; + } + public long getNextVersion() { return nextVersion; } From 196763378c0ab29e1240d4074cc6f9930d61ac1f Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 22 Jul 2019 14:18:14 +0800 Subject: [PATCH 06/79] modify schema change --- .../java/org/apache/doris/alter/Alter.java | 3 +- .../org/apache/doris/alter/AlterJobV2.java | 2 +- .../org/apache/doris/alter/RollupHandler.java | 11 +- .../org/apache/doris/alter/RollupJob.java | 1 - .../org/apache/doris/alter/RollupJobV2.java | 36 +-- .../doris/alter/SchemaChangeHandler.java | 23 +- .../apache/doris/alter/SchemaChangeJob.java | 14 +- .../apache/doris/alter/SchemaChangeJobV2.java | 270 +++++++++++++----- .../org/apache/doris/catalog/Partition.java | 3 +- .../org/apache/doris/catalog/Replica.java | 5 +- .../common/proc/SchemaChangeProcNode.java | 4 +- 11 files changed, 253 insertions(+), 119 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/Alter.java b/fe/src/main/java/org/apache/doris/alter/Alter.java index 2100b3a25f1658..136afc199da9d3 100644 --- a/fe/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/src/main/java/org/apache/doris/alter/Alter.java @@ -45,6 +45,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; import com.google.common.base.Preconditions; @@ -73,7 +74,7 @@ public void start() { clusterHandler.start(); } - public void processAlterTable(AlterTableStmt stmt) throws DdlException { + public void processAlterTable(AlterTableStmt stmt) throws UserException { TableName dbTableName = stmt.getTbl(); String dbName = dbTableName.getDb(); final String clusterName = stmt.getClusterName(); diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index ceed8fe110a12a..948dd52c3027b4 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -163,7 +163,7 @@ public synchronized void cancel(String errMsg) { throw new NotImplementedException(); } - protected void getInfo(List info) { + protected void getInfo(List> infos) { throw new NotImplementedException(); } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 24374f44405a0b..9a784a0cd29518 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -300,6 +300,10 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl rollupSchema, baseSchemaHash, rollupSchemaHash, rollupKeysType, rollupShortKeyColumnCount); + /* + * create all rollup indexes. and set state. + * After setting, Tables' state will be RO + */ for (Partition partition : olapTable.getPartitions()) { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); @@ -327,7 +331,8 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl continue; } Preconditions.checkState(baseReplica.getState() == ReplicaState.NORMAL); - Replica rollupReplica = new Replica(rollupReplicaId, backendId, rollupSchemaHash, ReplicaState.NORMAL); + // replica's init state is ALTER, so that tablet report process will ignore its report + Replica rollupReplica = new Replica(rollupReplicaId, backendId, rollupSchemaHash, ReplicaState.ALTER); newTablet.addReplica(rollupReplica); } // end for baseReplica } // end for baseTablets @@ -568,9 +573,7 @@ public List> getAlterJobInfosByDb(Database db) { private void getAlterJobV2Infos(List> rollupJobInfos) { for (AlterJobV2 alterJob : alterJobsV2.values()) { - List info = Lists.newArrayList(); - alterJob.getInfo(info); - rollupJobInfos.add(info); + alterJob.getInfo(rollupJobInfos); } } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJob.java b/fe/src/main/java/org/apache/doris/alter/RollupJob.java index f674a111ee7e1d..9b296e8084e464 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJob.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJob.java @@ -991,7 +991,6 @@ public void getJobInfo(List> jobInfos, OlapTable tbl) { // transaction id jobInfo.add(transactionId); - // job state jobInfo.add(state.name()); diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index a1d722ac2f58dc..827414723be6e7 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -396,19 +396,7 @@ protected void runRunningJob() { } // end for tablets } // end for partitions - // all partitions are good - // set table and rollup index state to NORMAL, then finish the job - for (Partition partition : tbl.getPartitions()) { - // get index from catalog, not from 'partitionIdToRollupIndex'. - // because if this alter job is recovered from edit log, rollup index in 'partitionIdToRollupIndex' - // is not the same object in catalog. So modification on that index can not reflect to the index - // in catalog. - MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); - Preconditions.checkNotNull(rollupIndex, rollupIndex); - rollupIndex.setState(IndexState.NORMAL); - } - - tbl.setState(OlapTableState.NORMAL); + onFinished(tbl); } finally { db.writeUnlock(); } @@ -420,6 +408,15 @@ protected void runRunningJob() { LOG.info("rollup job finished: {}", jobId); } + private void onFinished(OlapTable tbl) { + for (Partition partition : tbl.getPartitions()) { + MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); + Preconditions.checkNotNull(rollupIndex, rollupIndex); + rollupIndex.setState(IndexState.NORMAL); + } + tbl.setState(OlapTableState.NORMAL); + } + /* * cancel() can be called any time any place. * We need to clean any possible residual of this job. @@ -478,7 +475,7 @@ public static RollupJobV2 read(DataInput in) throws IOException { } @Override - public synchronized void write(DataOutput out) throws IOException { + public void write(DataOutput out) throws IOException { super.write(out); out.writeInt(partitionIdToRollupIndex.size()); @@ -634,12 +631,7 @@ private void replayFinished() { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl != null) { Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); - for (Partition partition : tbl.getPartitions()) { - MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); - Preconditions.checkNotNull(rollupIndex, rollupIndex); - rollupIndex.setState(IndexState.NORMAL); - } - tbl.setState(OlapTableState.NORMAL); + onFinished(tbl); } } finally { db.writeUnlock(); @@ -676,7 +668,8 @@ public void replay() { } @Override - protected void getInfo(List info) { + protected void getInfo(List> infos) { + List info = Lists.newArrayList(); info.add(jobId); info.add(tableName); info.add(TimeUtils.longToTimeString(createTimeMs)); @@ -694,6 +687,7 @@ protected void getInfo(List info) { info.add("N/A"); } info.add(timeoutMs / 1000); + infos.add(info); } public List> getUnfinishedTasks(int limit) { diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 54ca76b208bb95..65101cb069b581 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1033,7 +1033,7 @@ private void createJob(long dbId, OlapTable olapTable, Map> getAlterJobInfosByDb(Database db) { List> schemaChangeJobInfos = new LinkedList>(); + getOldAlterJobInfos(db, schemaChangeJobInfos); + getAlterJobV2Infos(schemaChangeJobInfos); + + // sort by "JobId", "PartitionName", "CreateTime", "FinishTime", "IndexName", "IndexState" + ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4, 5); + Collections.sort(schemaChangeJobInfos, comparator); + return schemaChangeJobInfos; + } + + private void getAlterJobV2Infos(List> schemaChangeJobInfos) { + for (AlterJobV2 alterJob : alterJobsV2.values()) { + alterJob.getInfo(schemaChangeJobInfos); + } + } + + private void getOldAlterJobInfos(Database db, List> schemaChangeJobInfos) { List selectedJobs = Lists.newArrayList(); lock(); @@ -1217,11 +1233,6 @@ public List> getAlterJobInfosByDb(Database db) { } finally { db.readUnlock(); } - - // sort by "JobId", "PartitionName", "CreateTime", "FinishTime", "IndexName", "IndexState" - ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4, 5); - Collections.sort(schemaChangeJobInfos, comparator); - return schemaChangeJobInfos; } @Override diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java index b7b7152e89216f..3db0921947dcdb 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java @@ -1108,12 +1108,13 @@ public void getJobInfo(List> jobInfos, OlapTable tbl) { jobInfo.add(TimeUtils.longToTimeString(finishedTime)); jobInfo.add("N/A"); // index name jobInfo.add("N/A"); // index id + jobInfo.add("N/A"); // origin id jobInfo.add("N/A"); // schema version - jobInfo.add("N/A"); // index state jobInfo.add(-1); // transaction id jobInfo.add(state.name()); // job state - jobInfo.add("N/A"); // progress jobInfo.add(cancelMsg); + jobInfo.add("N/A"); // progress + jobInfo.add(Config.alter_table_timeout_second); // timeout jobInfos.add(jobInfo); return; } @@ -1173,19 +1174,18 @@ public void getJobInfo(List> jobInfos, OlapTable tbl) { jobInfo.add(TimeUtils.longToTimeString(finishedTime)); jobInfo.add(tbl.getIndexNameById(indexId) == null ? "N/A" : tbl.getIndexNameById(indexId)); // index name jobInfo.add(indexId); + jobInfo.add(indexId); // origin index id // index schema version and schema hash - jobInfo.add(changedIndexIdToSchemaVersion.get(indexId) + "-" + changedIndexIdToSchemaHash.get(indexId)); - jobInfo.add(indexState.get(indexId)); // index state + jobInfo.add(changedIndexIdToSchemaVersion.get(indexId) + ":" + changedIndexIdToSchemaHash.get(indexId)); jobInfo.add(transactionId); jobInfo.add(state.name()); // job state - + jobInfo.add(cancelMsg); if (state == JobState.RUNNING) { jobInfo.add(indexProgress.get(indexId) == null ? "N/A" : indexProgress.get(indexId)); // progress } else { jobInfo.add("N/A"); } - - jobInfo.add(cancelMsg); + jobInfo.add(Config.alter_table_timeout_second); jobInfos.add(jobInfo); } // end for indexIds diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 1bcf2dabcdf932..c27aabb33e684e 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -32,6 +32,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.Pair; +import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; @@ -48,6 +49,7 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.collect.Table; import com.google.common.collect.Table.Cell; @@ -93,8 +95,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 { // bloom filter info private boolean hasBfChange; - private Set bfColumns; - private double bfFpp; + private Set bfColumns = null; + private double bfFpp = 0; // The schema change job will wait all transactions before this txn id finished, then send the schema change tasks. protected long watershedTxnId = -1; @@ -430,44 +432,7 @@ protected void runRunningJob() { } // end for partitions // all partitions are good - // replace the origin index with shadow index, set index state as NORMAL - for (Partition partition : tbl.getPartitions()) { - // drop the origin index from partitions - for (Map.Entry entry : indexIdMap.entrySet()) { - long shadowIdxId = entry.getKey(); - long originIdxId = entry.getValue(); - // get index from catalog, not from 'partitionIdToRollupIndex'. - // because if this alter job is recovered from edit log, index in 'partitionIndexMap' - // is not the same object in catalog. So modification on that index can not reflect to the index - // in catalog. - MaterializedIndex shadowIdx = partition.getIndex(shadowIdxId); - Preconditions.checkNotNull(shadowIdx, shadowIdxId); - // base index need special handling - if (originIdxId == partition.getBaseIndex().getId()) { - partition.setBaseIndex(shadowIdx); - } - partition.deleteRollupIndex(originIdxId); - shadowIdx.setState(IndexState.NORMAL); - } - } - - // update index schema info in table - for (Map.Entry entry : indexIdMap.entrySet()) { - long shadowIdxId = entry.getKey(); - long originIdxId = entry.getValue(); - String shadowIdxName = tbl.getIndexNameById(shadowIdxId); - String originIdxName = tbl.getIndexNameById(originIdxId); - tbl.deleteIndexInfo(originIdxName); - // the shadow index name is '__doris_shadow_xxx', rename it to origin name 'xxx' - tbl.renameIndexForSchemaChange(shadowIdxName, originIdxName); - - if (originIdxId == tbl.getId()) { - // set base index - tbl.setNewBaseSchema(indexSchemaMap.get(shadowIdxId)); - } - } - - tbl.setState(OlapTableState.NORMAL); + onFinished(tbl); } finally { db.writeUnlock(); } @@ -479,6 +444,52 @@ protected void runRunningJob() { LOG.info("rollup job finished: {}", jobId); } + private void onFinished(OlapTable tbl) { + // replace the origin index with shadow index, set index state as NORMAL + for (Partition partition : tbl.getPartitions()) { + // drop the origin index from partitions + for (Map.Entry entry : indexIdMap.entrySet()) { + long shadowIdxId = entry.getKey(); + long originIdxId = entry.getValue(); + // get index from catalog, not from 'partitionIdToRollupIndex'. + // because if this alter job is recovered from edit log, index in 'partitionIndexMap' + // is not the same object in catalog. So modification on that index can not reflect to the index + // in catalog. + MaterializedIndex shadowIdx = partition.getIndex(shadowIdxId); + Preconditions.checkNotNull(shadowIdx, shadowIdxId); + // base index need special handling + if (originIdxId == partition.getBaseIndex().getId()) { + partition.setBaseIndex(shadowIdx); + } + partition.deleteRollupIndex(originIdxId); + shadowIdx.setState(IndexState.NORMAL); + } + } + + // update index schema info in table + for (Map.Entry entry : indexIdMap.entrySet()) { + long shadowIdxId = entry.getKey(); + long originIdxId = entry.getValue(); + String shadowIdxName = tbl.getIndexNameById(shadowIdxId); + String originIdxName = tbl.getIndexNameById(originIdxId); + tbl.deleteIndexInfo(originIdxName); + // the shadow index name is '__doris_shadow_xxx', rename it to origin name 'xxx' + tbl.renameIndexForSchemaChange(shadowIdxName, originIdxName); + + if (originIdxId == tbl.getId()) { + // set base index + tbl.setNewBaseSchema(indexSchemaMap.get(shadowIdxId)); + } + } + + // update bloom filter + if (hasBfChange) { + tbl.setBloomFilterInfo(bfColumns, bfFpp); + } + + tbl.setState(OlapTableState.NORMAL); + } + /* * cancel() can be called any time any place. * We need to clean any possible residual of this job. @@ -542,16 +553,6 @@ public static SchemaChangeJobV2 read(DataInput in) throws IOException { return rollupJob; } - @Override - public synchronized void write(DataOutput out) throws IOException { - super.write(out); - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - } - /* * Replay job in PENDING state. * Should replay all changes before this job's state transfer to PENDING. @@ -634,13 +635,7 @@ private void replayFinished() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl != null) { - Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); - for (Partition partition : tbl.getPartitions()) { - MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); - Preconditions.checkNotNull(rollupIndex, rollupIndex); - rollupIndex.setState(IndexState.NORMAL); - } - tbl.setState(OlapTableState.NORMAL); + onFinished(tbl); } } finally { db.writeUnlock(); @@ -677,24 +672,33 @@ public void replay() { } @Override - protected void getInfo(List info) { - info.add(jobId); - info.add(tableName); - info.add(TimeUtils.longToTimeString(createTimeMs)); - info.add(TimeUtils.longToTimeString(finishedTimeMs)); - info.add(baseIndexName); - info.add(rollupIndexName); - info.add(rollupIndexId); - info.add(watershedTxnId); - info.add(jobState.name()); - info.add(errMsg); - // progress + protected void getInfo(List> infos) { + // calc progress first. all index share the same process + String progress = "N/A"; if (jobState == JobState.RUNNING && schemaChangeBatchTask.getTaskNum() > 0) { - info.add(schemaChangeBatchTask.getFinishedTaskNum() + "/" + schemaChangeBatchTask.getTaskNum()); - } else { - info.add("N/A"); + progress = schemaChangeBatchTask.getFinishedTaskNum() + "/" + schemaChangeBatchTask.getTaskNum(); + } + + // one line for one shadow index + for (Map.Entry entry : indexIdMap.entrySet()) { + long shadowIndexId = entry.getKey(); + List info = Lists.newArrayList(); + info.add(jobId); + info.add(tableName); + info.add(TimeUtils.longToTimeString(createTimeMs)); + info.add(TimeUtils.longToTimeString(finishedTimeMs)); + // only show the origin index name + info.add(indexIdToName.get(shadowIndexId).substring(SchemaChangeHandler.SHADOW_INDEX_NAME_PRFIX.length())); + info.add(shadowIndexId); + info.add(entry.getValue()); + info.add(indexSchemaVersionAndHashMap.get(shadowIndexId).toString()); + info.add(watershedTxnId); + info.add(jobState.name()); + info.add(errMsg); + info.add(progress); + info.add(timeoutMs / 1000); + infos.add(info); } - info.add(timeoutMs / 1000); } public List> getUnfinishedTasks(int limit) { @@ -712,4 +716,124 @@ public List> getUnfinishedTasks(int limit) { } return taskInfos; } + + @Override + public synchronized void write(DataOutput out) throws IOException { + super.write(out); + + out.writeInt(partitionIndexTabletMap.rowKeySet().size()); + for (Long partitionId : partitionIndexTabletMap.rowKeySet()) { + out.writeLong(partitionId); + Map> indexTabletMap = partitionIndexTabletMap.row(partitionId); + out.writeInt(indexTabletMap.size()); + for (Long shadowIndexId : indexTabletMap.keySet()) { + out.writeLong(shadowIndexId); + // tablet id map + Map tabletMap = indexTabletMap.get(shadowIndexId); + out.writeInt(tabletMap.size()); + for (Map.Entry entry : tabletMap.entrySet()) { + out.writeLong(entry.getKey()); + out.writeLong(entry.getValue()); + } + // shadow index + MaterializedIndex shadowIndex = partitionIndexMap.get(partitionId, shadowIndexId); + shadowIndex.write(out); + } + } + + // shadow index info + out.writeInt(indexIdMap.size()); + for (Map.Entry entry : indexIdMap.entrySet()) { + long shadowIndexId = entry.getKey(); + out.writeLong(shadowIndexId); + // index id map + out.writeLong(entry.getValue()); + // index name + Text.writeString(out, indexIdToName.get(shadowIndexId)); + // index schema + out.writeInt(indexSchemaMap.get(shadowIndexId).size()); + for (Column column : indexSchemaMap.get(shadowIndexId)) { + column.write(out); + } + // index schema version and hash + out.writeInt(indexSchemaVersionAndHashMap.get(shadowIndexId).first); + out.writeInt(indexSchemaVersionAndHashMap.get(shadowIndexId).second); + // short key count + out.writeShort(indexShortKeyMap.get(shadowIndexId)); + } + + // bloom filter + out.writeBoolean(hasBfChange); + if (hasBfChange) { + out.writeInt(bfColumns.size()); + for (String bfCol : bfColumns) { + Text.writeString(out, bfCol); + } + out.writeDouble(bfFpp); + } + + out.writeLong(watershedTxnId); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + + int partitionNum = in.readInt(); + for (int i = 0; i < partitionNum; i++) { + long partitionId = in.readLong(); + int indexNum = in.readInt(); + for (int j = 0; j < indexNum; j++) { + long shadowIndexId = in.readLong(); + int tabletNum = in.readInt(); + Map tabletMap = Maps.newHashMapWithExpectedSize(tabletNum); + for (int k = 0; k < tabletNum; k++) { + long shadowTabletId = in.readLong(); + long originTabletId = in.readLong(); + tabletMap.put(shadowTabletId, originTabletId); + } + partitionIndexTabletMap.put(partitionId, shadowIndexId, tabletMap); + // shadow index + MaterializedIndex shadowIndex = MaterializedIndex.read(in); + partitionIndexMap.put(partitionId, shadowIndexId, shadowIndex); + } + } + + // shadow index info + int indexNum = in.readInt(); + for (int i = 0; i < indexNum; i++) { + long shadowIndexId = in.readLong(); + long originIndexId = in.readLong(); + String indexName = Text.readString(in); + // index schema + int colNum = in.readInt(); + List schema = Lists.newArrayListWithCapacity(colNum); + for (int j = 0; j < colNum; j++) { + schema.add(Column.read(in)); + } + int schemaVersion = in.readInt(); + int schemaVersionHash = in.readInt(); + Pair schemaVersionAndHash = Pair.create(schemaVersion, schemaVersionHash); + short shortKeyCount = in.readShort(); + + indexIdMap.put(shadowIndexId, originIndexId); + indexIdToName.put(shadowIndexId, indexName); + indexSchemaMap.put(shadowIndexId, schema); + indexSchemaVersionAndHashMap.put(shadowIndexId, schemaVersionAndHash); + indexShortKeyMap.put(shadowIndexId, shortKeyCount); + } + + // bloom filter + hasBfChange = in.readBoolean(); + if (hasBfChange) { + int bfNum = in.readInt(); + bfColumns = Sets.newHashSetWithExpectedSize(bfNum); + for (int i = 0; i < bfNum; i++) { + bfColumns.add(Text.readString(in)); + } + bfFpp = in.readDouble(); + } + + watershedTxnId = in.readLong(); + } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 918bdc27789c96..2282c9d9d0acdf 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -45,11 +45,10 @@ public class Partition extends MetaObject implements Writable { public static final long PARTITION_INIT_VERSION = 1L; public static final long PARTITION_INIT_VERSION_HASH = 0L; + @Deprecated public enum PartitionState { NORMAL, - @Deprecated ROLLUP, - @Deprecated SCHEMA_CHANGE } diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index f567225ab9ca4b..7de077f2e223b1 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -39,13 +39,16 @@ public class Replica implements Writable { public enum ReplicaState { NORMAL, + @Deprecated ROLLUP, + @Deprecated SCHEMA_CHANGE, CLONE, + ALTER, // replica is under rollup or schema change DECOMMISSION; // replica is ready to be deleted public boolean isLoadable() { - return this == ReplicaState.NORMAL || this == ReplicaState.SCHEMA_CHANGE; + return this == ReplicaState.NORMAL || this == ReplicaState.SCHEMA_CHANGE || this == ReplicaState.ALTER; } } diff --git a/fe/src/main/java/org/apache/doris/common/proc/SchemaChangeProcNode.java b/fe/src/main/java/org/apache/doris/common/proc/SchemaChangeProcNode.java index 3ec874aa8c9c01..a5806beed5c477 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/SchemaChangeProcNode.java +++ b/fe/src/main/java/org/apache/doris/common/proc/SchemaChangeProcNode.java @@ -30,8 +30,8 @@ public class SchemaChangeProcNode implements ProcNodeInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("JobId").add("TableName").add("CreateTime").add("FinishTime") - .add("IndexName").add("IndexId").add("SchemaVersion").add("IndexState") - .add("TransactionId").add("State").add("Progress").add("Msg") + .add("IndexName").add("IndexId").add("OriginIndexId").add("SchemaVersion") + .add("TransactionId").add("State").add("Msg").add("Progress").add("Timeout") .build(); private SchemaChangeHandler schemaChangeHandler; From 1b3ce58ab6f6768c1708499b8950e6d03591fbcb Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Jul 2019 09:15:33 +0800 Subject: [PATCH 07/79] add get shadow index --- .../apache/doris/alter/SchemaChangeJob.java | 3 +- .../org/apache/doris/backup/BackupJob.java | 3 +- .../apache/doris/backup/BackupJobInfo.java | 3 +- .../org/apache/doris/backup/RestoreJob.java | 19 +- .../org/apache/doris/catalog/Catalog.java | 13 +- .../doris/catalog/CatalogRecycleBin.java | 17 +- .../org/apache/doris/catalog/Database.java | 5 +- .../doris/catalog/MaterializedIndex.java | 12 +- .../apache/doris/catalog/MetadataViewer.java | 5 +- .../org/apache/doris/catalog/OlapTable.java | 5 +- .../org/apache/doris/catalog/Partition.java | 96 ++-- .../apache/doris/catalog/TabletStatMgr.java | 3 +- .../doris/clone/ColocateTableBalancer.java | 7 +- .../org/apache/doris/clone/LoadBalancer.java | 3 + .../org/apache/doris/clone/TabletChecker.java | 9 +- .../apache/doris/clone/TabletScheduler.java | 3 +- .../apache/doris/common/CaseSensibility.java | 2 +- .../doris/common/proc/IndicesProcDir.java | 3 +- .../doris/common/proc/StatisticProcDir.java | 3 +- .../doris/consistency/ConsistencyChecker.java | 3 +- .../doris/http/rest/RowCountAction.java | 3 +- .../doris/http/rest/ShowMetaInfoAction.java | 8 +- .../main/java/org/apache/doris/load/Load.java | 458 +----------------- .../org/apache/doris/load/LoadChecker.java | 3 +- .../org/apache/doris/master/Checkpoint.java | 3 +- .../apache/doris/planner/OlapTableSink.java | 9 +- .../apache/doris/planner/RollupSelector.java | 15 +- .../org/apache/doris/qe/ShowExecutor.java | 3 +- .../org/apache/doris/task/LoadEtlTask.java | 3 +- .../transaction/GlobalTransactionMgr.java | 104 +--- .../transaction/PublishVersionDaemon.java | 12 +- .../org/apache/doris/alter/RollupJobTest.java | 3 +- .../doris/backup/BackupHandlerTest.java | 3 +- .../apache/doris/backup/RestoreJobTest.java | 3 +- .../apache/doris/catalog/CatalogTestUtil.java | 3 +- .../apache/doris/load/LoadCheckerTest.java | 7 +- .../org/apache/doris/qe/ShowExecutorTest.java | 1 - .../apache/doris/task/LoadEtlTaskTest.java | 3 +- 38 files changed, 203 insertions(+), 658 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java index 3db0921947dcdb..61a291367791df 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJob.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; @@ -333,7 +334,7 @@ public int checkOrResendClearTasks() { OUTER_LOOP: for (Partition partition : olapTable.getPartitions()) { long partitionId = partition.getId(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { List replicas = tablet.getReplicas(); for (Replica replica : replicas) { diff --git a/fe/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/src/main/java/org/apache/doris/backup/BackupJob.java index 13b656ae269448..a11b91f53b5ba9 100644 --- a/fe/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/src/main/java/org/apache/doris/backup/BackupJob.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.FsBroker; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -386,7 +387,7 @@ private void prepareAndSendSnapshotTask() { for (Partition partition : partitions) { long visibleVersion = partition.getVisibleVersion(); long visibleVersionHash = partition.getVisibleVersionHash(); - List indexes = partition.getMaterializedIndices(); + List indexes = partition.getMaterializedIndices(IndexExtState.VISIBLE); for (MaterializedIndex index : indexes) { int schemaHash = tbl.getSchemaHashByIndexId(index.getId()); List tablets = index.getTablets(); diff --git a/fe/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/src/main/java/org/apache/doris/backup/BackupJobInfo.java index c788659e0177aa..9f07fed710da41 100644 --- a/fe/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -19,6 +19,7 @@ import org.apache.doris.backup.RestoreFileMapping.IdChain; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; @@ -251,7 +252,7 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db partitionInfo.versionHash = partition.getVisibleVersionHash(); tableInfo.partitions.put(partitionInfo.name, partitionInfo); // indexes - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo idxInfo = new BackupIndexInfo(); idxInfo.id = index.getId(); idxInfo.name = olapTbl.getIndexNameById(index.getId()); diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java index 4c545a7e08699b..4e330646e374e1 100644 --- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -30,6 +30,7 @@ import org.apache.doris.catalog.FsBroker; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; @@ -613,7 +614,7 @@ private void checkAndPrepareMeta() { Set bfColumns = localTbl.getCopiedBfColumns(); double bfFpp = localTbl.getBfFpp(); - for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices()) { + for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { short shortKeyColumnCount = localTbl.getShortKeyColumnCountByIndexId(restoredIdx.getId()); int schemaHash = localTbl.getSchemaHashByIndexId(restoredIdx.getId()); KeysType keysType = localTbl.getKeysType(); @@ -646,7 +647,7 @@ private void checkAndPrepareMeta() { for (Partition restorePart : restoreTbl.getPartitions()) { Set bfColumns = restoreTbl.getCopiedBfColumns(); double bfFpp = restoreTbl.getBfFpp(); - for (MaterializedIndex index : restorePart.getMaterializedIndices()) { + for (MaterializedIndex index : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { short shortKeyColumnCount = restoreTbl.getShortKeyColumnCountByIndexId(index.getId()); int schemaHash = restoreTbl.getSchemaHashByIndexId(index.getId()); KeysType keysType = restoreTbl.getKeysType(); @@ -838,7 +839,7 @@ private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteT long visibleVersionHash = remotePart.getVisibleVersionHash(); // tablets - for (MaterializedIndex remoteIdx : remotePart.getMaterializedIndices()) { + for (MaterializedIndex remoteIdx : remotePart.getMaterializedIndices(IndexExtState.VISIBLE)) { int schemaHash = remoteTbl.getSchemaHashByIndexId(remoteIdx.getId()); int remotetabletSize = remoteIdx.getTablets().size(); remoteIdx.clearTabletsForRestore(); @@ -872,7 +873,7 @@ private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteT // files in repo to files in local private void genFileMapping(OlapTable localTbl, Partition localPartition, Long remoteTblId, BackupPartitionInfo backupPartInfo, boolean overwrite) { - for (MaterializedIndex localIdx : localPartition.getMaterializedIndices()) { + for (MaterializedIndex localIdx : localPartition.getMaterializedIndices(IndexExtState.VISIBLE)) { LOG.debug("get index id: {}, index name: {}", localIdx.getId(), localTbl.getIndexNameById(localIdx.getId())); BackupIndexInfo backupIdxInfo = backupPartInfo.getIdx(localTbl.getIndexNameById(localIdx.getId())); @@ -935,7 +936,7 @@ private void replayCheckAndPrepareMeta() { localTbl.addPartition(restorePart); // modify tablet inverted index - for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices()) { + for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { int schemaHash = localTbl.getSchemaHashByIndexId(restoreIdx.getId()); TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(), restoreIdx.getId(), schemaHash, TStorageMedium.HDD); @@ -953,7 +954,7 @@ private void replayCheckAndPrepareMeta() { db.createTable(restoreTbl); // modify tablet inverted index for (Partition restorePart : restoreTbl.getPartitions()) { - for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices()) { + for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { int schemaHash = restoreTbl.getSchemaHashByIndexId(restoreIdx.getId()); TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), restorePart.getId(), restoreIdx.getId(), schemaHash, TStorageMedium.HDD); @@ -1219,7 +1220,7 @@ private Status allTabletCommitted(boolean isReplay) { part.updateVersionForRestore(entry.getValue().first, entry.getValue().second); // we also need to update the replica version of these overwritten restored partitions - for (MaterializedIndex idx : part.getMaterializedIndices()) { + for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { for (Replica replica : tablet.getReplicas()) { if (!replica.checkVersionCatchUp(part.getVisibleVersion(), @@ -1369,7 +1370,7 @@ public void cancelInternal(boolean isReplay) { for (OlapTable restoreTbl : restoredTbls) { LOG.info("remove restored table when cancelled: {}", restoreTbl.getName()); for (Partition part : restoreTbl.getPartitions()) { - for (MaterializedIndex idx : part.getMaterializedIndices()) { + for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { Catalog.getCurrentInvertedIndex().deleteTablet(tablet.getId()); } @@ -1386,7 +1387,7 @@ public void cancelInternal(boolean isReplay) { } LOG.info("remove restored partition in table {} when cancelled: {}", restoreTbl.getName(), entry.second.getName()); - for (MaterializedIndex idx : entry.second.getMaterializedIndices()) { + for (MaterializedIndex idx : entry.second.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { Catalog.getCurrentInvertedIndex().deleteTablet(tablet.getId()); } diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 8601d42a1a6245..1040924eab1aa1 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -76,6 +76,7 @@ import org.apache.doris.catalog.Database.DbState; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.KuduPartition.KuduRange; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Replica.ReplicaState; @@ -1356,7 +1357,7 @@ private void recreateTabletInvertIndex() { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty( partitionId).getStorageMedium(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium); @@ -3006,7 +3007,7 @@ public void replayAddPartition(PartitionPersistInfo info) throws DdlException { if (!isCheckpointThread()) { // add to inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), partition.getId(), @@ -3983,7 +3984,7 @@ public void replayCreateTable(String dbName, Table table) { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty( partitionId).getStorageMedium(); - for (MaterializedIndex mIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = mIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium); @@ -5632,7 +5633,7 @@ public Set getMigrations() { for (Partition partition : olapTable.getPartitions()) { final short replicationNum = olapTable.getPartitionInfo() .getReplicationNum(partition.getId()); - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { if (materializedIndex.getState() != IndexState.NORMAL) { continue; } @@ -6020,7 +6021,7 @@ private void truncateTableInternal(OlapTable olapTable, List newParti for (Partition newPartition : newPartitions) { Partition oldPartition = olapTable.replacePartition(newPartition); // save old tablets to be removed - for (MaterializedIndex index : oldPartition.getMaterializedIndices()) { + for (MaterializedIndex index : oldPartition.getMaterializedIndices(IndexExtState.ALL)) { index.getTablets().stream().forEach(t -> { oldTabletIds.add(t.getId()); }); @@ -6047,7 +6048,7 @@ public void replayTruncateTable(TruncateTableInfo info) { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty( partitionId).getStorageMedium(); - for (MaterializedIndex mIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = mIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(db.getId(), olapTable.getId(), diff --git a/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index d14dfea83f413f..08dfc355608d23 100644 --- a/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -208,7 +209,7 @@ private void onEraseOlapTable(OlapTable olapTable) { // inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); for (Partition partition : olapTable.getPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } @@ -218,7 +219,7 @@ private void onEraseOlapTable(OlapTable olapTable) { // drop all replicas AgentBatchTask batchTask = new AgentBatchTask(); for (Partition partition : olapTable.getPartitions()) { - List allIndices = partition.getMaterializedIndices(); + List allIndices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex materializedIndex : allIndices) { long indexId = materializedIndex.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); @@ -272,7 +273,7 @@ public synchronized void replayEraseTable(long tableId) { // remove tablet from inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); for (Partition partition : olapTable.getPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } @@ -297,7 +298,7 @@ private synchronized void erasePartition(long currentTimeMs) { if (isExpire(partitionId, currentTimeMs)) { // remove tablet in inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } @@ -327,7 +328,7 @@ private synchronized void erasePartitionWithSameName(long dbId, long tableId, St if (partition.getName().equals(partitionName)) { // remove tablet in inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } @@ -349,7 +350,7 @@ public synchronized void replayErasePartition(long partitionId) { if (!Catalog.isCheckpointThread()) { // remove tablet from inverted index TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { invertedIndex.deleteTablet(tablet.getId()); } @@ -581,7 +582,7 @@ public void addTabletToInvertedIndex() { for (Partition partition : olapTable.getPartitions()) { long partitionId = partition.getId(); TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium); @@ -633,7 +634,7 @@ public void addTabletToInvertedIndex() { // storage medium should be got from RecyclePartitionInfo, not from olap table. because olap table // does not have this partition any more TStorageMedium medium = partitionInfo.getDataProperty().getStorageMedium(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium); diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java b/fe/src/main/java/org/apache/doris/catalog/Database.java index ea206094aad0ea..1b43ce11d16962 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/src/main/java/org/apache/doris/catalog/Database.java @@ -17,7 +17,7 @@ package org.apache.doris.catalog; -import com.google.common.collect.Lists; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Table.TableType; @@ -35,6 +35,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; @@ -206,7 +207,7 @@ public long getDataQuotaLeftWithLock() { OlapTable olapTable = (OlapTable) table; for (Partition partition : olapTable.getPartitions()) { - for (MaterializedIndex mIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { // skip ROLLUP index if (mIndex.getState() == IndexState.ROLLUP) { continue; diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 5ddae4144c6502..3324006a90b34b 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -39,7 +39,17 @@ public enum IndexState { NORMAL, ROLLUP, SCHEMA_CHANGE, - SHADOW // index in SHADOW state is visible to load process, but invisible to query + SHADOW; // index in SHADOW state is visible to load process, but invisible to query + + public boolean isVisible() { + return this == IndexState.NORMAL || this == IndexState.SCHEMA_CHANGE; + } + } + + public enum IndexExtState { + ALL, + VISIBLE, // index state in NORMAL and SCHEMA_CHANGE + SHADOW // index state in SHADOW } private long id; diff --git a/fe/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/src/main/java/org/apache/doris/catalog/MetadataViewer.java index 05e56e85603e5e..55a1657b27b4a2 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MetadataViewer.java +++ b/fe/src/main/java/org/apache/doris/catalog/MetadataViewer.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.AdminShowReplicaDistributionStmt; import org.apache.doris.analysis.AdminShowReplicaStatusStmt; import org.apache.doris.analysis.BinaryPredicate.Operator; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.Replica.ReplicaStatus; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.common.DdlException; @@ -79,7 +80,7 @@ private static List> getTabletStatus(String dbName, String tblName, long visibleVersion = partition.getVisibleVersion(); short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -210,7 +211,7 @@ private static List> getTabletDistribution(String dbName, String tb int totalReplicaNum = 0; for (String partName : partitions) { Partition partition = olapTable.getPartition(partName); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { if (!countMap.containsKey(replica.getBackendId())) { diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index a19cb3b3b6dbf7..5f0c0f85e62559 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -30,6 +30,7 @@ import org.apache.doris.backup.Status; import org.apache.doris.backup.Status.ErrCode; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.Replica.ReplicaState; @@ -938,7 +939,7 @@ public OlapTable selectiveCopy(Collection reservedPartNames, boolean res for (Partition partition : copied.getPartitions()) { partition.setState(PartitionState.NORMAL); copied.getPartitionInfo().setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD)); - for (MaterializedIndex idx : partition.getMaterializedIndices()) { + for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.ALL)) { idx.setState(IndexState.NORMAL); for (Tablet tablet : idx.getTablets()) { for (Replica replica : tablet.getReplicas()) { @@ -1009,7 +1010,7 @@ public boolean isStable(SystemInfoService infoService, TabletScheduler tabletSch long visibleVersion = partition.getVisibleVersion(); long visibleVersionHash = partition.getVisibleVersionHash(); short replicationNum = partitionInfo.getReplicationNum(partition.getId()); - for (MaterializedIndex mIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : mIndex.getTablets()) { if (tabletScheduler.containsTablet(tablet.getId())) { return false; diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 2282c9d9d0acdf..10b4a8ab15daf0 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -18,20 +18,22 @@ package org.apache.doris.catalog; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Util; import org.apache.doris.meta.MetaContext; +import com.google.common.collect.Maps; + +import org.apache.kudu.client.shaded.com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -57,7 +59,17 @@ public enum PartitionState { private PartitionState state; private MaterializedIndex baseIndex; - private Map idToRollupIndex; + /* + * Visible rollup indexes are indexes which are visible to user. + * User can do query on them, show them in related 'show' stmt. + */ + private Map idToVisibleRollupIndex = Maps.newHashMap(); + /* + * Shadow indexes are indexes which are not visible to user. + * Query will not run on these shadow indexes, and user can not see them neither. + * But load process will load data into these shadow indexes. + */ + private Map idToShadowIndex = Maps.newHashMap(); /* * committed version(hash): after txn is committed, set committed version(hash) @@ -74,8 +86,7 @@ public enum PartitionState { private DistributionInfo distributionInfo; - public Partition() { - this.idToRollupIndex = new HashMap(); + private Partition() { } public Partition(long id, String name, @@ -85,7 +96,6 @@ public Partition(long id, String name, this.state = PartitionState.NORMAL; this.baseIndex = baseIndex; - this.idToRollupIndex = new HashMap(); this.visibleVersion = PARTITION_INIT_VERSION; this.visibleVersionHash = PARTITION_INIT_VERSION_HASH; @@ -169,11 +179,19 @@ public DistributionInfo getDistributionInfo() { } public void createRollupIndex(MaterializedIndex mIndex) { - this.idToRollupIndex.put(mIndex.getId(), mIndex); + if (mIndex.getState().isVisible()) { + this.idToVisibleRollupIndex.put(mIndex.getId(), mIndex); + } else { + this.idToShadowIndex.put(mIndex.getId(), mIndex); + } } public MaterializedIndex deleteRollupIndex(long indexId) { - return this.idToRollupIndex.remove(indexId); + if (this.idToVisibleRollupIndex.containsKey(indexId)) { + return idToVisibleRollupIndex.remove(indexId); + } else { + return idToShadowIndex.remove(indexId); + } } public MaterializedIndex getBaseIndex() { @@ -210,36 +228,40 @@ public long getCommittedVersionHash() { return committedVersionHash; } - public List getRollupIndices() { - List rollupIndices = new ArrayList(idToRollupIndex.size()); - for (Map.Entry entry : idToRollupIndex.entrySet()) { - rollupIndices.add(entry.getValue()); - } - return rollupIndices; - } - public MaterializedIndex getIndex(long indexId) { if (baseIndex.getId() == indexId) { return baseIndex; } - if (idToRollupIndex.containsKey(indexId)) { - return idToRollupIndex.get(indexId); + if (idToVisibleRollupIndex.containsKey(indexId)) { + return idToVisibleRollupIndex.get(indexId); + } else { + return idToShadowIndex.get(indexId); } - return null; } - public List getMaterializedIndices() { - List indices = new ArrayList(); - indices.add(baseIndex); - for (MaterializedIndex rollupIndex : idToRollupIndex.values()) { - indices.add(rollupIndex); + public List getMaterializedIndices(IndexExtState extState) { + List indices = Lists.newArrayList(); + switch (extState) { + case ALL: + indices.add(baseIndex); + indices.addAll(idToVisibleRollupIndex.values()); + indices.addAll(idToShadowIndex.values()); + break; + case VISIBLE: + indices.add(baseIndex); + indices.addAll(idToVisibleRollupIndex.values()); + break; + case SHADOW: + indices.addAll(idToShadowIndex.values()); + default: + break; } return indices; } public long getDataSize() { long dataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices()) { + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { dataSize += mIndex.getDataSize(); } return dataSize; @@ -265,10 +287,10 @@ public void write(DataOutput out) throws IOException { baseIndex.write(out); - int rollupCount = (idToRollupIndex != null) ? idToRollupIndex.size() : 0; + int rollupCount = (idToVisibleRollupIndex != null) ? idToVisibleRollupIndex.size() : 0; out.writeInt(rollupCount); - if (idToRollupIndex != null) { - for (Map.Entry entry : idToRollupIndex.entrySet()) { + if (idToVisibleRollupIndex != null) { + for (Map.Entry entry : idToVisibleRollupIndex.entrySet()) { entry.getValue().write(out); } } @@ -297,7 +319,7 @@ public void readFields(DataInput in) throws IOException { int rollupCount = in.readInt(); for (int i = 0; i < rollupCount; ++i) { MaterializedIndex rollupTable = MaterializedIndex.read(in); - idToRollupIndex.put(rollupTable.getId(), rollupTable); + idToVisibleRollupIndex.put(rollupTable.getId(), rollupTable); } visibleVersion = in.readLong(); @@ -338,16 +360,16 @@ public boolean equals(Object obj) { } Partition partition = (Partition) obj; - if (idToRollupIndex != partition.idToRollupIndex) { - if (idToRollupIndex.size() != partition.idToRollupIndex.size()) { + if (idToVisibleRollupIndex != partition.idToVisibleRollupIndex) { + if (idToVisibleRollupIndex.size() != partition.idToVisibleRollupIndex.size()) { return false; } - for (Entry entry : idToRollupIndex.entrySet()) { + for (Entry entry : idToVisibleRollupIndex.entrySet()) { long key = entry.getKey(); - if (!partition.idToRollupIndex.containsKey(key)) { + if (!partition.idToVisibleRollupIndex.containsKey(key)) { return false; } - if (!entry.getValue().equals(partition.idToRollupIndex.get(key))) { + if (!entry.getValue().equals(partition.idToVisibleRollupIndex.get(key))) { return false; } } @@ -368,11 +390,11 @@ public String toString() { buffer.append("base_index: ").append(baseIndex.toString()).append("; "); - int rollupCount = (idToRollupIndex != null) ? idToRollupIndex.size() : 0; + int rollupCount = (idToVisibleRollupIndex != null) ? idToVisibleRollupIndex.size() : 0; buffer.append("rollup count: ").append(rollupCount).append("; "); - if (idToRollupIndex != null) { - for (Map.Entry entry : idToRollupIndex.entrySet()) { + if (idToVisibleRollupIndex != null) { + for (Map.Entry entry : idToVisibleRollupIndex.entrySet()) { buffer.append("rollup_index: ").append(entry.getValue().toString()).append("; "); } } diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java index 67911c78f7b458..d5e9b015c87fc2 100644 --- a/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java +++ b/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; @@ -119,7 +120,7 @@ protected void runOneCycle() { for (Partition partition : olapTable.getPartitions()) { long version = partition.getVisibleVersion(); long versionHash = partition.getVisibleVersionHash(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexRowCount = 0L; for (Tablet tablet : index.getTablets()) { long tabletRowCount = 0L; diff --git a/fe/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java b/fe/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java index 19707dc884dbfb..60a092c861b4c2 100644 --- a/fe/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java +++ b/fe/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -233,7 +234,7 @@ private long selectSubstituteBackend(int tabletOrderIdx, GroupId groupId, long u } for (Partition partition : tbl.getPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long tabletId = index.getTabletIdsInOrder().get(tabletOrderIdx); Tablet tablet = index.getTablet(tabletId); Replica replica = tablet.getReplicaByBackendId(unavailableBeId); @@ -344,7 +345,9 @@ private void matchGroup() { short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); long visibleVersion = partition.getVisibleVersion(); long visibleVersionHash = partition.getVisibleVersionHash(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + // Here we only get VISIBLE indexes. All other indexes are not queryable. + // So it does not matter if tablets of other indexes are not matched. + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { Preconditions.checkState(backendBucketsSeq.size() == index.getTablets().size(), backendBucketsSeq.size() + " vs. " + index.getTablets().size()); int idx = 0; diff --git a/fe/src/main/java/org/apache/doris/clone/LoadBalancer.java b/fe/src/main/java/org/apache/doris/clone/LoadBalancer.java index 77d3ac9a2814e2..16baa7a7cface8 100644 --- a/fe/src/main/java/org/apache/doris/clone/LoadBalancer.java +++ b/fe/src/main/java/org/apache/doris/clone/LoadBalancer.java @@ -82,6 +82,9 @@ public List selectAlternativeTablets() { * * Here we only select tablets from high load node, do not set its src or dest, all this will be set * when this tablet is being scheduled in tablet scheduler. + * + * NOTICE that we may select any available tablets here, ignore their state. + * The state will be checked when being scheduled in tablet scheduler. */ private List selectAlternativeTabletsForCluster( String clusterName, ClusterLoadStatistic clusterStat, TStorageMedium medium) { diff --git a/fe/src/main/java/org/apache/doris/clone/TabletChecker.java b/fe/src/main/java/org/apache/doris/clone/TabletChecker.java index d2e9d643a99a2d..9dbedda6758b9f 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletChecker.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletChecker.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Partition.PartitionState; @@ -207,7 +208,13 @@ private void checkTablets() { } boolean isInPrios = isInPrios(dbId, table.getId(), partition.getId()); boolean prioPartIsHealthy = true; - for (MaterializedIndex idx : partition.getMaterializedIndices()) { + /* + * Here we get all ALL indexes, including SHADOW indexes. + * SHADOW index should be treated as a special NORMAL index. + * It can be repaired, but CAN NOT be balanced, added or removed. + * The above restrictions will be checked in tablet scheduler. + */ + for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : idx.getTablets()) { totalTabletNum++; diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java index 06445cb612a89c..7994fcf463cd1a 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -514,7 +514,8 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throw new SchedException(Status.UNRECOVERABLE, "table's state is not NORMAL"); } - if (statusPair.first != TabletStatus.VERSION_INCOMPLETE && partition.getState() != PartitionState.NORMAL) { + if (statusPair.first != TabletStatus.VERSION_INCOMPLETE + && (partition.getState() != PartitionState.NORMAL || tableState != OlapTableState.NORMAL)) { // If table is under ALTER process(before FINISHING), do not allow to add or delete replica. // VERSION_INCOMPLETE will repair the replica in place, which is allowed. throw new SchedException(Status.UNRECOVERABLE, diff --git a/fe/src/main/java/org/apache/doris/common/CaseSensibility.java b/fe/src/main/java/org/apache/doris/common/CaseSensibility.java index 170f8b37ec2a76..fa11d1d5390ced 100644 --- a/fe/src/main/java/org/apache/doris/common/CaseSensibility.java +++ b/fe/src/main/java/org/apache/doris/common/CaseSensibility.java @@ -23,7 +23,7 @@ public enum CaseSensibility { TABLE(true), ROLUP(true), PARTITION(true), - COLUMN(true), + COLUMN(false), USER(true), ROLE(false), HOST(false), diff --git a/fe/src/main/java/org/apache/doris/common/proc/IndicesProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/IndicesProcDir.java index be4986cfca2970..63bf96d2f0c7be 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/IndicesProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/IndicesProcDir.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.common.AnalysisException; @@ -63,7 +64,7 @@ public ProcResult fetchResult() throws AnalysisException { db.readLock(); try { result.setNames(TITLE_NAMES); - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { List indexInfo = new ArrayList(); indexInfo.add(materializedIndex.getId()); indexInfo.add(olapTable.getIndexNameById(materializedIndex.getId())); diff --git a/fe/src/main/java/org/apache/doris/common/proc/StatisticProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/StatisticProcDir.java index da46a36c0d36fa..3e320e17180d98 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/StatisticProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/StatisticProcDir.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.Tablet.TabletStatus; import org.apache.doris.clone.TabletSchedCtx.Priority; import org.apache.doris.common.AnalysisException; @@ -117,7 +118,7 @@ public ProcResult fetchResult() throws AnalysisException { for (Partition partition : olapTable.getPartitions()) { short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId()); ++dbPartitionNum; - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { ++dbIndexNum; for (Tablet tablet : materializedIndex.getTablets()) { ++dbTabletNum; diff --git a/fe/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java b/fe/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java index 3e329a5cc531ff..fbd727b4d66815 100644 --- a/fe/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java +++ b/fe/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MetaObject; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -304,7 +305,7 @@ private long chooseTablet() { // sort materializedIndices Queue indexQueue = new PriorityQueue(1, COMPARATOR); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { indexQueue.add(index); } diff --git a/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java b/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java index 54a420b442d3e3..e902043fec0342 100644 --- a/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java +++ b/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -92,7 +93,7 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept for (Partition partition : olapTable.getPartitions()) { long version = partition.getVisibleVersion(); long versionHash = partition.getVisibleVersionHash(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexRowCount = 0L; for (Tablet tablet : index.getTablets()) { long tabletRowCount = 0L; diff --git a/fe/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java b/fe/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java index 9ff93a253a8418..28adf4ef65369a 100644 --- a/fe/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java +++ b/fe/src/main/java/org/apache/doris/http/rest/ShowMetaInfoAction.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -34,9 +35,8 @@ import org.apache.doris.http.BaseResponse; import org.apache.doris.http.IllegalArgException; import org.apache.doris.persist.Storage; -import com.google.gson.Gson; -import io.netty.handler.codec.http.HttpMethod; +import com.google.gson.Gson; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; @@ -49,6 +49,8 @@ import java.util.List; import java.util.Map; +import io.netty.handler.codec.http.HttpMethod; + public class ShowMetaInfoAction extends RestBaseAction { private enum Action { SHOW_DB_SIZE, @@ -164,7 +166,7 @@ public Map getDataSize() { long tableSize = 0; for (Partition partition : olapTable.getPartitions()) { long partitionSize = 0; - for (MaterializedIndex mIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex mIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexSize = 0; for (Tablet tablet : mIndex.getTablets()) { long maxReplicaSize = 0; diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 6daeaf87b09e72..1db6857274baa8 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -38,6 +38,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; @@ -61,12 +62,10 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.LoadException; -import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.common.util.Util; import org.apache.doris.load.AsyncDeleteJob.DeleteState; import org.apache.doris.load.FailMsg.CancelType; import org.apache.doris.load.LoadJob.JobState; @@ -76,17 +75,13 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.FrontendOptions; import org.apache.doris.system.Backend; -import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentClient; -import org.apache.doris.task.AgentTask; -import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.PushTask; import org.apache.doris.thrift.TEtlState; import org.apache.doris.thrift.TMiniLoadRequest; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TPriority; -import org.apache.doris.thrift.TPushType; import org.apache.doris.transaction.PartitionCommitInfo; import org.apache.doris.transaction.TableCommitInfo; import org.apache.doris.transaction.TransactionState; @@ -117,7 +112,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.UUID; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; public class Load { @@ -1862,8 +1856,8 @@ public void unprotectQuorumLoadJob(LoadJob job, Database db) { partitionLoadInfo.getVersionHash(), jobId); // update table row count - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { - long tableRowCount = 0L; + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { + long indexRowCount = 0L; for (Tablet tablet : materializedIndex.getTablets()) { long tabletRowCount = 0L; for (Replica replica : tablet.getReplicas()) { @@ -1872,9 +1866,9 @@ public void unprotectQuorumLoadJob(LoadJob job, Database db) { tabletRowCount = replicaRowCount; } } - tableRowCount += tabletRowCount; + indexRowCount += tabletRowCount; } - materializedIndex.setRowCount(tableRowCount); + materializedIndex.setRowCount(indexRowCount); } // end for indices } // end for partitions } // end for tables @@ -2392,7 +2386,7 @@ private boolean processQuorumFinished(LoadJob job, Database db) { updatePartitionVersion(partition, partitionLoadInfo.getVersion(), partitionLoadInfo.getVersionHash(), jobId); - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { long tableRowCount = 0L; for (Tablet tablet : materializedIndex.getTablets()) { long tabletRowCount = 0L; @@ -2751,7 +2745,7 @@ private void checkDeleteV2(OlapTable table, Partition partition, List slotRef.setCol(column.getName()); } Map> indexIdToSchema = table.getIndexIdToSchema(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { // check table has condition column Map indexNameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); for (Column column : indexIdToSchema.get(index.getId())) { @@ -2811,221 +2805,6 @@ private void checkDeleteV2(OlapTable table, Partition partition, List } } - private void checkDelete(OlapTable table, Partition partition, List conditions, - long checkVersion, long checkVersionHash, List deleteConditions, - Map> asyncTabletIdToBackends, boolean preCheck) - throws DdlException { - // check partition state - PartitionState state = partition.getState(); - if (state != PartitionState.NORMAL) { - // ErrorReport.reportDdlException(ErrorCode.ERR_BAD_PARTITION_STATE, partition.getName(), state.name()); - throw new DdlException("Partition[" + partition.getName() + "]' state is not NORNAL: " + state.name()); - } - - // check running load job - List quorumFinishedLoadJobs = Lists.newArrayList(); - if (!checkPartitionLoadFinished(partition.getId(), quorumFinishedLoadJobs)) { - // ErrorReport.reportDdlException(ErrorCode.ERR_PARTITION_HAS_LOADING_JOBS, partition.getName()); - throw new DdlException("Partition[" + partition.getName() + "] has unfinished load jobs"); - } - - // get running async delete job - List asyncDeleteJobs = getCopiedAsyncDeleteJobs(); - - // check condition column is key column and condition value - Map nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - for (Column column : table.getBaseSchema()) { - nameToColumn.put(column.getName(), column); - } - for (Predicate condition : conditions) { - SlotRef slotRef = null; - if (condition instanceof BinaryPredicate) { - BinaryPredicate binaryPredicate = (BinaryPredicate) condition; - slotRef = (SlotRef) binaryPredicate.getChild(0); - } else if (condition instanceof IsNullPredicate) { - IsNullPredicate isNullPredicate = (IsNullPredicate) condition; - slotRef = (SlotRef) isNullPredicate.getChild(0); - } - String columnName = slotRef.getColumnName(); - if (!nameToColumn.containsKey(columnName)) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_FIELD_ERROR, columnName, table.getName()); - } - - Column column = nameToColumn.get(columnName); - if (!column.isKey()) { - // ErrorReport.reportDdlException(ErrorCode.ERR_NOT_KEY_COLUMN, columnName); - throw new DdlException("Column[" + columnName + "] is not key column"); - } - - if (condition instanceof BinaryPredicate) { - String value = null; - try { - BinaryPredicate binaryPredicate = (BinaryPredicate) condition; - value = ((LiteralExpr) binaryPredicate.getChild(1)).getStringValue(); - LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType())); - } catch (AnalysisException e) { - // ErrorReport.reportDdlException(ErrorCode.ERR_INVALID_VALUE, value); - throw new DdlException("Invalid column value[" + value + "]"); - } - } - - // set schema column name - slotRef.setCol(column.getName()); - } - - long tableId = table.getId(); - long partitionId = partition.getId(); - Map> indexIdToSchema = table.getIndexIdToSchema(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { - // check table has condition column - Map indexNameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - for (Column column : indexIdToSchema.get(index.getId())) { - indexNameToColumn.put(column.getName(), column); - } - String indexName = table.getIndexNameById(index.getId()); - for (Predicate condition : conditions) { - String columnName = null; - if (condition instanceof BinaryPredicate) { - BinaryPredicate binaryPredicate = (BinaryPredicate) condition; - columnName = ((SlotRef) binaryPredicate.getChild(0)).getColumnName(); - } else if (condition instanceof IsNullPredicate) { - IsNullPredicate isNullPredicate = (IsNullPredicate) condition; - columnName = ((SlotRef) isNullPredicate.getChild(0)).getColumnName(); - } - Column column = indexNameToColumn.get(columnName); - if (column == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_FIELD_ERROR, columnName, indexName); - } - - if (table.getKeysType() == KeysType.DUP_KEYS && !column.isKey()) { - throw new DdlException("Column[" + columnName + "] is not key column in index[" + indexName + "]"); - } - } - - // check replica version and backend alive - short replicationNum = table.getPartitionInfo().getReplicationNum(partition.getId()); - for (Tablet tablet : index.getTablets()) { - Set needAsyncBackendIds = Sets.newHashSet(); - for (Replica replica : tablet.getReplicas()) { - if (!Catalog.getCurrentSystemInfo().checkBackendAvailable(replica.getBackendId())) { - LOG.warn("backend[{}] is not alive when delete check. pre: {}", - replica.getBackendId(), preCheck); - needAsyncBackendIds.add(replica.getBackendId()); - continue; - } - - // check replica version. - // here is a little bit confused. the main idea is - // 1. check if replica catch up the version - // 2. if not catch up and this is pre check, make sure there will be right quorum finished load jobs - // to fill the version gap between 'replica committed version' and 'partition committed version'. - // 3. if not catch up and this is after check - // 1) if diff version == 1, some sync delete task may failed. add async delete task. - // 2) if diff version > 1, make sure there will be right quorum finished load jobs - // to fill the version gap between 'replica committed version' and 'delete version - 1'. - // if ok, add async delete task. - if (!replica.checkVersionCatchUp(checkVersion, checkVersionHash)) { - long replicaVersion = replica.getVersion(); - if (replicaVersion == checkVersion) { - // in this case, version is same but version hash is not. - // which mean the current replica version is a non-committed version. - // so the replica's committed version should be the previous one. - --replicaVersion; - } - - // the *diffVersion* is number of versions need to be check - // for now: - // *replicaVersion* : the 'committed version' of the replica - // *checkVersion* : - // 1) if preCheck, this is partition committed version - // 2) if not preCheck, this is delete version - long diffVersion = checkVersion - replicaVersion; - Preconditions.checkState(diffVersion > 0); - for (int i = 1; i <= diffVersion; i++) { - boolean find = false; - long theVersion = replicaVersion + i; - for (LoadJob loadJob : quorumFinishedLoadJobs) { - if (theVersion == loadJob.getPartitionLoadInfo(tableId, partitionId).getVersion()) { - find = true; - break; - } - } - - for (AsyncDeleteJob deleteJob : asyncDeleteJobs) { - if (tableId == deleteJob.getTableId() && partitionId == deleteJob.getPartitionId() - && theVersion == deleteJob.getPartitionVersion()) { - find = true; - break; - } - } - - if (!find) { - if (theVersion == checkVersion && !preCheck) { - // the sync delete task of this replica may failed. - // add async delete task after. - continue; - } else { - // this should not happend. add log to observe. - LOG.error("replica version does not catch up with version: {}-{}. " - + "replica: {}-{}-{}-{}", - checkVersion, checkVersionHash, replica.getId(), tablet.getId(), - replica.getBackendId(), replica.getState()); - throw new DdlException("Replica[" + tablet.getId() + "-" + replica.getId() - + "] is not catch up with version: " + checkVersion + "-" - + replica.getVersion()); - } - } - } - - needAsyncBackendIds.add(replica.getBackendId()); - } // end check replica version - } // end for replicas - - if (replicationNum - needAsyncBackendIds.size() < replicationNum / 2 + 1) { - String backendsStr = Joiner.on(", ").join(needAsyncBackendIds); - LOG.warn("too many unavailable replica in tablet[{}], backends:[{}]", tablet.getId(), backendsStr); - throw new DdlException("Too many replicas are not available. Wait 10 mins and try again." - + " if still not work, contact Palo RD"); - } - - if (!needAsyncBackendIds.isEmpty()) { - LOG.info("add tablet[{}] to async delete. backends: {}", - tablet.getId(), needAsyncBackendIds); - asyncTabletIdToBackends.put(tablet.getId(), needAsyncBackendIds); - } - } // end for tablets - } // end for indices - - if (deleteConditions == null) { - return; - } - - // save delete conditions - for (Predicate condition : conditions) { - if (condition instanceof BinaryPredicate) { - BinaryPredicate binaryPredicate = (BinaryPredicate) condition; - SlotRef slotRef = (SlotRef) binaryPredicate.getChild(0); - String columnName = slotRef.getColumnName(); - StringBuilder sb = new StringBuilder(); - sb.append(columnName).append(" ").append(binaryPredicate.getOp().name()).append(" \"") - .append(((LiteralExpr) binaryPredicate.getChild(1)).getStringValue()).append("\""); - deleteConditions.add(sb.toString()); - } else if (condition instanceof IsNullPredicate) { - IsNullPredicate isNullPredicate = (IsNullPredicate) condition; - SlotRef slotRef = (SlotRef) isNullPredicate.getChild(0); - String columnName = slotRef.getColumnName(); - StringBuilder sb = new StringBuilder(); - sb.append(columnName); - if (isNullPredicate.isNotNull()) { - sb.append(" IS NOT NULL"); - } else { - sb.append(" IS NULL"); - } - deleteConditions.add(sb.toString()); - } - } - } - private boolean checkAndAddRunningSyncDeleteJob(long partitionId, String partitionName) throws DdlException { // check if there are synchronized delete job under going writeLock(); @@ -3140,7 +2919,7 @@ public void delete(DeleteStmt stmt) throws DdlException { loadDeleteJob = new LoadJob(jobId, db.getId(), tableId, partitionId, jobLabel, olapTable.getIndexIdToSchemaHash(), conditions, deleteInfo); Map idToTabletLoadInfo = Maps.newHashMap(); - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : materializedIndex.getTablets()) { long tabletId = tablet.getId(); // tabletLoadInfo is empty, because delete load does not need filepath filesize info @@ -3210,227 +2989,6 @@ public void delete(DeleteStmt stmt) throws DdlException { } } - @Deprecated - public void deleteOld(DeleteStmt stmt) throws DdlException { - String dbName = stmt.getDbName(); - String tableName = stmt.getTableName(); - String partitionName = stmt.getPartitionName(); - List conditions = stmt.getDeleteConditions(); - Database db = Catalog.getInstance().getDb(dbName); - if (db == null) { - throw new DdlException("Db does not exist. name: " + dbName); - } - - DeleteInfo deleteInfo = null; - - long tableId = -1; - long partitionId = -1; - long visibleVersion = -1; - long visibleVersionHash = -1; - long newVersion = -1; - long newVersionHash = -1; - AgentBatchTask deleteBatchTask = null; - int totalReplicaNum = 0; - Map> asyncTabletIdToBackends = Maps.newHashMap(); - db.readLock(); - try { - Table table = db.getTable(tableName); - if (table == null) { - throw new DdlException("Table does not exist. name: " + tableName); - } - - if (table.getType() != TableType.OLAP) { - throw new DdlException("Not olap type table. type: " + table.getType().name()); - } - OlapTable olapTable = (OlapTable) table; - - if (olapTable.getState() != OlapTableState.NORMAL) { - throw new DdlException("Table's state is not normal: " + tableName); - } - - tableId = olapTable.getId(); - Partition partition = olapTable.getPartition(partitionName); - if (partition == null) { - throw new DdlException("Partition does not exist. name: " + partitionName); - } - partitionId = partition.getId(); - - // pre check - visibleVersion = partition.getVisibleVersion(); - visibleVersionHash = partition.getVisibleVersionHash(); - checkDelete(olapTable, partition, conditions, visibleVersion, visibleVersionHash, - null, asyncTabletIdToBackends, true); - - newVersion = visibleVersion + 1; - newVersionHash = Util.generateVersionHash(); - deleteInfo = new DeleteInfo(db.getId(), tableId, tableName, - partition.getId(), partitionName, - newVersion, newVersionHash, null); - - checkAndAddRunningSyncDeleteJob(deleteInfo.getPartitionId(), partitionName); - - // create sync delete tasks - deleteBatchTask = new AgentBatchTask(); - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { - int schemaHash = olapTable.getSchemaHashByIndexId(materializedIndex.getId()); - for (Tablet tablet : materializedIndex.getTablets()) { - long tabletId = tablet.getId(); - for (Replica replica : tablet.getReplicas()) { - - if (asyncTabletIdToBackends.containsKey(tabletId) - && asyncTabletIdToBackends.get(tabletId).contains(replica.getBackendId())) { - continue; - } - - AgentTask pushTask = new PushTask(null, replica.getBackendId(), db.getId(), - tableId, partition.getId(), - materializedIndex.getId(), tabletId, replica.getId(), - schemaHash, newVersion, - newVersionHash, null, -1L, 0, -1L, TPushType.DELETE, - conditions, false, TPriority.HIGH); - if (AgentTaskQueue.addTask(pushTask)) { - deleteBatchTask.addTask(pushTask); - ++totalReplicaNum; - } - } - } - } - } finally { - db.readUnlock(); - } - - // send tasks to backends - MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalReplicaNum); - for (AgentTask task : deleteBatchTask.getAllTasks()) { - countDownLatch.addMark(task.getBackendId(), task.getSignature()); - ((PushTask) task).setCountDownLatch(countDownLatch); - } - AgentTaskExecutor.submit(deleteBatchTask); - long timeout = Config.tablet_delete_timeout_second * 1000L * totalReplicaNum; - boolean ok = false; - try { - ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - LOG.warn("InterruptedException: ", e); - ok = false; - } - - if (!ok) { - // sync delete failed for unknown reason. - // use async delete to try to make up after. - LOG.warn("sync delete failed. try async delete. table: {}, partition: {}", tableName, partitionName); - } - - Partition partition = null; - try { - // after check - db.writeLock(); - try { - OlapTable table = (OlapTable) db.getTable(tableName); - if (table == null) { - throw new DdlException("Table does not exist. name: " + tableName); - } - - partition = table.getPartition(partitionName); - if (partition == null) { - throw new DdlException("Partition does not exist. name: " + partitionName); - } - - // after check - // 1. check partition committed version first - if (partition.getVisibleVersion() > visibleVersion - || (visibleVersion == partition.getVisibleVersion() - && visibleVersionHash != partition.getVisibleVersionHash())) { - LOG.warn("before delete version: {}-{}. after delete version: {}-{}", - visibleVersion, visibleVersionHash, - partition.getVisibleVersion(), partition.getVisibleVersionHash()); - throw new DdlException("There may have some load job done during delete job. Try again"); - } - - // 2. after check - List deleteConditions = Lists.newArrayList(); - checkDelete(table, partition, conditions, newVersion, newVersionHash, deleteConditions, - asyncTabletIdToBackends, false); - deleteInfo.setDeleteConditions(deleteConditions); - - // update partition's version - updatePartitionVersion(partition, newVersion, newVersionHash, -1); - - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { - long indexId = materializedIndex.getId(); - for (Tablet tablet : materializedIndex.getTablets()) { - long tabletId = tablet.getId(); - for (Replica replica : tablet.getReplicas()) { - ReplicaPersistInfo info = - ReplicaPersistInfo.createForCondDelete(indexId, - tabletId, - replica.getId(), - replica.getVersion(), - replica.getVersionHash(), - table.getSchemaHashByIndexId(indexId), - replica.getDataSize(), - replica.getRowCount(), - replica.getLastFailedVersion(), - replica.getLastFailedVersionHash(), - replica.getLastSuccessVersion(), - replica.getLastSuccessVersionHash()); - deleteInfo.addReplicaPersistInfo(info); - } - } - } - - writeLock(); - try { - // handle async delete jobs - if (!asyncTabletIdToBackends.isEmpty()) { - AsyncDeleteJob asyncDeleteJob = new AsyncDeleteJob(db.getId(), tableId, partition.getId(), - newVersion, newVersionHash, - conditions); - for (Long tabletId : asyncTabletIdToBackends.keySet()) { - asyncDeleteJob.addTabletId(tabletId); - } - deleteInfo.setAsyncDeleteJob(asyncDeleteJob); - idToQuorumFinishedDeleteJob.put(asyncDeleteJob.getJobId(), asyncDeleteJob); - LOG.info("finished create async delete job: {}", asyncDeleteJob.getJobId()); - } - - // save delete info - List deleteInfos = dbToDeleteInfos.get(db.getId()); - if (deleteInfos == null) { - deleteInfos = Lists.newArrayList(); - dbToDeleteInfos.put(db.getId(), deleteInfos); - } - deleteInfos.add(deleteInfo); - } finally { - writeUnlock(); - } - - // Write edit log - Catalog.getInstance().getEditLog().logFinishSyncDelete(deleteInfo); - LOG.info("delete job finished at: {}. table: {}, partition: {}", - TimeUtils.longToTimeString(System.currentTimeMillis()), tableName, partitionName); - } finally { - db.writeUnlock(); - } - } finally { - // clear tasks - List tasks = deleteBatchTask.getAllTasks(); - for (AgentTask task : tasks) { - PushTask pushTask = (PushTask) task; - AgentTaskQueue.removePushTask(pushTask.getBackendId(), pushTask.getSignature(), - pushTask.getVersion(), pushTask.getVersionHash(), - pushTask.getPushType(), pushTask.getTaskType()); - } - - writeLock(); - try { - partitionUnderDelete.remove(partitionId); - } finally { - writeUnlock(); - } - } - } - public List> getAsyncDeleteJobInfo(long jobId) { LinkedList> infos = new LinkedList>(); readLock(); diff --git a/fe/src/main/java/org/apache/doris/load/LoadChecker.java b/fe/src/main/java/org/apache/doris/load/LoadChecker.java index 83bb0abe3ca7f3..486c32c9cdb81c 100644 --- a/fe/src/main/java/org/apache/doris/load/LoadChecker.java +++ b/fe/src/main/java/org/apache/doris/load/LoadChecker.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -395,7 +396,7 @@ private Set submitPushTasks(LoadJob job, Database db) { short replicationNum = table.getPartitionInfo().getReplicationNum(partition.getId()); // check all indices (base + roll up (not include ROLLUP state index)) - List indices = partition.getMaterializedIndices(); + List indices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : indices) { long indexId = index.getId(); // if index is in rollup, then not load into it, be will automatically convert the data diff --git a/fe/src/main/java/org/apache/doris/master/Checkpoint.java b/fe/src/main/java/org/apache/doris/master/Checkpoint.java index b40c77fb82ac88..f356dc6fd30a8b 100644 --- a/fe/src/main/java/org/apache/doris/master/Checkpoint.java +++ b/fe/src/main/java/org/apache/doris/master/Checkpoint.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.common.Config; import org.apache.doris.common.util.Daemon; import org.apache.doris.metric.MetricRepo; @@ -249,7 +250,7 @@ private boolean checkMemoryEnoughToDoCheckpoint() { OlapTable olapTable = (OlapTable) table; for (Partition partition : olapTable.getPartitions()) { totalPartitionNum++; - for (MaterializedIndex materializedIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex materializedIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { totalIndexNum++; for (Tablet tablet : materializedIndex.getTablets()) { totalTabletNum++; diff --git a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java index c1afb3170919aa..79ba622f860391 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionKey; @@ -246,8 +247,8 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) thr tPartition.addToEnd_keys(range.upperEndpoint().getKeys().get(i).treeToThrift().getNodes().get(0)); } } - - for (MaterializedIndex index : partition.getMaterializedIndices()) { + + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); tPartition.setNum_buckets(index.getTablets().size()); @@ -277,7 +278,7 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) thr TOlapTablePartition tPartition = new TOlapTablePartition(); tPartition.setId(partition.getId()); // No lowerBound and upperBound for this range - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); tPartition.setNum_buckets(index.getTablets().size()); @@ -300,7 +301,7 @@ private TOlapTableLocationParam createLocation(OlapTable table) throws UserExcep Multimap allBePathsMap = HashMultimap.create(); for (Partition partition : table.getPartitions()) { int quorum = table.getPartitionInfo().getReplicationNum(partition.getId()) / 2 + 1; - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { // we should ensure the replica backend is alive // otherwise, there will be a 'unknown node id, id=xxx' error for stream load for (Tablet tablet : index.getTablets()) { diff --git a/fe/src/main/java/org/apache/doris/planner/RollupSelector.java b/fe/src/main/java/org/apache/doris/planner/RollupSelector.java index b9abc376f15a4c..610040552efa02 100644 --- a/fe/src/main/java/org/apache/doris/planner/RollupSelector.java +++ b/fe/src/main/java/org/apache/doris/planner/RollupSelector.java @@ -17,22 +17,25 @@ package org.apache.doris.planner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.common.UserException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -104,9 +107,7 @@ private List selectBestPrefixIndexRollup( outputColumns.add(col.getName()); } - final List rollups = Lists.newArrayList(); - rollups.add(partition.getBaseIndex()); - rollups.addAll(partition.getRollupIndices()); + final List rollups = partition.getMaterializedIndices(IndexExtState.VISIBLE); LOG.debug("num of rollup(base included): {}, pre aggr: {}", rollups.size(), isPreAggregation); // 1. find all rollup indexes which contains all tuple columns diff --git a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8176ff3dad6e42..c95c4964727dab 100644 --- a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -70,6 +70,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MetadataViewer; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -1180,7 +1181,7 @@ private void handleShowTablet() throws AnalysisException { if (stop) { break; } - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { if (indexId > -1 && index.getId() != indexId) { continue; } diff --git a/fe/src/main/java/org/apache/doris/task/LoadEtlTask.java b/fe/src/main/java/org/apache/doris/task/LoadEtlTask.java index 236775cde2d56a..4f37819c4800b1 100644 --- a/fe/src/main/java/org/apache/doris/task/LoadEtlTask.java +++ b/fe/src/main/java/org/apache/doris/task/LoadEtlTask.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Tablet; @@ -264,7 +265,7 @@ protected Map getTabletLoadInfos(Map allIndices = new ArrayList<>(); - allIndices.addAll(partition.getMaterializedIndices()); - MaterializedIndex rollingUpIndex = null; - RollupJob rollupJob = null; - if (table.getState() == OlapTableState.ROLLUP) { - rollupJob = (RollupJob) catalog.getRollupHandler().getAlterJob(tableId); - rollingUpIndex = rollupJob.getRollupIndex(partition.getId()); - } - + List allIndices = partition.getMaterializedIndices(IndexExtState.ALL); if (table.getState() == OlapTableState.ROLLUP || table.getState() == OlapTableState.SCHEMA_CHANGE) { /* * This is just a optimization that do our best to not let publish version tasks @@ -355,15 +346,7 @@ public void commitTransaction(long dbId, long transactionId, List errorReplicaIds) thr } int quorumReplicaNum = partitionInfo.getReplicationNum(partitionId) / 2 + 1; MaterializedIndex baseIndex = partition.getBaseIndex(); - MaterializedIndex rollingUpIndex = null; - RollupJob rollupJob = null; - if (table.getState() == OlapTableState.ROLLUP) { - rollupJob = (RollupJob) catalog.getRollupHandler().getAlterJob(tableId); - rollingUpIndex = rollupJob.getRollupIndex(partitionId); - } - List allInices = new ArrayList<>(); - allInices.addAll(partition.getMaterializedIndices()); - if (rollingUpIndex != null) { - allInices.add(rollingUpIndex); - } + List allInices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : allInices) { for (Tablet tablet : index.getTablets()) { int healthReplicaNum = 0; @@ -726,17 +689,9 @@ public void finishTransaction(long transactionId, Set errorReplicaIds) thr errorReplicaIds.remove(replica.getId()); ++healthReplicaNum; } - if (replica.getLastFailedVersion() > 0) { - // if this error replica is a base replica and it is under rollup - // then remove the rollup task and rollup job will remove the rollup replica automatically - if (index.getId() == baseIndex.getId() && rollupJob != null) { - LOG.info("base replica [{}] has errors during load, remove rollup task on related replica", replica); - rollupJob.removeReplicaRelatedTask(partition.getId(), - tablet.getId(), replica.getId(), replica.getBackendId()); - } - } } - if (index.getState() != IndexState.ROLLUP && healthReplicaNum < quorumReplicaNum) { + + if (healthReplicaNum < quorumReplicaNum) { LOG.info("publish version failed for transaction {} on tablet {}, with only {} replicas less than quorum {}", transactionState, tablet, healthReplicaNum, quorumReplicaNum); hasError = true; @@ -1030,18 +985,7 @@ private void updateCatalogAfterCommitted(TransactionState transactionState, Data for (PartitionCommitInfo partitionCommitInfo : tableCommitInfo.getIdToPartitionCommitInfo().values()) { long partitionId = partitionCommitInfo.getPartitionId(); Partition partition = table.getPartition(partitionId); - List allIndices = new ArrayList<>(); - allIndices.addAll(partition.getMaterializedIndices()); - MaterializedIndex baseIndex = partition.getBaseIndex(); - MaterializedIndex rollingUpIndex = null; - RollupJob rollupJob = null; - if (table.getState() == OlapTableState.ROLLUP) { - rollupJob = (RollupJob) catalog.getRollupHandler().getAlterJob(tableId); - rollingUpIndex = rollupJob.getRollupIndex(partition.getId()); - } - if (rollingUpIndex != null) { - allIndices.add(rollingUpIndex); - } + List allIndices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : allIndices) { List tablets = index.getTablets(); for (Tablet tablet : tablets) { @@ -1051,14 +995,6 @@ private void updateCatalogAfterCommitted(TransactionState transactionState, Data // should get from transaction state replica.updateLastFailedVersion(partitionCommitInfo.getVersion(), partitionCommitInfo.getVersionHash()); - // if this error replica is a base replica and it is under rollup - // then remove the rollup task and rollup job will remove the rollup replica automatically - if (index.getId() == baseIndex.getId() && rollupJob != null) { - LOG.debug("the base replica [{}] has error, remove the related rollup replica from rollupjob [{}]", - replica, rollupJob); - rollupJob.removeReplicaRelatedTask(partition.getId(), - tablet.getId(), replica.getId(), replica.getBackendId()); - } } } } @@ -1086,18 +1022,7 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat long newCommitVersion = partitionCommitInfo.getVersion(); long newCommitVersionHash = partitionCommitInfo.getVersionHash(); Partition partition = table.getPartition(partitionId); - MaterializedIndex baseIndex = partition.getBaseIndex(); - MaterializedIndex rollingUpIndex = null; - RollupJob rollupJob = null; - if (table.getState() == OlapTableState.ROLLUP) { - rollupJob = (RollupJob) catalog.getRollupHandler().getAlterJob(tableId); - rollingUpIndex = rollupJob.getRollupIndex(partitionId); - } - List allIndices = new ArrayList<>(); - allIndices.addAll(partition.getMaterializedIndices()); - if (rollingUpIndex != null) { - allIndices.add(rollingUpIndex); - } + List allIndices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : allIndices) { for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { @@ -1142,15 +1067,6 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat } } replica.updateVersionInfo(newVersion, newVersionHash, lastFailedVersion, lastFailedVersionHash, lastSucessVersion, lastSuccessVersionHash); - // if this error replica is a base replica and it is under rollup - // then remove the rollup task and rollup job will remove the rollup replica automatically - if (index.getId() == baseIndex.getId() - && replica.getLastFailedVersion() > 0 - && rollupJob != null) { - LOG.debug("base replica [{}] has errors during load, remove rollup task on related replica", replica); - rollupJob.removeReplicaRelatedTask(partition.getId(), - tablet.getId(), replica.getId(), replica.getBackendId()); - } } } } // end for indices diff --git a/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java b/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java index 129c03948bd531..1e4c40799205b4 100644 --- a/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java +++ b/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java @@ -40,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; public class PublishVersionDaemon extends Daemon { @@ -134,7 +133,7 @@ private void publishVersion() throws UserException { continue; } Map transTasks = transactionState.getPublishVersionTasks(); - Set transErrorReplicas = Sets.newHashSet(); + Set publishErrorReplicaIds = Sets.newHashSet(); List unfinishedTasks = Lists.newArrayList(); for (PublishVersionTask publishVersionTask : transTasks.values()) { if (publishVersionTask.isFinished()) { @@ -153,7 +152,7 @@ private void publishVersion() throws UserException { } Replica replica = tabletInvertedIndex.getReplica(tabletId, publishVersionTask.getBackendId()); if (replica != null) { - transErrorReplicas.add(replica); + publishErrorReplicaIds.add(replica.getId()); } else { LOG.info("could not find related replica with tabletid={}, backendid={}", tabletId, publishVersionTask.getBackendId()); @@ -189,7 +188,7 @@ private void publishVersion() throws UserException { Replica replica = tabletInvertedIndex.getReplica(tabletId, unfinishedTask.getBackendId()); if (replica != null) { - transErrorReplicas.add(replica); + publishErrorReplicaIds.add(replica.getId()); } else { LOG.info("could not find related replica with tabletid={}, backendid={}", tabletId, unfinishedTask.getBackendId()); @@ -207,14 +206,13 @@ private void publishVersion() throws UserException { } if (shouldFinishTxn) { - Set allErrorReplicas = transErrorReplicas.stream().map(v -> v.getId()).collect(Collectors.toSet()); - globalTransactionMgr.finishTransaction(transactionState.getTransactionId(), allErrorReplicas); + globalTransactionMgr.finishTransaction(transactionState.getTransactionId(), publishErrorReplicaIds); if (transactionState.getTransactionStatus() != TransactionStatus.VISIBLE) { // if finish transaction state failed, then update publish version time, should check // to finish after some interval transactionState.updateSendTaskTime(); LOG.debug("publish version for transation {} failed, has {} error replicas during publish", - transactionState, transErrorReplicas.size()); + transactionState, publishErrorReplicaIds.size()); } } diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java index d635d38fb0d2b5..59d4c28de454ab 100644 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java @@ -30,6 +30,7 @@ import org.apache.doris.catalog.FakeCatalog; import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.OlapTable.OlapTableState; @@ -346,6 +347,6 @@ public void testRollup3() throws Exception { // rollup hander run one cycle again, the rollup job is finishing rollupHandler.runOneCycle(); Assert.assertEquals(JobState.CANCELLED, rollupJob.getState()); - assertEquals(1, testPartition.getMaterializedIndices().size()); + assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.ALL).size()); } } diff --git a/fe/src/test/java/org/apache/doris/backup/BackupHandlerTest.java b/fe/src/test/java/org/apache/doris/backup/BackupHandlerTest.java index db90cba79c5d9b..2a4c22cf0f8f29 100644 --- a/fe/src/test/java/org/apache/doris/backup/BackupHandlerTest.java +++ b/fe/src/test/java/org/apache/doris/backup/BackupHandlerTest.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; @@ -205,7 +206,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List snapshotInfos = Maps.newHashMap(); for (Partition part : tbl.getPartitions()) { - for (MaterializedIndex idx : part.getMaterializedIndices()) { + for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { List files = Lists.newArrayList(); SnapshotInfo sinfo = new SnapshotInfo(db.getId(), tbl.getId(), part.getId(), idx.getId(), diff --git a/fe/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/src/test/java/org/apache/doris/backup/RestoreJobTest.java index 304cf58e0d43eb..07d8c770c6c5cd 100644 --- a/fe/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; @@ -214,7 +215,7 @@ boolean await(long timeout, TimeUnit unit) { partInfo.name = partition.getName(); tblInfo.partitions.put(partInfo.name, partInfo); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { BackupIndexInfo idxInfo = new BackupIndexInfo(); idxInfo.id = index.getId(); idxInfo.name = expectedRestoreTbl.getIndexNameById(index.getId()); diff --git a/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index d8548a967d6680..648b898971abe4 100644 --- a/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.SingleRangePartitionDesc; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.common.DdlException; @@ -122,7 +123,7 @@ public static boolean compareCatalog(Catalog masterCatalog, Catalog slaveCatalog || masterPartition.getCommittedVersionHash() != slavePartition.getCommittedVersionHash()) { return false; } - List allMaterializedIndices = masterPartition.getMaterializedIndices(); + List allMaterializedIndices = masterPartition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex masterIndex : allMaterializedIndices) { MaterializedIndex slaveIndex = slavePartition.getIndex(masterIndex.getId()); if (slaveIndex == null) { diff --git a/fe/src/test/java/org/apache/doris/load/LoadCheckerTest.java b/fe/src/test/java/org/apache/doris/load/LoadCheckerTest.java index f69ca373f7c5bb..1b99e0dec09fb7 100644 --- a/fe/src/test/java/org/apache/doris/load/LoadCheckerTest.java +++ b/fe/src/test/java/org/apache/doris/load/LoadCheckerTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -254,7 +255,7 @@ public void testRunLoadingJobs() throws Exception { // set tablet load infos int replicaNum = 0; Map tabletLoadInfos = new HashMap(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { replicaNum += tablet.getReplicas().size(); TabletLoadInfo tabletLoadInfo = new TabletLoadInfo("/label/path", 1L); @@ -285,7 +286,7 @@ public void testRunLoadingJobs() throws Exception { Assert.assertEquals(0, AgentTaskQueue.getTaskNum()); // update replica to new version - for (MaterializedIndex olapIndex : partition.getMaterializedIndices()) { + for (MaterializedIndex olapIndex : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : olapIndex.getTablets()) { for (Replica replica : tablet.getReplicas()) { replica.updateVersionInfo(newVersion, newVersionHash, 0L, 0L); @@ -327,7 +328,7 @@ public void testRunQuorumFinishedJobs() throws Exception { job.setIdToTableLoadInfo(idToTableLoadInfo); // set tablet load infos Map tabletLoadInfos = new HashMap(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { replica.updateVersionInfo(newVersion, newVersionHash, 0L, 0L); diff --git a/fe/src/test/java/org/apache/doris/qe/ShowExecutorTest.java b/fe/src/test/java/org/apache/doris/qe/ShowExecutorTest.java index 62ee101b1deceb..f89fae150002d8 100644 --- a/fe/src/test/java/org/apache/doris/qe/ShowExecutorTest.java +++ b/fe/src/test/java/org/apache/doris/qe/ShowExecutorTest.java @@ -95,7 +95,6 @@ public void setUp() throws Exception { // mock partition Partition partition = EasyMock.createMock(Partition.class); - EasyMock.expect(partition.getRollupIndices()).andReturn(Lists.newArrayList(index1, index2)).anyTimes(); EasyMock.expect(partition.getBaseIndex()).andReturn(index1).anyTimes(); EasyMock.replay(partition); diff --git a/fe/src/test/java/org/apache/doris/task/LoadEtlTaskTest.java b/fe/src/test/java/org/apache/doris/task/LoadEtlTaskTest.java index 32cd8fa9f7ac63..392d6ce243825e 100644 --- a/fe/src/test/java/org/apache/doris/task/LoadEtlTaskTest.java +++ b/fe/src/test/java/org/apache/doris/task/LoadEtlTaskTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Tablet; @@ -175,7 +176,7 @@ public void testRunEtlTask() throws Exception { .getIdToPartitionLoadInfo().get(paritionId).getVersion()); int tabletNum = 0; Map tabletLoadInfos = job.getIdToTabletLoadInfo(); - for (MaterializedIndex olapTable : partition.getMaterializedIndices()) { + for (MaterializedIndex olapTable : partition.getMaterializedIndices(IndexExtState.ALL)) { for (Tablet tablet : olapTable.getTablets()) { ++tabletNum; Assert.assertTrue(tabletLoadInfos.containsKey(tablet.getId())); From 5cef6197c4fa5c8b1308548b5ccf956f05f7b89d Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Jul 2019 16:37:29 +0800 Subject: [PATCH 08/79] merge diff id and alter task on BE --- be/src/agent/agent_server.cpp | 2 +- be/src/agent/task_worker_pool.cpp | 8 ++++---- be/src/olap/schema_change.cpp | 2 +- be/src/olap/task/engine_alter_tablet_task.cpp | 5 ----- .../org/apache/doris/alter/RollupJobV2.java | 8 +++++++- .../apache/doris/alter/SchemaChangeJobV2.java | 18 +++++++++++++----- .../org/apache/doris/task/AgentBatchTask.java | 2 +- gensrc/thrift/AgentService.thrift | 2 +- 8 files changed, 28 insertions(+), 19 deletions(-) diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index b7cfc91cbcec98..e0709c07015dde 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -329,7 +329,7 @@ void AgentServer::submit_tasks( break; case TTaskType::ROLLUP: case TTaskType::SCHEMA_CHANGE: - case TTaskType::ALTER_TASK: + case TTaskType::ALTER: if (task.__isset.alter_tablet_req) { _alter_tablet_workers->submit_task(task); } else { diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 3bba1abcd03903..793fe008525531 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -564,7 +564,7 @@ void* TaskWorkerPool::_alter_tablet_worker_thread_callback(void* arg_this) { switch (task_type) { case TTaskType::SCHEMA_CHANGE: case TTaskType::ROLLUP: - case TTaskType::ALTER_TASK: + case TTaskType::ALTER: worker_pool_this->_alter_tablet(worker_pool_this, agent_task_req, signatrue, @@ -602,8 +602,8 @@ void TaskWorkerPool::_alter_tablet( case TTaskType::SCHEMA_CHANGE: process_name = "schema change"; break; - case TTaskType::ALTER_TASK: - process_name = "alter table"; + case TTaskType::ALTER: + process_name = "alter"; break; default: std::string task_name; @@ -621,7 +621,7 @@ void TaskWorkerPool::_alter_tablet( TSchemaHash new_schema_hash = 0; if (status == DORIS_SUCCESS) { OLAPStatus sc_status = OLAP_SUCCESS; - if (task_type == TTaskType::ALTER_TASK) { + if (task_type == TTaskType::ALTER) { new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id; new_schema_hash = agent_task_req.alter_tablet_req_v2.new_schema_hash; EngineAlterTabletTask engine_task(agent_task_req.alter_tablet_req_v2, signature, task_type, &error_msgs, process_name); diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 091d4897b8323d..263526f4450b94 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -2134,7 +2134,7 @@ OLAPStatus SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_table Version max_continuous_version = {-1, 0}; VersionHash max_continuous_version_hash = 0; new_tablet->max_continuous_version_from_begining(&max_continuous_version, &max_continuous_version_hash); - LOG(INFO) << "find max continuous version " + LOG(INFO) << "find max continuous version of tablet=" << new_tablet->full_name() << ", start_version=" << max_continuous_version.first << ", end_version=" << max_continuous_version.second << ", version_hash=" << max_continuous_version_hash; diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp b/be/src/olap/task/engine_alter_tablet_task.cpp index 54fb1eadd911a9..5a127a14a4062d 100644 --- a/be/src/olap/task/engine_alter_tablet_task.cpp +++ b/be/src/olap/task/engine_alter_tablet_task.cpp @@ -33,11 +33,6 @@ EngineAlterTabletTask::EngineAlterTabletTask(const TAlterTabletReqV2& request, _process_name(process_name) { } OLAPStatus EngineAlterTabletTask::execute() { - LOG(INFO) << "begin to create new alter tablet. base_tablet_id=" << _alter_tablet_req.base_tablet_id - << ", base_schema_hash=" << _alter_tablet_req.base_schema_hash - << ", new_tablet_id=" << _alter_tablet_req.new_tablet_id - << ", new_schema_hash=" << _alter_tablet_req.new_schema_hash; - DorisMetrics::create_rollup_requests_total.increment(1); SchemaChangeHandler handler; diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 827414723be6e7..815aa4006765de 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; @@ -411,7 +412,12 @@ protected void runRunningJob() { private void onFinished(OlapTable tbl) { for (Partition partition : tbl.getPartitions()) { MaterializedIndex rollupIndex = partition.getIndex(rollupIndexId); - Preconditions.checkNotNull(rollupIndex, rollupIndex); + Preconditions.checkNotNull(rollupIndex, rollupIndexId); + for (Tablet tablet : rollupIndex.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + replica.setState(ReplicaState.NORMAL); + } + } rollupIndex.setState(IndexState.NORMAL); } tbl.setState(OlapTableState.NORMAL); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index c27aabb33e684e..6c1b4ccdc874ff 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; @@ -324,9 +325,9 @@ protected void runWaitingTxnJob() { Partition partition = tbl.getPartition(partitionId); Preconditions.checkNotNull(partition, partitionId); - // the rollup task will transform the data before visible version(included). - long committedVersion = partition.getCommittedVersion(); - long committedVersionHash = partition.getCommittedVersionHash(); + // the schema change task will transform the data before visible version(included). + long visibleVersion = partition.getVisibleVersion(); + long visibleVersionHash = partition.getVisibleVersionHash(); Map shadowIndexMap = partitionIndexMap.row(partitionId); for (Map.Entry entry : shadowIndexMap.entrySet()) { @@ -347,7 +348,7 @@ protected void runWaitingTxnJob() { shadowIdxId, originIdxId, shadowTabletId, originTabletId, shadowReplica.getId(), shadowSchemaHash, originSchemaHash, - committedVersion, committedVersionHash, jobId); + visibleVersion, visibleVersionHash, jobId); schemaChangeBatchTask.addTask(rollupTask); } } @@ -462,6 +463,13 @@ private void onFinished(OlapTable tbl) { partition.setBaseIndex(shadowIdx); } partition.deleteRollupIndex(originIdxId); + // set replica state + for (Tablet tablet : shadowIdx.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + replica.setState(ReplicaState.NORMAL); + } + } + shadowIdx.setState(IndexState.NORMAL); } } @@ -476,7 +484,7 @@ private void onFinished(OlapTable tbl) { // the shadow index name is '__doris_shadow_xxx', rename it to origin name 'xxx' tbl.renameIndexForSchemaChange(shadowIdxName, originIdxName); - if (originIdxId == tbl.getId()) { + if (originIdxId == tbl.getBaseIndexId()) { // set base index tbl.setNewBaseSchema(indexSchemaMap.get(shadowIdxId)); } diff --git a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java index 0a34403332bb97..0af965a7437159 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java +++ b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java @@ -25,7 +25,6 @@ import org.apache.doris.thrift.TAgentTaskRequest; import org.apache.doris.thrift.TAlterTabletReq; import org.apache.doris.thrift.TAlterTabletReqV2; -import org.apache.doris.thrift.TCancelDeleteDataReq; import org.apache.doris.thrift.TCheckConsistencyReq; import org.apache.doris.thrift.TClearAlterTaskRequest; import org.apache.doris.thrift.TClearTransactionTaskRequest; @@ -360,6 +359,7 @@ private TAgentTaskRequest toAgentTaskRequest(AgentTask task) { LOG.debug(request.toString()); } tAgentTaskRequest.setUpdate_tablet_meta_info_req(request); + } case ALTER: { CreateRollupTaskV2 createRollupTask = (CreateRollupTaskV2) task; TAlterTabletReqV2 request = createRollupTask.toThrift(); diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index a82f0b10caef25..3f6c906d90be70 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -65,7 +65,7 @@ struct TDropTabletReq { 2: optional Types.TSchemaHash schema_hash } -struct TAlterTabletReq{ +struct TAlterTabletReq { 1: required Types.TTabletId base_tablet_id 2: required Types.TSchemaHash base_schema_hash 3: required TCreateTabletReq new_tablet_req From e67a3c18ce49a12133f2b75fae4a03cc8efd422b Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Jul 2019 17:35:07 +0800 Subject: [PATCH 09/79] fix bug 1 --- be/src/agent/agent_server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index e0709c07015dde..54537f9a24e3bd 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -330,7 +330,7 @@ void AgentServer::submit_tasks( case TTaskType::ROLLUP: case TTaskType::SCHEMA_CHANGE: case TTaskType::ALTER: - if (task.__isset.alter_tablet_req) { + if (task.__isset.alter_tablet_req || task.__isset.alter_tablet_req_v2) { _alter_tablet_workers->submit_task(task); } else { status_code = TStatusCode::ANALYSIS_ERROR; From 69e2d8fa872ca632eb7842b41ae76f40d01f244f Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Jul 2019 20:20:47 +0800 Subject: [PATCH 10/79] fix cancel not delete index bug --- fe/src/main/java/org/apache/doris/alter/RollupJobV2.java | 3 ++- .../java/org/apache/doris/alter/SchemaChangeJobV2.java | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 815aa4006765de..cebb3bb2e62a81 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -185,7 +185,7 @@ protected void runPendingJob() { for (Replica rollupReplica : rollupReplicas) { long backendId = rollupReplica.getBackendId(); Preconditions.checkNotNull(tabletIdMap.get(rollupTabletId)); // baseTabletId - + countDownLatch.addMark(backendId, rollupTabletId); CreateReplicaTask createReplicaTask = new CreateReplicaTask( backendId, dbId, tableId, partitionId, rollupIndexId, rollupTabletId, rollupShortKeyColumnCount, rollupSchemaHash, @@ -461,6 +461,7 @@ private void cancelInternal() { Partition partition = tbl.getPartition(partitionId); partition.deleteRollupIndex(rollupIndexId); } + tbl.deleteIndexInfo(rollupIndexName); tbl.setState(OlapTableState.NORMAL); } } finally { diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 6c1b4ccdc874ff..e6de4eb709120f 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -85,7 +85,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { private Table partitionIndexMap = HashBasedTable.create(); // shadow index id -> origin index id private Map indexIdMap = Maps.newHashMap(); - // shadow index id -> origin index name + // shadow index id -> shadow index name(__doris_shadow_xxx) private Map indexIdToName = Maps.newHashMap(); // shadow index id -> index schema private Map> indexSchemaMap = Maps.newHashMap(); @@ -202,7 +202,7 @@ protected void runPendingJob() { List shadowReplicas = shadowTablet.getReplicas(); for (Replica shadowReplica : shadowReplicas) { long backendId = shadowReplica.getBackendId(); - + countDownLatch.addMark(backendId, shadowTabletId); CreateReplicaTask createReplicaTask = new CreateReplicaTask( backendId, dbId, tableId, partitionId, shadowIdxId, shadowTabletId, shadowShortKeyColumnCount, shadowSchemaHash, @@ -541,7 +541,9 @@ private void cancelInternal() { partition.deleteRollupIndex(shadowIdx.getId()); } } - + for (String shadowIndexName : indexIdToName.values()) { + tbl.deleteIndexInfo(shadowIndexName); + } tbl.setState(OlapTableState.NORMAL); } } finally { From ed60483b5dd36505436ba5d58522cbc7dd97b891 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 10:57:59 +0800 Subject: [PATCH 11/79] persist rollup --- .../org/apache/doris/alter/AlterHandler.java | 4 +++ .../org/apache/doris/alter/RollupHandler.java | 4 ++- .../doris/alter/SchemaChangeHandler.java | 6 +++-- .../org/apache/doris/catalog/Catalog.java | 26 ++++++++++++++++++- .../apache/doris/common/FeMetaVersion.java | 2 ++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 7d264014c8f1cc..f526ae37ee487a 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -92,6 +92,10 @@ public AlterJobV2 getAlterJobV2(long tblId) { return null; } + public Map getAlterJobsV2() { + return this.alterJobsV2; + } + @Deprecated protected void addAlterJob(AlterJob alterJob) { this.alterJobs.put(alterJob.getTableId(), alterJob); diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 9a784a0cd29518..f67003576e37e7 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -332,7 +332,9 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl } Preconditions.checkState(baseReplica.getState() == ReplicaState.NORMAL); // replica's init state is ALTER, so that tablet report process will ignore its report - Replica rollupReplica = new Replica(rollupReplicaId, backendId, rollupSchemaHash, ReplicaState.ALTER); + Replica rollupReplica = new Replica(rollupReplicaId, backendId, ReplicaState.ALTER, + Partition.PARTITION_INIT_VERSION, Partition.PARTITION_INIT_VERSION_HASH, + rollupSchemaHash); newTablet.addReplica(rollupReplica); } // end for baseReplica } // end for baseTablets diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 65101cb069b581..0bf96311ad52a1 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1033,8 +1033,10 @@ private void createJob(long dbId, OlapTable olapTable, Map alterJobs = null; ConcurrentLinkedQueue finishedOrCancelledAlterJobs = null; + Map alterJobsV2 = Maps.newHashMap(); if (type == JobType.ROLLUP) { alterJobs = this.getRollupHandler().unprotectedGetAlterJobs(); finishedOrCancelledAlterJobs = this.getRollupHandler().unprotectedGetFinishedOrCancelledAlterJobs(); + alterJobsV2 = this.getRollupHandler().getAlterJobsV2(); } else if (type == JobType.SCHEMA_CHANGE) { alterJobs = this.getSchemaChangeHandler().unprotectedGetAlterJobs(); finishedOrCancelledAlterJobs = this.getSchemaChangeHandler().unprotectedGetFinishedOrCancelledAlterJobs(); + alterJobsV2 = this.getSchemaChangeHandler().getAlterJobsV2(); } else if (type == JobType.DECOMMISSION_BACKEND) { alterJobs = this.getClusterHandler().unprotectedGetAlterJobs(); finishedOrCancelledAlterJobs = this.getClusterHandler().unprotectedGetFinishedOrCancelledAlterJobs(); } - // alter jobs int size = dis.readInt(); long newChecksum = checksum ^ size; @@ -1613,6 +1616,16 @@ public long loadAlterJob(DataInputStream dis, long checksum, JobType type) throw } } + // alter job v2 + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_58) { + size = dis.readInt(); + newChecksum ^= size; + for (int i = 0; i < size; i++) { + AlterJobV2 alterJobV2 = AlterJobV2.read(dis); + alterJobsV2.put(alterJobV2.getJobId(), alterJobV2); + } + } + return newChecksum; } @@ -1919,12 +1932,15 @@ public long saveAlterJob(DataOutputStream dos, long checksum) throws IOException public long saveAlterJob(DataOutputStream dos, long checksum, JobType type) throws IOException { Map alterJobs = null; ConcurrentLinkedQueue finishedOrCancelledAlterJobs = null; + Map alterJobsV2 = Maps.newHashMap(); if (type == JobType.ROLLUP) { alterJobs = this.getRollupHandler().unprotectedGetAlterJobs(); finishedOrCancelledAlterJobs = this.getRollupHandler().unprotectedGetFinishedOrCancelledAlterJobs(); + alterJobsV2 = this.getRollupHandler().getAlterJobsV2(); } else if (type == JobType.SCHEMA_CHANGE) { alterJobs = this.getSchemaChangeHandler().unprotectedGetAlterJobs(); finishedOrCancelledAlterJobs = this.getSchemaChangeHandler().unprotectedGetFinishedOrCancelledAlterJobs(); + alterJobsV2 = this.getSchemaChangeHandler().getAlterJobsV2(); } else if (type == JobType.DECOMMISSION_BACKEND) { alterJobs = this.getClusterHandler().unprotectedGetAlterJobs(); finishedOrCancelledAlterJobs = this.getClusterHandler().unprotectedGetFinishedOrCancelledAlterJobs(); @@ -1952,6 +1968,14 @@ public long saveAlterJob(DataOutputStream dos, long checksum, JobType type) thro alterJob.write(dos); } + // alter job v2 + size = alterJobsV2.size(); + checksum ^= size; + dos.writeInt(size); + for (AlterJobV2 alterJobV2 : alterJobsV2.values()) { + alterJobV2.write(dos); + } + return checksum; } diff --git a/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java index 9e7c42ce2c5357..f2e7b0e5a188b9 100644 --- a/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -130,4 +130,6 @@ public final class FeMetaVersion { public static final int VERSION_59 = 59; // refactor date literal public static final int VERSION_60 = 60; + // for alter job v2 + public static final int VERSION_61 = 61; } From aa64fe0ecda7a9da35f220dcd56627295b27aa23 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 16:04:35 +0800 Subject: [PATCH 12/79] add handle finish alter task logic --- be/src/olap/schema_change.cpp | 5 +- .../org/apache/doris/alter/AlterHandler.java | 72 +++++++++++++++++++ .../org/apache/doris/alter/RollupJobV2.java | 10 +-- .../apache/doris/alter/SchemaChangeJobV2.java | 8 +-- .../doris/catalog/TabletInvertedIndex.java | 6 ++ .../org/apache/doris/master/MasterImpl.java | 15 +++- .../org/apache/doris/task/AgentBatchTask.java | 2 +- ...ollupTaskV2.java => AlterReplicaTask.java} | 41 ++++++----- 8 files changed, 130 insertions(+), 29 deletions(-) rename fe/src/main/java/org/apache/doris/task/{CreateRollupTaskV2.java => AlterReplicaTask.java} (65%) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 263526f4450b94..21d312eae4e90b 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1335,9 +1335,12 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe if (res != OLAP_SUCCESS) { break; } - res = _validate_alter_result(new_tablet, request); } while(0); + // _validate_alter_result should be outside the above while loop. + // to avoid requiring the header lock twice. + res = _validate_alter_result(new_tablet, request); + // if failed convert history data, then just remove the new tablet if (res != OLAP_SUCCESS) { LOG(WARNING) << "failed to alter tablet. base_tablet=" << base_tablet->full_name() diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index f526ae37ee487a..7fd1c19f073165 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -22,7 +22,11 @@ import org.apache.doris.analysis.CancelStmt; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Tablet; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; @@ -30,8 +34,10 @@ import org.apache.doris.common.util.Daemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.thrift.TTabletInfo; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; @@ -335,4 +341,70 @@ public Integer getAlterJobNumByState(JobState state) { } return jobNum; } + + /* + * Handle the finish report of alter task. + * If task is success, which means the history data before specified version has been transformed successfully. + * So here we should modify the replica's version. + * We assume that the specified version is X. + * Case 1: + * After alter table process starts, there is no new load job being submitted. So the new replica + * should be with version (1-0). So we just modify the replica's version to partition's visible version, which is X. + * Case 2: + * After alter table process starts, there are some load job being processed. + * Case 2.1: + * Only one new load job, and it failed on this replica. so the replica's last failed version should be X + 1 + * and version is still 1. We should modify the replica's version to (last failed version - 1) + * Case 2.2 + * There are new load jobs after alter task, and at least one of them is succeed on this replica. + * So the replica's version should be larger than X. So we don't need to modify the replica version + * because its already looks like normal. + */ + public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundException { + Database db = Catalog.getCurrentCatalog().getDb(task.getDbId()); + if (db == null) { + throw new MetaNotFoundException("database " + task.getDbId() + " does not exist"); + } + + db.writeLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(task.getTableId()); + if (tbl == null) { + throw new MetaNotFoundException("tbl " + task.getTableId() + " does not exist"); + } + Partition partition = tbl.getPartition(task.getPartitionId()); + if (partition == null) { + throw new MetaNotFoundException("partition " + task.getPartitionId() + " does not exist"); + } + MaterializedIndex index = partition.getIndex(task.getIndexId()); + if (index == null) { + throw new MetaNotFoundException("index " + task.getIndexId() + " does not exist"); + } + Tablet tablet = index.getTablet(task.getTabletId()); + Preconditions.checkNotNull(tablet, task.getTabletId()); + Replica replica = tablet.getReplicaById(task.getNewReplicaId()); + if (replica == null) { + throw new MetaNotFoundException("replica " + task.getNewReplicaId() + " does not exist"); + } + + LOG.info("before handle alter task replica: {}, task version: {}-{}", + replica, task.getVersion(), task.getVersionHash()); + if (replica.getVersion() > task.getVersion()) { + // Case 2.2, do nothing + } else { + if (replica.getLastFailedVersion() > task.getVersion()) { + // Case 2.1 + replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); + } else { + Preconditions.checkState(replica.getLastFailedVersion() == -1); + // Case 1 + replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); + } + } + + LOG.info("after handle alter task replica: {}", replica); + } finally { + db.writeUnlock(); + } + } } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index cebb3bb2e62a81..d929faf6feb553 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -39,8 +39,8 @@ import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.task.CreateReplicaTask; -import org.apache.doris.task.CreateRollupTaskV2; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTaskType; @@ -186,6 +186,8 @@ protected void runPendingJob() { long backendId = rollupReplica.getBackendId(); Preconditions.checkNotNull(tabletIdMap.get(rollupTabletId)); // baseTabletId countDownLatch.addMark(backendId, rollupTabletId); + // create replica with version 1. + // version will be updated by following load process, or when rollup task finished. CreateReplicaTask createReplicaTask = new CreateReplicaTask( backendId, dbId, tableId, partitionId, rollupIndexId, rollupTabletId, rollupShortKeyColumnCount, rollupSchemaHash, @@ -311,12 +313,12 @@ protected void runWaitingTxnJob() { List rollupReplicas = rollupTablet.getReplicas(); for (Replica rollupReplica : rollupReplicas) { - CreateRollupTaskV2 rollupTask = new CreateRollupTaskV2( + AlterReplicaTask rollupTask = new AlterReplicaTask( rollupReplica.getBackendId(), dbId, tableId, partitionId, rollupIndexId, baseIndexId, rollupTabletId, baseTabletId, rollupReplica.getId(), rollupSchemaHash, baseSchemaHash, - visibleVersion, visibleVersionHash, jobId); + visibleVersion, visibleVersionHash, jobId, JobType.ROLLUP); rollupBatchTask.addTask(rollupTask); } } @@ -702,7 +704,7 @@ public List> getUnfinishedTasks(int limit) { if (jobState == JobState.RUNNING) { List tasks = rollupBatchTask.getUnfinishedTasks(limit); for (AgentTask agentTask : tasks) { - CreateRollupTaskV2 rollupTask = (CreateRollupTaskV2)agentTask; + AlterReplicaTask rollupTask = (AlterReplicaTask)agentTask; List info = Lists.newArrayList(); info.add(String.valueOf(rollupTask.getBackendId())); info.add(String.valueOf(rollupTask.getBaseTabletId())); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index e6de4eb709120f..5f2795863e788c 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -39,8 +39,8 @@ import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.task.CreateReplicaTask; -import org.apache.doris.task.CreateRollupTaskV2; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTaskType; @@ -343,12 +343,12 @@ protected void runWaitingTxnJob() { long originTabletId = partitionIndexTabletMap.get(partitionId, shadowIdxId).get(shadowTabletId); List shadowReplicas = shadowTablet.getReplicas(); for (Replica shadowReplica : shadowReplicas) { - CreateRollupTaskV2 rollupTask = new CreateRollupTaskV2( + AlterReplicaTask rollupTask = new AlterReplicaTask( shadowReplica.getBackendId(), dbId, tableId, partitionId, shadowIdxId, originIdxId, shadowTabletId, originTabletId, shadowReplica.getId(), shadowSchemaHash, originSchemaHash, - visibleVersion, visibleVersionHash, jobId); + visibleVersion, visibleVersionHash, jobId, JobType.SCHEMA_CHANGE); schemaChangeBatchTask.addTask(rollupTask); } } @@ -716,7 +716,7 @@ public List> getUnfinishedTasks(int limit) { if (jobState == JobState.RUNNING) { List tasks = schemaChangeBatchTask.getUnfinishedTasks(limit); for (AgentTask agentTask : tasks) { - CreateRollupTaskV2 rollupTask = (CreateRollupTaskV2)agentTask; + AlterReplicaTask rollupTask = (AlterReplicaTask)agentTask; List info = Lists.newArrayList(); info.add(String.valueOf(rollupTask.getBackendId())); info.add(String.valueOf(rollupTask.getBaseTabletId())); diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index a6006620554bff..9cdee89fbe2668 100644 --- a/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -347,6 +347,12 @@ private boolean needSync(Replica replicaInFe, TTabletInfo backendTabletInfo) { // it will be handled in needRecovery() return false; } + + if (replicaInFe.getState() == ReplicaState.ALTER) { + // ignore the replica is ALTER state. its version will be taken care by load process and alter table process + return false; + } + long versionInFe = replicaInFe.getVersion(); long versionHashInFe = replicaInFe.getVersionHash(); diff --git a/fe/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/src/main/java/org/apache/doris/master/MasterImpl.java index 770059dffa2812..cf2f079342994e 100644 --- a/fe/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/src/main/java/org/apache/doris/master/MasterImpl.java @@ -40,13 +40,13 @@ import org.apache.doris.system.Backend; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.task.CheckConsistencyTask; import org.apache.doris.task.ClearAlterTask; import org.apache.doris.task.ClearTransactionTask; import org.apache.doris.task.CloneTask; import org.apache.doris.task.CreateReplicaTask; import org.apache.doris.task.CreateRollupTask; -import org.apache.doris.task.CreateRollupTaskV2; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; import org.apache.doris.task.PublishVersionTask; @@ -771,8 +771,17 @@ public TFetchResourceResult fetchResource() { } private void finishAlterTask(AgentTask task) { - CreateRollupTaskV2 createRollupTaskV2 = (CreateRollupTaskV2) task; - createRollupTaskV2.setFinished(true); + AlterReplicaTask alterTask = (AlterReplicaTask) task; + try { + if (alterTask.getTaskType() == TTaskType.ROLLUP) { + Catalog.getCurrentCatalog().getRollupHandler().handleFinishAlterTask(alterTask); + } else if (alterTask.getTaskType() == TTaskType.SCHEMA_CHANGE) { + Catalog.getCurrentCatalog().getSchemaChangeHandler().handleFinishAlterTask(alterTask); + } + alterTask.setFinished(true); + } catch (MetaNotFoundException e) { + LOG.warn("failed to handle finish alter task: {}", e.getMessage()); + } AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); } } diff --git a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java index 0af965a7437159..ea5077b475e6eb 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java +++ b/fe/src/main/java/org/apache/doris/task/AgentBatchTask.java @@ -361,7 +361,7 @@ private TAgentTaskRequest toAgentTaskRequest(AgentTask task) { tAgentTaskRequest.setUpdate_tablet_meta_info_req(request); } case ALTER: { - CreateRollupTaskV2 createRollupTask = (CreateRollupTaskV2) task; + AlterReplicaTask createRollupTask = (AlterReplicaTask) task; TAlterTabletReqV2 request = createRollupTask.toThrift(); if (LOG.isDebugEnabled()) { LOG.debug(request.toString()); diff --git a/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java b/fe/src/main/java/org/apache/doris/task/AlterReplicaTask.java similarity index 65% rename from fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java rename to fe/src/main/java/org/apache/doris/task/AlterReplicaTask.java index 86be00a1035cda..d682383b14c38c 100644 --- a/fe/src/main/java/org/apache/doris/task/CreateRollupTaskV2.java +++ b/fe/src/main/java/org/apache/doris/task/AlterReplicaTask.java @@ -17,51 +17,56 @@ package org.apache.doris.task; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.thrift.TAlterTabletReqV2; import org.apache.doris.thrift.TTaskType; /* - * This is to replace the old CreateRollupTask. - * This task only do data transformation from base tablet to rollup tablet. - * The rollup tablet has already been created before. + * This task is used for alter table process, such as rollup and schema change + * The task will do data transformation from base replica to new replica. + * The new replica should be created before. + * The new replica can be a rollup replica, or a shadow replica of schema change. */ -public class CreateRollupTaskV2 extends AgentTask { +public class AlterReplicaTask extends AgentTask { private long baseTabletId; - private long rollupReplicaId; + private long newReplicaId; private int baseSchemaHash; - private int rollupSchemaHash; + private int newSchemaHash; private long version; private long versionHash; private long jobId; + private AlterJobV2.JobType jobType; - public CreateRollupTaskV2(long backendId, long dbId, long tableId, + public AlterReplicaTask(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId, long baseIndexId, long rollupTabletId, - long baseTabletId, long rollupReplicaId, int rollupSchemaHash, int baseSchemaHash, - long version, long versionHash, long jobId) { + long baseTabletId, long newReplicaId, int newSchemaHash, int baseSchemaHash, + long version, long versionHash, long jobId, AlterJobV2.JobType jobType) { super(null, backendId, TTaskType.ALTER, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); this.baseTabletId = baseTabletId; - this.rollupReplicaId = rollupReplicaId; + this.newReplicaId = newReplicaId; - this.rollupSchemaHash = rollupSchemaHash; + this.newSchemaHash = newSchemaHash; this.baseSchemaHash = baseSchemaHash; this.version = version; this.versionHash = versionHash; this.jobId = jobId; + + this.jobType = jobType; } public long getBaseTabletId() { return baseTabletId; } - public long getRollupReplicaId() { - return rollupReplicaId; + public long getNewReplicaId() { + return newReplicaId; } - public int getRollupSchemaHash() { - return rollupSchemaHash; + public int getNewSchemaHash() { + return newSchemaHash; } public int getBaseSchemaHash() { @@ -80,8 +85,12 @@ public long getJobId() { return jobId; } + public AlterJobV2.JobType getJobType() { + return jobType; + } + public TAlterTabletReqV2 toThrift() { - TAlterTabletReqV2 req = new TAlterTabletReqV2(baseTabletId, signature, baseSchemaHash, rollupSchemaHash); + TAlterTabletReqV2 req = new TAlterTabletReqV2(baseTabletId, signature, baseSchemaHash, newSchemaHash); req.setAlter_version(version); req.setAlter_version_hash(versionHash); return req; From ab3f9948e67000c051f6cf96e3f5c6c3ca3aebdf Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 18:28:36 +0800 Subject: [PATCH 13/79] visualize index --- .../org/apache/doris/alter/AlterHandler.java | 7 +++++++ .../org/apache/doris/alter/AlterJobV2.java | 4 ++++ .../org/apache/doris/alter/RollupHandler.java | 5 ----- .../org/apache/doris/alter/RollupJobV2.java | 4 +++- .../apache/doris/alter/SchemaChangeJobV2.java | 4 +++- .../org/apache/doris/catalog/Partition.java | 18 ++++++++++++++++++ .../org/apache/doris/master/MasterImpl.java | 5 +++-- .../java/org/apache/doris/persist/EditLog.java | 6 +++++- 8 files changed, 43 insertions(+), 10 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 7fd1c19f073165..5fba994300bfa8 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -407,4 +407,11 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce db.writeUnlock(); } } + + // replay the alter job v2 + public void replayAlterJobV2(AlterJobV2 alterJob) { + // always replace the alter job with the newly replayed one + alterJobsV2.put(alterJob.getJobId(), alterJob); + alterJob.replay(); + } } diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 948dd52c3027b4..736826dda084a0 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -167,6 +167,10 @@ protected void getInfo(List> infos) { throw new NotImplementedException(); } + public void replay() { + throw new NotImplementedException(); + } + public static AlterJobV2 read(DataInput in) throws IOException { JobType type = JobType.valueOf(Text.readString(in)); switch (type) { diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index f67003576e37e7..49ddef518e8960 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -680,9 +680,4 @@ public void cancel(CancelStmt stmt) throws DdlException { jobDone(rollupJob); } } - - // replay the rollup job by job state - public void replayRollupJobV2(RollupJobV2 rollupJob) { - rollupJob.replay(); - } } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index d929faf6feb553..1c0e6ab788353a 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -258,6 +258,7 @@ private void addRollupIndexToCatalog(OlapTable tbl) { long partitionId = partition.getId(); MaterializedIndex rollupIndex = this.partitionIdToRollupIndex.get(partitionId); Preconditions.checkNotNull(rollupIndex); + Preconditions.checkState(rollupIndex.getState() == IndexState.SHADOW, rollupIndex.getState()); partition.createRollupIndex(rollupIndex); } @@ -420,7 +421,7 @@ private void onFinished(OlapTable tbl) { replica.setState(ReplicaState.NORMAL); } } - rollupIndex.setState(IndexState.NORMAL); + partition.visualiseShadowIndex(rollupIndexId); } tbl.setState(OlapTableState.NORMAL); } @@ -657,6 +658,7 @@ private void replayCancelled() { LOG.info("replay cancelled rollup job: {}", jobId); } + @Override public void replay() { switch (jobState) { case PENDING: diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 5f2795863e788c..a9251a89b29f4a 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -277,6 +277,7 @@ private void addShadowIndexToCatalog(OlapTable tbl) { } Map shadowIndexMap = partitionIndexMap.row(partitionId); for (MaterializedIndex shadowIndex : shadowIndexMap.values()) { + Preconditions.checkState(shadowIndex.getState() == IndexState.SHADOW, shadowIndex.getState()); partition.createRollupIndex(shadowIndex); } } @@ -470,7 +471,7 @@ private void onFinished(OlapTable tbl) { } } - shadowIdx.setState(IndexState.NORMAL); + partition.visualiseShadowIndex(shadowIdx.getId()); } } @@ -662,6 +663,7 @@ private void replayCancelled() { LOG.info("replay cancelled rollup job: {}", jobId); } + @Override public void replay() { switch (jobState) { case PENDING: diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 10b4a8ab15daf0..10e49d59d7ca47 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -19,12 +19,14 @@ import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; +import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Util; import org.apache.doris.meta.MetaContext; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.kudu.client.shaded.com.google.common.collect.Lists; @@ -271,6 +273,22 @@ public boolean hasData() { return !(visibleVersion == PARTITION_INIT_VERSION && visibleVersionHash == PARTITION_INIT_VERSION_HASH); } + /* + * Change the index' state from SHADOW to NORMAL + * Also move it to idToVisibleRollupIndex + */ + public boolean visualiseShadowIndex(long shadowIndexId) { + MaterializedIndex shadowIdx = idToShadowIndex.remove(shadowIndexId); + if (shadowIdx == null) { + return false; + } + Preconditions.checkState(!idToVisibleRollupIndex.containsKey(shadowIndexId), shadowIndexId); + shadowIdx.setState(IndexState.NORMAL); + idToVisibleRollupIndex.put(shadowIndexId, shadowIdx); + LOG.info("visualise the shadow index: {}", shadowIndexId); + return true; + } + public static Partition read(DataInput in) throws IOException { Partition partition = new Partition(); partition.readFields(in); diff --git a/fe/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/src/main/java/org/apache/doris/master/MasterImpl.java index cf2f079342994e..75d29f0b156f40 100644 --- a/fe/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/src/main/java/org/apache/doris/master/MasterImpl.java @@ -18,6 +18,7 @@ package org.apache.doris.master; import org.apache.doris.alter.AlterJob; +import org.apache.doris.alter.AlterJobV2.JobType; import org.apache.doris.alter.RollupHandler; import org.apache.doris.alter.RollupJob; import org.apache.doris.alter.SchemaChangeHandler; @@ -773,9 +774,9 @@ public TFetchResourceResult fetchResource() { private void finishAlterTask(AgentTask task) { AlterReplicaTask alterTask = (AlterReplicaTask) task; try { - if (alterTask.getTaskType() == TTaskType.ROLLUP) { + if (alterTask.getJobType() == JobType.ROLLUP) { Catalog.getCurrentCatalog().getRollupHandler().handleFinishAlterTask(alterTask); - } else if (alterTask.getTaskType() == TTaskType.SCHEMA_CHANGE) { + } else if (alterTask.getJobType() == JobType.SCHEMA_CHANGE) { Catalog.getCurrentCatalog().getSchemaChangeHandler().handleFinishAlterTask(alterTask); } alterTask.setFinished(true); diff --git a/fe/src/main/java/org/apache/doris/persist/EditLog.java b/fe/src/main/java/org/apache/doris/persist/EditLog.java index 9581829e627176..9fe20493981176 100644 --- a/fe/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/src/main/java/org/apache/doris/persist/EditLog.java @@ -22,6 +22,7 @@ import org.apache.doris.alter.RollupJob; import org.apache.doris.alter.RollupJobV2; import org.apache.doris.alter.SchemaChangeJob; +import org.apache.doris.alter.SchemaChangeJobV2; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Repository; @@ -684,7 +685,10 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { AlterJobV2 alterJob = (AlterJobV2) journal.getData(); switch (alterJob.getType()) { case ROLLUP: - catalog.getRollupHandler().replayRollupJobV2((RollupJobV2) alterJob); + catalog.getRollupHandler().replayAlterJobV2((RollupJobV2) alterJob); + break; + case SCHEMA_CHANGE: + catalog.getSchemaChangeHandler().replayAlterJobV2((SchemaChangeJobV2) alterJob); break; default: break; From 2dce605c1f24f8f398841638362ece59fd4a3891 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 19:06:58 +0800 Subject: [PATCH 14/79] fix table name null --- fe/src/main/java/org/apache/doris/alter/AlterJobV2.java | 2 ++ fe/src/main/java/org/apache/doris/catalog/Replica.java | 8 ++++++-- fe/src/main/java/org/apache/doris/catalog/Tablet.java | 5 ++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 736826dda084a0..4aa4954dc388fe 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -190,6 +190,7 @@ public synchronized void write(DataOutput out) throws IOException { out.writeLong(jobId); out.writeLong(dbId); out.writeLong(tableId); + Text.writeString(out, tableName); Text.writeString(out, errMsg); out.writeLong(createTimeMs); @@ -206,6 +207,7 @@ public synchronized void readFields(DataInput in) throws IOException { jobId = in.readLong(); dbId = in.readLong(); tableId = in.readLong(); + tableName = Text.readString(in); errMsg = Text.readString(in); createTimeMs = in.readLong(); diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index 7de077f2e223b1..20d9c679d55288 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -47,8 +47,12 @@ public enum ReplicaState { ALTER, // replica is under rollup or schema change DECOMMISSION; // replica is ready to be deleted - public boolean isLoadable() { - return this == ReplicaState.NORMAL || this == ReplicaState.SCHEMA_CHANGE || this == ReplicaState.ALTER; + public boolean canLoad() { + return this == NORMAL || this == SCHEMA_CHANGE || this == ALTER; + } + + public boolean canQuery() { + return this == NORMAL || this == SCHEMA_CHANGE; } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/src/main/java/org/apache/doris/catalog/Tablet.java index 2753e7d539e94a..c0fa1816bf5496 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java @@ -178,8 +178,7 @@ public List getNormalReplicaBackendIds() { } ReplicaState state = replica.getState(); - if (infoService.checkBackendAlive(replica.getBackendId()) - && (state == ReplicaState.NORMAL || state == ReplicaState.SCHEMA_CHANGE)) { + if (infoService.checkBackendAlive(replica.getBackendId()) && state.canLoad()) { beIds.add(replica.getBackendId()); } } @@ -213,7 +212,7 @@ public void getQueryableReplicas(List allQuerableReplica, List } ReplicaState state = replica.getState(); - if (state == ReplicaState.NORMAL || state == ReplicaState.SCHEMA_CHANGE) { + if (state.canQuery()) { // replica.getSchemaHash() == -1 is for compatibility if (replica.checkVersionCatchUp(visibleVersion, visibleVersionHash) && (replica.getSchemaHash() == -1 || replica.getSchemaHash() == schemaHash)) { From 63776e26ea8fb108e6efc466380647e0988473ac Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 19:52:25 +0800 Subject: [PATCH 15/79] add ignore alter in replica --- .../java/org/apache/doris/alter/AlterHandler.java | 2 +- .../java/org/apache/doris/alter/AlterJobV2.java | 3 ++- .../org/apache/doris/alter/RollupHandler.java | 4 +++- .../java/org/apache/doris/alter/RollupJobV2.java | 7 ++++--- .../org/apache/doris/alter/SchemaChangeJobV2.java | 7 ++++--- .../java/org/apache/doris/backup/RestoreJob.java | 2 +- .../java/org/apache/doris/catalog/Replica.java | 15 ++++++++++++++- .../java/org/apache/doris/catalog/Tablet.java | 2 +- .../org/apache/doris/catalog/TabletStatMgr.java | 2 +- .../org/apache/doris/clone/TabletSchedCtx.java | 2 +- .../org/apache/doris/clone/TabletScheduler.java | 2 +- .../apache/doris/http/rest/RowCountAction.java | 2 +- fe/src/main/java/org/apache/doris/load/Load.java | 2 +- .../doris/transaction/GlobalTransactionMgr.java | 6 +++--- .../org/apache/doris/catalog/ReplicaTest.java | 6 +++--- 15 files changed, 41 insertions(+), 23 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 5fba994300bfa8..b94ff24acc8d06 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -396,7 +396,7 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce // Case 2.1 replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); } else { - Preconditions.checkState(replica.getLastFailedVersion() == -1); + Preconditions.checkState(replica.getLastFailedVersion() == -1, replica.getLastFailedVersion()); // Case 1 replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); } diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 4aa4954dc388fe..ddb635eb33d9a2 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -130,6 +130,7 @@ public boolean isDone() { public synchronized void run() { if (isTimeout()) { cancel("Timeout"); + return; } switch (jobState) { @@ -159,7 +160,7 @@ protected void runRunningJob() { throw new NotImplementedException(); } - public synchronized void cancel(String errMsg) { + public synchronized boolean cancel(String errMsg) { throw new NotImplementedException(); } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 49ddef518e8960..fccb7de7e7aabe 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -663,7 +663,9 @@ public void cancel(CancelStmt stmt) throws DdlException { // find from new alter jobs first rollupJobV2 = getAlterJobV2(olapTable.getId()); if (rollupJobV2 != null) { - rollupJobV2.cancel("user cancelled"); + if (!rollupJobV2.cancel("user cancelled")) { + throw new DdlException("Job can not be cancelled. State: " + rollupJobV2.getJobState()); + } } else { rollupJob = getAlterJob(olapTable.getId()); Preconditions.checkNotNull(rollupJob, olapTable.getId()); diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 1c0e6ab788353a..62352ebccf3b81 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -386,7 +386,7 @@ protected void runRunningJob() { int healthyReplicaNum = 0; for (Replica replica : replicas) { if (replica.getLastFailedVersion() < 0 - && replica.checkVersionCatchUp(visiableVersion, visiableVersionHash)) { + && replica.checkVersionCatchUp(visiableVersion, visiableVersionHash, false)) { healthyReplicaNum++; } } @@ -431,9 +431,9 @@ private void onFinished(OlapTable tbl) { * We need to clean any possible residual of this job. */ @Override - public synchronized void cancel(String errMsg) { + public synchronized boolean cancel(String errMsg) { if (jobState.isFinalState()) { - return; + return false; } cancelInternal(); @@ -443,6 +443,7 @@ public synchronized void cancel(String errMsg) { this.finishedTimeMs = System.currentTimeMillis(); LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + return true; } private void cancelInternal() { diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index a9251a89b29f4a..8e7b70d1f7f7ab 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -418,7 +418,7 @@ protected void runRunningJob() { int healthyReplicaNum = 0; for (Replica replica : replicas) { if (replica.getLastFailedVersion() < 0 - && replica.checkVersionCatchUp(visiableVersion, visiableVersionHash)) { + && replica.checkVersionCatchUp(visiableVersion, visiableVersionHash, false)) { healthyReplicaNum++; } } @@ -504,9 +504,9 @@ private void onFinished(OlapTable tbl) { * We need to clean any possible residual of this job. */ @Override - public synchronized void cancel(String errMsg) { + public synchronized boolean cancel(String errMsg) { if (jobState.isFinalState()) { - return; + return false; } cancelInternal(); @@ -516,6 +516,7 @@ public synchronized void cancel(String errMsg) { this.finishedTimeMs = System.currentTimeMillis(); LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); + return true; } private void cancelInternal() { diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java index 4e330646e374e1..a0ef11a00b275b 100644 --- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1224,7 +1224,7 @@ private Status allTabletCommitted(boolean isReplay) { for (Tablet tablet : idx.getTablets()) { for (Replica replica : tablet.getReplicas()) { if (!replica.checkVersionCatchUp(part.getVisibleVersion(), - part.getVisibleVersionHash())) { + part.getVisibleVersionHash(), false)) { replica.updateVersionInfo(part.getVisibleVersion(), part.getVisibleVersionHash(), replica.getDataSize(), replica.getRowCount()); } diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index 20d9c679d55288..7513644c7915c4 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -386,7 +386,20 @@ public synchronized void updateLastFailedVersion(long lastFailedVersion, long la this.lastSuccessVersion, this.lastSuccessVersionHash, dataSize, rowCount); } - public boolean checkVersionCatchUp(long expectedVersion, long expectedVersionHash) { + /* + * Check whether the replica's version catch up with the expected version. + * If ignoreAlter is true, and state is ALTER, and replica's version is PARTITION_INIT_VERSION, just return true, ignore the version. + * This is for the case that when altering table, the newly created replica's version is PARTITION_INIT_VERSION, + * but we need to treat it as a "normal" replica which version is supposed to be "catch-up". + * But if state is ALTER but version larger than PARTITION_INIT_VERSION, which means this replica + * is already updated by load process, so we need to consider its version. + */ + public boolean checkVersionCatchUp(long expectedVersion, long expectedVersionHash, boolean ignoreAlter) { + if (ignoreAlter && state == ReplicaState.ALTER && version == Partition.PARTITION_INIT_VERSION + && versionHash == Partition.PARTITION_INIT_VERSION_HASH) { + return true; + } + if (expectedVersion == Partition.PARTITION_INIT_VERSION && expectedVersionHash == Partition.PARTITION_INIT_VERSION_HASH) { // no data is loaded into this replica, just return true diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/src/main/java/org/apache/doris/catalog/Tablet.java index c0fa1816bf5496..485053ae69e8fa 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java @@ -214,7 +214,7 @@ public void getQueryableReplicas(List allQuerableReplica, List ReplicaState state = replica.getState(); if (state.canQuery()) { // replica.getSchemaHash() == -1 is for compatibility - if (replica.checkVersionCatchUp(visibleVersion, visibleVersionHash) + if (replica.checkVersionCatchUp(visibleVersion, visibleVersionHash, false) && (replica.getSchemaHash() == -1 || replica.getSchemaHash() == schemaHash)) { allQuerableReplica.add(replica); if (localBeId != -1 && replica.getBackendId() == localBeId) { diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java index d5e9b015c87fc2..a2fcd62d055e92 100644 --- a/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java +++ b/fe/src/main/java/org/apache/doris/catalog/TabletStatMgr.java @@ -125,7 +125,7 @@ protected void runOneCycle() { for (Tablet tablet : index.getTablets()) { long tabletRowCount = 0L; for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(version, versionHash) + if (replica.checkVersionCatchUp(version, versionHash, false) && replica.getRowCount() > tabletRowCount) { tabletRowCount = replica.getRowCount(); } diff --git a/fe/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 77166eb51d22b5..8e1c22f230aa4f 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -478,7 +478,7 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc continue; } - if (!replica.checkVersionCatchUp(visibleVersion, visibleVersionHash)) { + if (!replica.checkVersionCatchUp(visibleVersion, visibleVersionHash, false)) { continue; } diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java index 7994fcf463cd1a..5c62f885ccd7be 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -744,7 +744,7 @@ private boolean deleteReplicaWithFailedVersion(TabletSchedCtx tabletCtx, boolean private boolean deleteReplicaWithLowerVersion(TabletSchedCtx tabletCtx, boolean force) throws SchedException { for (Replica replica : tabletCtx.getReplicas()) { - if (!replica.checkVersionCatchUp(tabletCtx.getCommittedVersion(), tabletCtx.getCommittedVersionHash())) { + if (!replica.checkVersionCatchUp(tabletCtx.getCommittedVersion(), tabletCtx.getCommittedVersionHash(), false)) { deleteReplicaInternal(tabletCtx, replica, "lower version", force); return true; } diff --git a/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java b/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java index e902043fec0342..0ed9bc7e8a0a87 100644 --- a/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java +++ b/fe/src/main/java/org/apache/doris/http/rest/RowCountAction.java @@ -98,7 +98,7 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept for (Tablet tablet : index.getTablets()) { long tabletRowCount = 0L; for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(version, versionHash) + if (replica.checkVersionCatchUp(version, versionHash, false) && replica.getRowCount() > tabletRowCount) { tabletRowCount = replica.getRowCount(); } diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 1db6857274baa8..88ae45cd57ffbd 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -1634,7 +1634,7 @@ public List> getLoadJobUnfinishedInfo(long jobId) { long versionHash = partitionLoadInfo.getVersionHash(); for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(version, versionHash)) { + if (replica.checkVersionCatchUp(version, versionHash, false)) { continue; } diff --git a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index aa94bee62931c3..e911311bb04268 100644 --- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -639,7 +639,7 @@ public void finishTransaction(long transactionId, Set errorReplicaIds) thr continue; } int quorumReplicaNum = partitionInfo.getReplicationNum(partitionId) / 2 + 1; - MaterializedIndex baseIndex = partition.getBaseIndex(); + List allInices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : allInices) { for (Tablet tablet : index.getTablets()) { @@ -651,7 +651,7 @@ public void finishTransaction(long transactionId, Set errorReplicaIds) thr // it is healthy in the past and does not have error in current load if (replica.checkVersionCatchUp(partition.getVisibleVersion(), - partition.getVisibleVersionHash())) { + partition.getVisibleVersionHash(), true)) { // during rollup, the rollup replica's last failed version < 0, // it may be treated as a normal replica. // the replica is not failed during commit or publish @@ -1038,7 +1038,7 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat newVersion = replica.getVersion(); newVersionHash = replica.getVersionHash(); } else if (!replica.checkVersionCatchUp(partition.getVisibleVersion(), - partition.getVisibleVersionHash())) { + partition.getVisibleVersionHash(), false)) { // this means the replica has error in the past, but we did not observe it // during upgrade, one job maybe in quorum finished state, for example, A,B,C 3 replica // A,B 's version is 10, C's version is 10 but C' 10 is abnormal should be rollback diff --git a/fe/src/test/java/org/apache/doris/catalog/ReplicaTest.java b/fe/src/test/java/org/apache/doris/catalog/ReplicaTest.java index 4f0e210d6c00ef..2b396c8a90fcad 100644 --- a/fe/src/test/java/org/apache/doris/catalog/ReplicaTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/ReplicaTest.java @@ -85,9 +85,9 @@ public void getMethodTest() { Assert.assertEquals(newRowCount, replica.getRowCount()); // check version catch up - Assert.assertFalse(replica.checkVersionCatchUp(5, 98765)); - Assert.assertFalse(replica.checkVersionCatchUp(newVersion, 76543)); - Assert.assertTrue(replica.checkVersionCatchUp(newVersion, newVersionHash)); + Assert.assertFalse(replica.checkVersionCatchUp(5, 98765, false)); + Assert.assertFalse(replica.checkVersionCatchUp(newVersion, 76543, false)); + Assert.assertTrue(replica.checkVersionCatchUp(newVersion, newVersionHash, false)); } @Test From 89f326dec0fa88877613420a8a8ade209960b4e3 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 20:04:09 +0800 Subject: [PATCH 16/79] fix cancel job bug --- fe/src/main/java/org/apache/doris/alter/AlterHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index b94ff24acc8d06..37f8bcf21bdd8d 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -91,7 +91,7 @@ protected void addAlterJobV2(AlterJobV2 alterJob) { public AlterJobV2 getAlterJobV2(long tblId) { for (AlterJobV2 alterJob : alterJobsV2.values()) { - if (alterJob.getTableId() == tblId) { + if (alterJob.getTableId() == tblId && alterJob.getJobState() == AlterJobV2.JobState.RUNNING) { return alterJob; } } From 4486dfdb0ca61713ea1cb4872bc67c3651412501 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 21:43:23 +0800 Subject: [PATCH 17/79] fix schema change bug 1 --- fe/src/main/java/org/apache/doris/alter/AlterJobV2.java | 2 ++ .../java/org/apache/doris/alter/SchemaChangeJobV2.java | 5 +++-- fe/src/main/java/org/apache/doris/catalog/OlapTable.java | 8 ++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index ddb635eb33d9a2..83aac9026eab3c 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -177,6 +177,8 @@ public static AlterJobV2 read(DataInput in) throws IOException { switch (type) { case ROLLUP: return RollupJobV2.read(in); + case SCHEMA_CHANGE: + return SchemaChangeJobV2.read(in); default: Preconditions.checkState(false); return null; diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 8e7b70d1f7f7ab..e2a237a3e35701 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -209,7 +209,7 @@ protected void runPendingJob() { Partition.PARTITION_INIT_VERSION, Partition.PARTITION_INIT_VERSION_HASH, tbl.getKeysType(), TStorageType.COLUMN, storageMedium, shadowSchema, bfColumns, bfFpp, countDownLatch); - createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdx).get(shadowTabletId), originSchemaHash); + createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId).get(shadowTabletId), originSchemaHash); batchTask.addTask(createReplicaTask); } // end for rollupReplicas @@ -471,7 +471,7 @@ private void onFinished(OlapTable tbl) { } } - partition.visualiseShadowIndex(shadowIdx.getId()); + partition.visualiseShadowIndex(shadowIdxId); } } @@ -488,6 +488,7 @@ private void onFinished(OlapTable tbl) { if (originIdxId == tbl.getBaseIndexId()) { // set base index tbl.setNewBaseSchema(indexSchemaMap.get(shadowIdxId)); + tbl.setBaseIndexId(shadowIdxId); } } diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 5f0c0f85e62559..15e1ee18ba1da1 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -232,14 +232,18 @@ public void setIndexStorageType(Long indexId, TStorageType newStorageType) { indexIdToStorageType.put(indexId, newStorageType); } - public void deleteIndexInfo(String indexName) { - long indexId = this.indexNameToId.remove(indexName); + public boolean deleteIndexInfo(String indexName) { + if (!indexNameToId.containsKey(indexName)) { + return false; + } + long indexId = this.indexNameToId.remove(indexName); indexIdToSchema.remove(indexId); indexIdToSchemaVersion.remove(indexId); indexIdToSchemaHash.remove(indexId); indexIdToShortKeyColumnCount.remove(indexId); indexIdToStorageType.remove(indexId); + return true; } public Map getIndexNameToId() { From 880c0e2cabdb4347177b96a97db751ab7a89677b Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 25 Jul 2019 22:13:14 +0800 Subject: [PATCH 18/79] fix schema change base index visualize bug --- .../org/apache/doris/alter/RollupJobV2.java | 4 +-- .../apache/doris/alter/SchemaChangeJobV2.java | 32 ++++++++----------- .../org/apache/doris/catalog/Partition.java | 15 ++++----- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 62352ebccf3b81..0641b9e7dfc04e 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -421,7 +421,7 @@ private void onFinished(OlapTable tbl) { replica.setState(ReplicaState.NORMAL); } } - partition.visualiseShadowIndex(rollupIndexId); + partition.visualiseShadowIndex(rollupIndexId, false); } tbl.setState(OlapTableState.NORMAL); } @@ -448,7 +448,7 @@ public synchronized boolean cancel(String errMsg) { private void cancelInternal() { // clear tasks if has - AgentTaskQueue.removeBatchTask(rollupBatchTask, TTaskType.ROLLUP); + AgentTaskQueue.removeBatchTask(rollupBatchTask, TTaskType.ALTER); // remove all rollup indexes, and set state to NORMAL TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); Database db = Catalog.getCurrentCatalog().getDb(dbId); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index e2a237a3e35701..e8fc6a601dd248 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -424,7 +424,7 @@ protected void runRunningJob() { } if (healthyReplicaNum < expectReplicationNum / 2 + 1) { - LOG.warn("rollup tablet {} has few healthy replicas: {}, rollup job: {}", + LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}", shadowTablet.getId(), replicas, jobId); cancel("shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); return; @@ -443,7 +443,7 @@ protected void runRunningJob() { this.finishedTimeMs = System.currentTimeMillis(); Catalog.getCurrentCatalog().getEditLog().logAlterJob(this); - LOG.info("rollup job finished: {}", jobId); + LOG.info("schema change job finished: {}", jobId); } private void onFinished(OlapTable tbl) { @@ -459,10 +459,6 @@ private void onFinished(OlapTable tbl) { // in catalog. MaterializedIndex shadowIdx = partition.getIndex(shadowIdxId); Preconditions.checkNotNull(shadowIdx, shadowIdxId); - // base index need special handling - if (originIdxId == partition.getBaseIndex().getId()) { - partition.setBaseIndex(shadowIdx); - } partition.deleteRollupIndex(originIdxId); // set replica state for (Tablet tablet : shadowIdx.getTablets()) { @@ -471,7 +467,7 @@ private void onFinished(OlapTable tbl) { } } - partition.visualiseShadowIndex(shadowIdxId); + partition.visualiseShadowIndex(shadowIdxId, originIdxId == partition.getBaseIndex().getId()); } } @@ -522,8 +518,8 @@ public synchronized boolean cancel(String errMsg) { private void cancelInternal() { // clear tasks if has - AgentTaskQueue.removeBatchTask(schemaChangeBatchTask, TTaskType.ROLLUP); - // remove all rollup indexes, and set state to NORMAL + AgentTaskQueue.removeBatchTask(schemaChangeBatchTask, TTaskType.ALTER); + // remove all shadow indexes, and set state to NORMAL TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db != null) { @@ -561,9 +557,9 @@ protected boolean isPreviousLoadFinished() { } public static SchemaChangeJobV2 read(DataInput in) throws IOException { - SchemaChangeJobV2 rollupJob = new SchemaChangeJobV2(); - rollupJob.readFields(in); - return rollupJob; + SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(); + schemaChangeJob.readFields(in); + return schemaChangeJob; } /* @@ -654,7 +650,7 @@ private void replayFinished() { db.writeUnlock(); } } - LOG.info("replay finished rollup job: {}", jobId); + LOG.info("replay finished schema change job: {}", jobId); } /* @@ -662,7 +658,7 @@ private void replayFinished() { */ private void replayCancelled() { cancelInternal(); - LOG.info("replay cancelled rollup job: {}", jobId); + LOG.info("replay cancelled schema change job: {}", jobId); } @Override @@ -720,11 +716,11 @@ public List> getUnfinishedTasks(int limit) { if (jobState == JobState.RUNNING) { List tasks = schemaChangeBatchTask.getUnfinishedTasks(limit); for (AgentTask agentTask : tasks) { - AlterReplicaTask rollupTask = (AlterReplicaTask)agentTask; + AlterReplicaTask alterTask = (AlterReplicaTask) agentTask; List info = Lists.newArrayList(); - info.add(String.valueOf(rollupTask.getBackendId())); - info.add(String.valueOf(rollupTask.getBaseTabletId())); - info.add(String.valueOf(rollupTask.getSignature())); + info.add(String.valueOf(alterTask.getBackendId())); + info.add(String.valueOf(alterTask.getBaseTabletId())); + info.add(String.valueOf(alterTask.getSignature())); taskInfos.add(info); } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 10e49d59d7ca47..8ad13fc702e124 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -200,11 +200,6 @@ public MaterializedIndex getBaseIndex() { return baseIndex; } - // this is now only for schema change job - public void setBaseIndex(MaterializedIndex baseIndex) { - this.baseIndex = baseIndex; - } - public long getNextVersion() { return nextVersion; } @@ -275,16 +270,20 @@ public boolean hasData() { /* * Change the index' state from SHADOW to NORMAL - * Also move it to idToVisibleRollupIndex + * Also move it to idToVisibleRollupIndex if it is not the base index. */ - public boolean visualiseShadowIndex(long shadowIndexId) { + public boolean visualiseShadowIndex(long shadowIndexId, boolean isBaseIndex) { MaterializedIndex shadowIdx = idToShadowIndex.remove(shadowIndexId); if (shadowIdx == null) { return false; } Preconditions.checkState(!idToVisibleRollupIndex.containsKey(shadowIndexId), shadowIndexId); shadowIdx.setState(IndexState.NORMAL); - idToVisibleRollupIndex.put(shadowIndexId, shadowIdx); + if (isBaseIndex) { + baseIndex = shadowIdx; + } else { + idToVisibleRollupIndex.put(shadowIndexId, shadowIdx); + } LOG.info("visualise the shadow index: {}", shadowIndexId); return true; } From 7ad3547f865c9151687271a227bb1ff193746e2d Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 26 Jul 2019 09:08:12 +0800 Subject: [PATCH 19/79] change ignore to equalsIgnoreCase --- .../doris/alter/SchemaChangeHandler.java | 36 +++++++++---------- .../main/java/org/apache/doris/load/Load.java | 6 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 0bf96311ad52a1..c118a1cf598db5 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -155,7 +155,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable List baseSchema = indexSchemaMap.get(baseIndexId); boolean isKey = false; for (Column column : baseSchema) { - if (column.isKey() && column.getName().equals(dropColName)) { + if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { isKey = true; break; } @@ -173,7 +173,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable boolean isKey = false; boolean hasReplaceColumn = false; for (Column column : baseSchema) { - if (column.isKey() && column.getName().equals(dropColName)) { + if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { isKey = true; } else if (AggregateType.REPLACE == column.getAggregationType()) { hasReplaceColumn = true; @@ -190,7 +190,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable boolean isKey = false; boolean hasReplaceColumn = false; for (Column column : targetIndexSchema) { - if (column.isKey() && column.getName().equals(dropColName)) { + if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { isKey = true; } else if (AggregateType.REPLACE == column.getAggregationType()) { hasReplaceColumn = true; @@ -220,7 +220,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable Iterator baseIter = baseSchema.iterator(); while (baseIter.hasNext()) { Column column = baseIter.next(); - if (column.getName().equals(dropColName)) { + if (column.getName().equalsIgnoreCase(dropColName)) { baseIter.remove(); found = true; break; @@ -236,7 +236,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable Iterator iter = rollupSchema.iterator(); while (iter.hasNext()) { Column column = iter.next(); - if (column.getName().equals(dropColName)) { + if (column.getName().equalsIgnoreCase(dropColName)) { iter.remove(); break; } @@ -251,7 +251,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable Iterator iter = targetIndexSchema.iterator(); while (iter.hasNext()) { Column column = iter.next(); - if (column.getName().equals(dropColName)) { + if (column.getName().equalsIgnoreCase(dropColName)) { iter.remove(); found = true; break; @@ -317,12 +317,12 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT int lastColIndex = -1; for (int i = 0; i < schemaForFinding.size(); i++) { Column col = schemaForFinding.get(i); - if (col.getName().equals(newColName)) { + if (col.getName().equalsIgnoreCase(newColName)) { modColIndex = i; found = true; } if (hasColPos) { - if (col.getName().equals(columnPos.getLastCol())) { + if (col.getName().equalsIgnoreCase(columnPos.getLastCol())) { lastColIndex = i; } } else { @@ -382,7 +382,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT } List schema = entry.getValue(); for (Column column : schema) { - if (column.getName().equals(modColumn.getName())) { + if (column.getName().equalsIgnoreCase(modColumn.getName())) { otherIndexIds.add(entry.getKey()); break; } @@ -394,7 +394,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT List otherIndexSchema = indexSchemaMap.get(otherIndexId); modColIndex = -1; for (int i = 0; i < otherIndexSchema.size(); i++) { - if (otherIndexSchema.get(i).getName().equals(modColumn.getName())) { + if (otherIndexSchema.get(i).getName().equalsIgnoreCase(modColumn.getName())) { modColIndex = i; break; } @@ -409,7 +409,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT List otherIndexSchema = indexSchemaMap.get(otherIndexId); modColIndex = -1; for (int i = 0; i < otherIndexSchema.size(); i++) { - if (otherIndexSchema.get(i).getName().equals(modColumn.getName())) { + if (otherIndexSchema.get(i).getName().equalsIgnoreCase(modColumn.getName())) { modColIndex = i; break; } @@ -454,7 +454,7 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol for (String colName : orderedColNames) { Column oneCol = null; for (Column column : targetIndexSchema) { - if (column.getName().equals(colName)) { + if (column.getName().equalsIgnoreCase(colName)) { oneCol = column; break; } @@ -524,7 +524,7 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi List baseSchema = olapTable.getBaseSchema(); boolean found = false; for (Column column : baseSchema) { - if (column.getName().equals(newColName)) { + if (column.getName().equalsIgnoreCase(newColName)) { found = true; break; } @@ -624,7 +624,7 @@ private void checkAndAddColumn(List modIndexSchema, Column newColumn, Co boolean hasPos = (columnPos != null && !columnPos.isFirst()); for (int i = 0; i < modIndexSchema.size(); i++) { Column col = modIndexSchema.get(i); - if (col.getName().equals(newColName)) { + if (col.getName().equalsIgnoreCase(newColName)) { if (!isBaseIndex || !newColNameSet.contains(newColName)) { // if this is not a base index, we should check if user repeatedly add columns throw new DdlException("Repeatedly add column: " + newColName); @@ -642,7 +642,7 @@ private void checkAndAddColumn(List modIndexSchema, Column newColumn, Co if (hasPos) { // after the field - if (col.getName().equals(columnPos.getLastCol())) { + if (col.getName().equalsIgnoreCase(columnPos.getLastCol())) { posIndex = i; } } else { @@ -906,7 +906,7 @@ private void createJob(long dbId, OlapTable olapTable, Map Map> indexIdToSchema = table.getIndexIdToSchema(); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { // check table has condition column - Map indexNameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + Map indexColNameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); for (Column column : indexIdToSchema.get(index.getId())) { - indexNameToColumn.put(column.getName(), column); + indexColNameToColumn.put(column.getName(), column); } String indexName = table.getIndexNameById(index.getId()); for (Predicate condition : conditions) { @@ -2761,7 +2761,7 @@ private void checkDeleteV2(OlapTable table, Partition partition, List IsNullPredicate isNullPredicate = (IsNullPredicate) condition; columnName = ((SlotRef) isNullPredicate.getChild(0)).getColumnName(); } - Column column = indexNameToColumn.get(columnName); + Column column = indexColNameToColumn.get(columnName); if (column == null) { ErrorReport.reportDdlException(ErrorCode.ERR_BAD_FIELD_ERROR, columnName, indexName); } From a5cad3641861f818517668d811fe1e15324ecb86 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 29 Jul 2019 10:51:39 +0800 Subject: [PATCH 20/79] schema change try 1 --- .../org/apache/doris/alter/AlterHandler.java | 2 +- .../doris/alter/SchemaChangeHandler.java | 25 ++++++++++++++++--- .../apache/doris/alter/SchemaChangeJobV2.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 9 +++++++ .../java/org/apache/doris/catalog/Table.java | 8 +++--- .../apache/doris/planner/DataSplitSink.java | 2 +- .../apache/doris/planner/OlapTableSink.java | 5 ---- .../doris/planner/OlapTableSinkTest.java | 2 +- 8 files changed, 38 insertions(+), 17 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 37f8bcf21bdd8d..7715dcdb37a97d 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -396,8 +396,8 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce // Case 2.1 replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); } else { - Preconditions.checkState(replica.getLastFailedVersion() == -1, replica.getLastFailedVersion()); // Case 1 + Preconditions.checkState(replica.getLastFailedVersion() == -1, replica.getLastFailedVersion()); replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); } } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index c118a1cf598db5..08f38cefd716bc 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -82,7 +82,7 @@ public class SchemaChangeHandler extends AlterHandler { private static final Logger LOG = LogManager.getLogger(SchemaChangeHandler.class); // all shadow indexes should have this prefix in name - public static final String SHADOW_INDEX_NAME_PRFIX = "__doris_shadow_"; + public static final String SHADOW_NAME_PRFIX = "__doris_shadow_"; public SchemaChangeHandler() { super("schema change"); @@ -282,7 +282,6 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT modColumn.setAggregationType(AggregateType.REPLACE, true); } } else { - // DUPLICATE table also has key and value column. if (null != modColumn.getAggregationType()) { throw new DdlException("Can not assign aggregation method on column in Duplicate data model table: " + modColumn.getName()); } @@ -313,6 +312,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT String newColName = modColumn.getName(); boolean hasColPos = (columnPos != null && !columnPos.isFirst()); boolean found = false; + boolean typeChanged = false; int modColIndex = -1; int lastColIndex = -1; for (int i = 0; i < schemaForFinding.size(); i++) { @@ -320,6 +320,9 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT if (col.getName().equalsIgnoreCase(newColName)) { modColIndex = i; found = true; + if (!col.equals(modColumn)) { + typeChanged = true; + } } if (hasColPos) { if (col.getName().equalsIgnoreCase(columnPos.getLastCol())) { @@ -429,6 +432,20 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT } } } // end for handling other indices + + if (typeChanged) { + /* + * In the new alter table process (AlterJobV2), any modified column is treated as a new column. + * But the modified column's name does not changed. So in order to distinguish this, we will add + * a prefix in the name of the modified column. + * This prefix only exist during the schema change process. Once the schema change is finished, + * it will be removed. + * + * And if the column type not changed, the same column name is still to the same column type, + * so no need to add prefix. + */ + modColumn.setName(SHADOW_NAME_PRFIX + modColumn.getName()); + } } private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable olapTable, @@ -521,7 +538,7 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi // check if the new column already exist in base schema. // do not support adding new column which already exist in base schema. - List baseSchema = olapTable.getBaseSchema(); + List baseSchema = olapTable.getBaseSchema(false); boolean found = false; for (Column column : baseSchema) { if (column.getName().equalsIgnoreCase(newColName)) { @@ -1007,7 +1024,7 @@ private void createJob(long dbId, OlapTable olapTable, Map> infos) { info.add(TimeUtils.longToTimeString(createTimeMs)); info.add(TimeUtils.longToTimeString(finishedTimeMs)); // only show the origin index name - info.add(indexIdToName.get(shadowIndexId).substring(SchemaChangeHandler.SHADOW_INDEX_NAME_PRFIX.length())); + info.add(indexIdToName.get(shadowIndexId).substring(SchemaChangeHandler.SHADOW_NAME_PRFIX.length())); info.add(shadowIndexId); info.add(entry.getValue()); info.add(indexSchemaVersionAndHashMap.get(shadowIndexId).toString()); diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 15e1ee18ba1da1..f19d53cf026c0b 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -226,6 +226,10 @@ public void setIndexSchemaInfo(Long indexId, String indexName, List sche indexIdToSchemaVersion.put(indexId, schemaVersion); indexIdToSchemaHash.put(indexId, schemaHash); indexIdToShortKeyColumnCount.put(indexId, shortKeyColumnCount); + + if (schema.size() > baseSchemaForLoad.size()) { + + } } public void setIndexStorageType(Long indexId, TStorageType newStorageType) { Preconditions.checkState(newStorageType == TStorageType.COLUMN); @@ -1079,4 +1083,9 @@ public long proximateRowCount() { } return totalCount; } + + @Override + public List getBaseSchema() { + return indexIdToSchema.get(baseIndexId); + } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Table.java b/fe/src/main/java/org/apache/doris/catalog/Table.java index aedad489ff059d..af42aeb8e61bda 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/src/main/java/org/apache/doris/catalog/Table.java @@ -24,6 +24,7 @@ import org.apache.doris.thrift.TTableDescriptor; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang.NotImplementedException; @@ -33,7 +34,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -58,7 +58,8 @@ public enum TableType { protected String name; protected TableType type; protected List baseSchema; - // tree map for case-insensitive lookup + // tree map for case-insensitive lookup. + // This should contains all columns in both baseSchema and baseSchemaForLoad protected Map nameToColumn; // DO NOT persist this variable. @@ -66,7 +67,7 @@ public enum TableType { public Table(TableType type) { this.type = type; - this.baseSchema = new LinkedList(); + this.baseSchema = Lists.newArrayList(); this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); } @@ -75,7 +76,6 @@ public Table(long id, String tableName, TableType type, List baseSchema) this.name = tableName; this.type = type; this.baseSchema = baseSchema; - this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); if (baseSchema != null) { for (Column col : baseSchema) { diff --git a/fe/src/main/java/org/apache/doris/planner/DataSplitSink.java b/fe/src/main/java/org/apache/doris/planner/DataSplitSink.java index 6bdfb2e81d648f..05e7fb74dd9dc5 100644 --- a/fe/src/main/java/org/apache/doris/planner/DataSplitSink.java +++ b/fe/src/main/java/org/apache/doris/planner/DataSplitSink.java @@ -62,7 +62,7 @@ // This class used to split data read from file to batch @Deprecated public class DataSplitSink extends DataSink { - private static final Logger LOG = LogManager.getLogger(Planner.class); + private static final Logger LOG = LogManager.getLogger(DataSplitSink.class); private final OlapTable targetTable; diff --git a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java index 79ba622f860391..a9ce89734797ba 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -82,11 +82,6 @@ public class OlapTableSink extends DataSink { // set after init called private TDataSink tDataSink; - public OlapTableSink(OlapTable dstTable, TupleDescriptor tupleDescriptor) { - this.dstTable = dstTable; - this.tupleDescriptor = tupleDescriptor; - } - public OlapTableSink(OlapTable dstTable, TupleDescriptor tupleDescriptor, String partitions) { this.dstTable = dstTable; this.tupleDescriptor = tupleDescriptor; diff --git a/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java b/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java index 891c78de6c2490..9aad7920cf1ad0 100644 --- a/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java +++ b/fe/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java @@ -98,7 +98,7 @@ public void testSinglePartition() throws UserException { dstTable.getPartitions(); result = Lists.newArrayList(partition); }}; - OlapTableSink sink = new OlapTableSink(dstTable, tuple); + OlapTableSink sink = new OlapTableSink(dstTable, tuple, ""); sink.init(new TUniqueId(1, 2), 3, 4); sink.finalize(); LOG.info("sink is {}", sink.toThrift()); From bad03afb832e6eca3cd13ca2ff09e8ae3209fe19 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 29 Jul 2019 19:41:18 +0800 Subject: [PATCH 21/79] add --- .../apache/doris/analysis/DataDescription.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index cb6c9d1dff0423..2dfc574d73369b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -35,6 +35,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -333,6 +334,8 @@ public static void validateMappingFunction(String functionName, List arg validateHllHash(args, columnNameMap); } else if (functionName.equalsIgnoreCase("now")) { validateNowFunction(mappingColumn); + } else if (functionName.equalsIgnoreCase("replace")) { + validateReplaceFunction(args, columnNameMap); } else { if (isHadoopLoad) { throw new AnalysisException("Unknown function: " + functionName); @@ -340,6 +343,20 @@ public static void validateMappingFunction(String functionName, List arg } } + private static void validateReplaceFunction(List args, Map columnNameMap) + throws AnalysisException { + if (args.size() != 1) { + throw new AnalysisException("Should has only one argument: " + args); + } + + String argColumn = args.get(0); + if (!columnNameMap.containsKey(argColumn)) { + throw new AnalysisException("Column is not in sources, column: " + argColumn); + } + + args.set(0, columnNameMap.get(argColumn)); + } + private static void validateAlignmentTimestamp(List args, Map columnNameMap) throws AnalysisException { if (args.size() != 2) { From 304a7dd132814dbcef96825818fa5412a4ecc9ae Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 30 Jul 2019 15:51:18 +0800 Subject: [PATCH 22/79] handle __doris_shadow column --- .../operation/tablet-meta-tool.md | 2 +- .../doris/alter/SchemaChangeHandler.java | 20 +-- .../apache/doris/alter/SchemaChangeJob.java | 4 +- .../apache/doris/alter/SchemaChangeJobV2.java | 9 +- .../doris/analysis/DataDescription.java | 8 +- .../org/apache/doris/catalog/BrokerTable.java | 2 +- .../java/org/apache/doris/catalog/Column.java | 36 ++++++ .../org/apache/doris/catalog/EsTable.java | 2 +- .../org/apache/doris/catalog/MysqlTable.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 42 +++++- .../java/org/apache/doris/catalog/Table.java | 47 +++---- .../org/apache/doris/common/FeNameFormat.java | 4 + .../main/java/org/apache/doris/load/Load.java | 43 +++++-- .../doris/load/loadv2/LoadingTaskPlanner.java | 3 +- .../doris/planner/StreamLoadPlanner.java | 3 +- .../doris/planner/StreamLoadScanNode.java | 120 ++++++++++-------- .../apache/doris/task/CreateReplicaTask.java | 6 + .../doris/task/HadoopLoadPendingTask.java | 2 +- .../doris/task/PullLoadTaskPlanner.java | 2 +- .../org/apache/doris/task/StreamLoadTask.java | 18 ++- 20 files changed, 257 insertions(+), 118 deletions(-) diff --git a/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md b/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md index 87a6230b5f7603..5846d3b40ba02b 100644 --- a/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md +++ b/docs/documentation/cn/administrator-guide/operation/tablet-meta-tool.md @@ -72,7 +72,7 @@ api： 命令: ``` -./lib/meta_tool --operation=delete_header --root_path=/path/to/root_path --tablet_id=xxx --schema_hash=xxx` +./lib/meta_tool --operation=delete_header --root_path=/path/to/root_path --tablet_id=xxx --schema_hash=xxx ``` ### 展示 pb 格式的 TabletMeta diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 08f38cefd716bc..de12c9c0efa58f 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -435,13 +435,15 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT if (typeChanged) { /* - * In the new alter table process (AlterJobV2), any modified column is treated as a new column. - * But the modified column's name does not changed. So in order to distinguish this, we will add - * a prefix in the name of the modified column. + * In new alter table process (AlterJobV2), any modified columns are treated as new columns. + * But the modified columns' name does not changed. So in order to distinguish this, we will add + * a prefix in the name of these modified columns. * This prefix only exist during the schema change process. Once the schema change is finished, * it will be removed. * - * And if the column type not changed, the same column name is still to the same column type, + * After adding this prefix, modify a column is just same as 'add' a column. + * + * And if the column type is not changed, the same column name is still to the same column type, * so no need to add prefix. */ modColumn.setName(SHADOW_NAME_PRFIX + modColumn.getName()); @@ -538,7 +540,7 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi // check if the new column already exist in base schema. // do not support adding new column which already exist in base schema. - List baseSchema = olapTable.getBaseSchema(false); + List baseSchema = olapTable.getBaseSchema(); boolean found = false; for (Column column : baseSchema) { if (column.getName().equalsIgnoreCase(newColName)) { @@ -923,7 +925,7 @@ private void createJob(long dbId, OlapTable olapTable, Map entry : indexIdMap.entrySet()) { long shadowIdxId = entry.getKey(); long originIdxId = entry.getValue(); @@ -479,14 +481,17 @@ private void onFinished(OlapTable tbl) { String originIdxName = tbl.getIndexNameById(originIdxId); tbl.deleteIndexInfo(originIdxName); // the shadow index name is '__doris_shadow_xxx', rename it to origin name 'xxx' + // this will also remove the prefix of columns tbl.renameIndexForSchemaChange(shadowIdxName, originIdxName); + tbl.renameColumnNamePrefix(shadowIdxId); if (originIdxId == tbl.getBaseIndexId()) { // set base index - tbl.setNewBaseSchema(indexSchemaMap.get(shadowIdxId)); tbl.setBaseIndexId(shadowIdxId); } } + // rebuild table's full schema + tbl.rebuildFullSchema(); // update bloom filter if (hasBfChange) { diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index 2dfc574d73369b..9bff9c8e9f8003 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -334,8 +334,8 @@ public static void validateMappingFunction(String functionName, List arg validateHllHash(args, columnNameMap); } else if (functionName.equalsIgnoreCase("now")) { validateNowFunction(mappingColumn); - } else if (functionName.equalsIgnoreCase("replace")) { - validateReplaceFunction(args, columnNameMap); + } else if (functionName.equalsIgnoreCase("substitute")) { + validateSubstituteFunction(args, columnNameMap); } else { if (isHadoopLoad) { throw new AnalysisException("Unknown function: " + functionName); @@ -343,7 +343,9 @@ public static void validateMappingFunction(String functionName, List arg } } - private static void validateReplaceFunction(List args, Map columnNameMap) + // eg: k2 = substitute(k1) + // this is used for creating derivative column from existing column + private static void validateSubstituteFunction(List args, Map columnNameMap) throws AnalysisException { if (args.size() != 1) { throw new AnalysisException("Should has only one argument: " + args); diff --git a/fe/src/main/java/org/apache/doris/catalog/BrokerTable.java b/fe/src/main/java/org/apache/doris/catalog/BrokerTable.java index d3251ee81b5771..0e4d8852a5b8fc 100644 --- a/fe/src/main/java/org/apache/doris/catalog/BrokerTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/BrokerTable.java @@ -202,7 +202,7 @@ private void validate(Map properties) throws DdlException { public TTableDescriptor toThrift() { TBrokerTable tBrokerTable = new TBrokerTable(); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.BROKER_TABLE, - baseSchema.size(), 0, getName(), ""); + fullSchema.size(), 0, getName(), ""); tTableDescriptor.setBrokerTable(tBrokerTable); return tTableDescriptor; } diff --git a/fe/src/main/java/org/apache/doris/catalog/Column.java b/fe/src/main/java/org/apache/doris/catalog/Column.java index ed46953f69725c..7fc66369ec5da8 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/src/main/java/org/apache/doris/catalog/Column.java @@ -17,6 +17,8 @@ package org.apache.doris.catalog; +import org.apache.doris.alter.SchemaChangeHandler; +import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; @@ -118,6 +120,17 @@ public String getName() { return this.name; } + public String getNameWithoutPrefix(String prefix) { + if (isNameWithPrefix(prefix)) { + return name.substring(prefix.length()); + } + return name; + } + + public boolean isNameWithPrefix(String prefix) { + return this.name.startsWith(prefix); + } + public void setIsKey(boolean isKey) { this.isKey = isKey; } @@ -252,6 +265,29 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException { } } + public boolean nameEquals(String otherColName, boolean ignorePrefix) { + if (CaseSensibility.COLUMN.getCaseSensibility()) { + if (!ignorePrefix) { + return name.equals(otherColName); + } else { + return removeNamePrefix(name).equals(removeNamePrefix(otherColName)); + } + } else { + if (!ignorePrefix) { + return name.equalsIgnoreCase(otherColName); + } else { + return removeNamePrefix(name).equalsIgnoreCase(removeNamePrefix(otherColName)); + } + } + } + + public static String removeNamePrefix(String colName) { + if (colName.startsWith(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + return colName.substring(SchemaChangeHandler.SHADOW_NAME_PRFIX.length()); + } + return colName; + } + public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("`").append(name).append("` "); diff --git a/fe/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/src/main/java/org/apache/doris/catalog/EsTable.java index 8713ac6ce1271b..bf5c420f804c55 100644 --- a/fe/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/EsTable.java @@ -121,7 +121,7 @@ private void validate(Map properties) throws DdlException { public TTableDescriptor toThrift() { TEsTable tEsTable = new TEsTable(); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ES_TABLE, - baseSchema.size(), 0, getName(), ""); + fullSchema.size(), 0, getName(), ""); tTableDescriptor.setEsTable(tEsTable); return tTableDescriptor; } diff --git a/fe/src/main/java/org/apache/doris/catalog/MysqlTable.java b/fe/src/main/java/org/apache/doris/catalog/MysqlTable.java index e60a8749697c19..a31872faf4c89f 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MysqlTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/MysqlTable.java @@ -144,7 +144,7 @@ public TTableDescriptor toThrift() { TMySQLTable tMySQLTable = new TMySQLTable(host, port, userName, passwd, mysqlDatabaseName, mysqlTableName); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.MYSQL_TABLE, - baseSchema.size(), 0, getName(), ""); + fullSchema.size(), 0, getName(), ""); tTableDescriptor.setMysqlTable(tMySQLTable); return tTableDescriptor; } diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index f19d53cf026c0b..bf2ab130a74bb0 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -215,6 +215,9 @@ public boolean hasMaterializedIndex(String indexName) { return indexNameToId.containsKey(indexName); } + /* + * Set index schema info for specified index. + */ public void setIndexSchemaInfo(Long indexId, String indexName, List schema, int schemaVersion, int schemaHash, short shortKeyColumnCount) { if (indexName == null) { @@ -226,16 +229,28 @@ public void setIndexSchemaInfo(Long indexId, String indexName, List sche indexIdToSchemaVersion.put(indexId, schemaVersion); indexIdToSchemaHash.put(indexId, schemaHash); indexIdToShortKeyColumnCount.put(indexId, shortKeyColumnCount); - - if (schema.size() > baseSchemaForLoad.size()) { - - } } + public void setIndexStorageType(Long indexId, TStorageType newStorageType) { Preconditions.checkState(newStorageType == TStorageType.COLUMN); indexIdToStorageType.put(indexId, newStorageType); } + // rebuild the full schema of table + // the order of columns in fullSchema is meaningless + public void rebuildFullSchema() { + fullSchema.clear(); + nameToColumn.clear(); + for (List columns : indexIdToSchema.values()) { + for (Column column : columns) { + if (!nameToColumn.containsKey(column.getName())) { + fullSchema.add(column); + nameToColumn.put(column.getName(), column); + } + } + } + } + public boolean deleteIndexInfo(String indexName) { if (!indexNameToId.containsKey(indexName)) { return false; @@ -273,6 +288,13 @@ public void renameIndexForSchemaChange(String name, String newName) { indexNameToId.put(newName, idxId); } + public void renameColumnNamePrefix(long idxId) { + List columns = indexIdToSchema.get(idxId); + for (Column column : columns) { + column.setName(Column.removeNamePrefix(column.getName())); + } + } + public Status resetIdsForRestore(Catalog catalog, Database db, int restoreReplicationNum) { // table id id = catalog.getNextId(); @@ -587,7 +609,7 @@ public boolean shouldLoadToNewRollup() { public TTableDescriptor toThrift() { TOlapTable tOlapTable = new TOlapTable(getName()); TTableDescriptor tTableDescriptor = new TTableDescriptor(id, TTableType.OLAP_TABLE, - baseSchema.size(), 0, getName(), ""); + fullSchema.size(), 0, getName(), ""); tTableDescriptor.setOlapTable(tOlapTable); return tTableDescriptor; } @@ -1088,4 +1110,14 @@ public long proximateRowCount() { public List getBaseSchema() { return indexIdToSchema.get(baseIndexId); } + + public int getKeysNum() { + int keysNum = 0; + for (Column column : getBaseSchema()) { + if (column.isKey()) { + keysNum += 1; + } + } + return keysNum; + } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Table.java b/fe/src/main/java/org/apache/doris/catalog/Table.java index af42aeb8e61bda..e7c8469f893eac 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/src/main/java/org/apache/doris/catalog/Table.java @@ -57,9 +57,17 @@ public enum TableType { protected long id; protected String name; protected TableType type; - protected List baseSchema; + /* + * fullSchema and nameToColumn should contains all columns, both visible and shadow. + * eg. for OlapTable, when doing schema change, there will be some shadow columns which are not visible + * to query but visible to load process. + * If you want to get all visible columns, you should call getBaseSchema() method, which is override in + * sub classes. + * + * NOTICE: the order of this fullSchema is meaningless to OlapTable + */ + protected List fullSchema; // tree map for case-insensitive lookup. - // This should contains all columns in both baseSchema and baseSchemaForLoad protected Map nameToColumn; // DO NOT persist this variable. @@ -67,18 +75,18 @@ public enum TableType { public Table(TableType type) { this.type = type; - this.baseSchema = Lists.newArrayList(); + this.fullSchema = Lists.newArrayList(); this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); } - public Table(long id, String tableName, TableType type, List baseSchema) { + public Table(long id, String tableName, TableType type, List fullSchema) { this.id = id; this.name = tableName; this.type = type; - this.baseSchema = baseSchema; + this.fullSchema = fullSchema; this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - if (baseSchema != null) { - for (Column col : baseSchema) { + if (fullSchema != null) { + for (Column col : fullSchema) { nameToColumn.put(col.getName(), col); } } else { @@ -107,24 +115,19 @@ public TableType getType() { return type; } - public int getKeysNum() { - int keysNum = 0; - for (Column column : baseSchema) { - if (column.isKey()) { - keysNum += 1; - } - } - return keysNum; + public List getFullSchema() { + return fullSchema; } + // should override in subclass if necessary public List getBaseSchema() { - return baseSchema; + return fullSchema; } - public void setNewBaseSchema(List newSchema) { - this.baseSchema = newSchema; + public void setNewFullSchema(List newSchema) { + this.fullSchema = newSchema; this.nameToColumn.clear(); - for (Column col : baseSchema) { + for (Column col : fullSchema) { nameToColumn.put(col.getName(), col); } } @@ -182,9 +185,9 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, name); // base schema - int columnCount = baseSchema.size(); + int columnCount = fullSchema.size(); out.writeInt(columnCount); - for (Column column : baseSchema) { + for (Column column : fullSchema) { column.write(out); } } @@ -205,7 +208,7 @@ public void readFields(DataInput in) throws IOException { int columnCount = in.readInt(); for (int i = 0; i < columnCount; i++) { Column column = Column.read(in); - this.baseSchema.add(column); + this.fullSchema.add(column); this.nameToColumn.put(column.getName(), column); } } diff --git a/fe/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/src/main/java/org/apache/doris/common/FeNameFormat.java index 722d825a6a230b..f050e1ad2c64fb 100644 --- a/fe/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -17,6 +17,7 @@ package org.apache.doris.common; +import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.mysql.privilege.PaloRole; import org.apache.doris.system.SystemInfoService; @@ -63,6 +64,9 @@ public static void checkColumnName(String columnName) throws AnalysisException { if (Strings.isNullOrEmpty(columnName) || !columnName.matches(COMMON_NAME_REGEX)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName); } + if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName); + } } public static void checkLabel(String label) throws AnalysisException { diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 159046b7135b6b..8ba271eb7808b1 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -17,6 +17,7 @@ package org.apache.doris.load; +import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CancelLoadStmt; import org.apache.doris.analysis.ColumnSeparator; @@ -642,11 +643,9 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip } // get table schema - List tableSchema = table.getBaseSchema(); - Map nameToTableColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - for (Column column : tableSchema) { - nameToTableColumn.put(column.getName(), column); - } + List baseSchema = table.getBaseSchema(); + + // source columns List columnNames = Lists.newArrayList(); List assignColumnNames = Lists.newArrayList(); if (dataDescription.getColumnNames() != null) { @@ -657,14 +656,14 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip } if (assignColumnNames.isEmpty()) { // use table columns - for (Column column : tableSchema) { + for (Column column : baseSchema) { columnNames.add(column.getName()); } } else { // convert column to schema format for (String assignCol : assignColumnNames) { - if (nameToTableColumn.containsKey(assignCol)) { - columnNames.add(nameToTableColumn.get(assignCol).getName()); + if (table.getColumn(assignCol) != null) { + columnNames.add(table.getColumn(assignCol).getName()); } else { columnNames.add(assignCol); } @@ -698,7 +697,7 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // check negative for sum aggregate type if (dataDescription.isNegative()) { - for (Column column : tableSchema) { + for (Column column : baseSchema) { if (!column.isKey() && column.getAggregationType() != AggregateType.SUM) { throw new DdlException("Column is not SUM AggreateType. column:" + column.getName()); } @@ -706,16 +705,36 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip } // check hll - for (Column column : tableSchema) { + for (Column column : baseSchema) { if (column.getDataType() == PrimitiveType.HLL) { if (columnToHadoopFunction != null && !columnToHadoopFunction.containsKey(column.getName())) { throw new DdlException("Hll column is not assigned. column:" + column.getName()); } } } + // check mapping column exist in table // check function // convert mapping column and func arg columns to schema format + + // When doing schema change, there may have some 'shadow' columns, with prefix '__doris_shadow_' in + // their names. These columns are visible to user, but we need to generate data for these columns. + // So we add column mappings for these column. + // eg: + // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' + // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = B); + List fullSchema = table.getFullSchema(); + for (Column column : fullSchema) { + if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + if (assignColumnToFunction == null) { + assignColumnToFunction = Maps.newHashMap(); + dataDescription.setColumnToFunction(assignColumnToFunction); + } + assignColumnToFunction.put(column.getName(), Pair.create("substitute", + Lists.newArrayList(column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)))); + } + } + Map columnNameMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); for (String columnName : columnNames) { columnNameMap.put(columnName, columnName); @@ -724,11 +743,11 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip columnToFunction = Maps.newHashMap(); for (Entry>> entry : columnToHadoopFunction.entrySet()) { String mappingColumnName = entry.getKey(); - if (!nameToTableColumn.containsKey(mappingColumnName)) { + if (table.getColumn(mappingColumnName) == null) { throw new DdlException("Mapping column is not in table. column: " + mappingColumnName); } - Column mappingColumn = nameToTableColumn.get(mappingColumnName); + Column mappingColumn = table.getColumn(mappingColumnName); Pair> function = entry.getValue(); try { DataDescription.validateMappingFunction(function.first, function.second, columnNameMap, diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index e45d502dc24327..d1f8990633fbc3 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -95,7 +95,8 @@ public void plan(TUniqueId loadId, List> fileStatusesLis // Generate tuple descriptor List slotRefs = Lists.newArrayList(); TupleDescriptor tupleDesc = descTable.createTupleDescriptor(); - for (Column col : table.getBaseSchema()) { + // use full schema to fill the descriptor table + for (Column col : table.getFullSchema()) { SlotDescriptor slotDesc = descTable.addSlotDescriptor(tupleDesc); slotDesc.setIsMaterialized(true); slotDesc.setColumn(col); diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index bb6b28de23f543..e38d660d62714d 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -85,7 +85,8 @@ public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { // construct tuple descriptor, used for scanNode and dataSink TupleDescriptor tupleDesc = descTable.createTupleDescriptor("DstTableTuple"); boolean negative = streamLoadTask.getNegative(); - for (Column col : destTable.getBaseSchema()) { + // here we should be full schema to fill the descriptor table + for (Column col : destTable.getFullSchema()) { SlotDescriptor slotDesc = descTable.addSlotDescriptor(tupleDesc); slotDesc.setIsMaterialized(true); slotDesc.setColumn(col); diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index b012d083e520b9..06d5f47cda42ea 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -17,6 +17,7 @@ package org.apache.doris.planner; +import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.Expr; @@ -49,6 +50,7 @@ import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TUniqueId; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -124,67 +126,83 @@ public void init(Analyzer analyzer) throws UserException { // columns: k1, k2, v1, v2=k1 + k2 // this means that there are three columns(k1, k2, v1) in source file, // and v2 is derived from (k1 + k2) - if (streamLoadTask.getColumnExprDesc() != null && !streamLoadTask.getColumnExprDesc().isEmpty()) { - for (ImportColumnDesc importColumnDesc : streamLoadTask.getColumnExprDesc()) { - // make column name case match with real column name - String columnName = importColumnDesc.getColumnName(); - String realColName = dstTable.getColumn(columnName) == null ? columnName - : dstTable.getColumn(columnName).getName(); - if (importColumnDesc.getExpr() != null) { - exprsByName.put(realColName, importColumnDesc.getExpr()); - } else { - SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); - slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); - slotDesc.setIsMaterialized(true); - // ISSUE A: src slot should be nullable even if the column is not nullable. - // because src slot is what we read from file, not represent to real column value. - // If column is not nullable, error will be thrown when filling the dest slot, - // which is not nullable - slotDesc.setIsNullable(true); - params.addToSrc_slot_ids(slotDesc.getId().asInt()); - slotDescByName.put(realColName, slotDesc); - } - } - - // analyze all exprs - for (Map.Entry entry : exprsByName.entrySet()) { - ExprSubstitutionMap smap = new ExprSubstitutionMap(); - List slots = Lists.newArrayList(); - entry.getValue().collect(SlotRef.class, slots); - for (SlotRef slot : slots) { - SlotDescriptor slotDesc = slotDescByName.get(slot.getColumnName()); - if (slotDesc == null) { - throw new UserException("unknown reference column, column=" + entry.getKey() - + ", reference=" + slot.getColumnName()); - } - smap.getLhs().add(slot); - smap.getRhs().add(new SlotRef(slotDesc)); - } - Expr expr = entry.getValue().clone(smap); - expr.analyze(analyzer); - // check if contain aggregation - List funcs = Lists.newArrayList(); - expr.collect(FunctionCallExpr.class, funcs); - for (FunctionCallExpr fn : funcs) { - if (fn.isAggregateFunction()) { - throw new AnalysisException("Don't support aggregation function in load expression"); - } - } + // If user does not specify the column expr desc, generate it by using base schema of table. + // So that the following process can be unified + if (streamLoadTask.getColumnExprDescs() == null || streamLoadTask.getColumnExprDescs().isEmpty()) { + List columns = dstTable.getBaseSchema(); + List columnDescs = Lists.newArrayList(); + for (Column column : columns) { + ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName()); + columnDescs.add(columnDesc); + streamLoadTask.addColumnExprDesc(columnDesc); + } + } - exprsByName.put(entry.getKey(), expr); + // When doing schema change, there may have some 'shadow' columns, with prefix '__doris_shadow_' in + // their names. These columns are visible to user, but we need to generate data for these columns. + // So we add column mappings for these column. + // eg: + // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' + // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = B); + List fullSchema = dstTable.getFullSchema(); + for (Column column : fullSchema) { + if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + String baseColName = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); + ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, baseColName)); + streamLoadTask.addColumnExprDesc(columnDesc); } - } else { - for (Column column : dstTable.getBaseSchema()) { + } + Preconditions.checkState(streamLoadTask.getColumnExprDescs() != null); + Preconditions.checkState(!streamLoadTask.getColumnExprDescs().isEmpty()); + + for (ImportColumnDesc importColumnDesc : streamLoadTask.getColumnExprDescs()) { + // make column name case match with real column name + String columnName = importColumnDesc.getColumnName(); + String realColName = dstTable.getColumn(columnName) == null ? columnName + : dstTable.getColumn(columnName).getName(); + if (importColumnDesc.getExpr() != null) { + exprsByName.put(realColName, importColumnDesc.getExpr()); + } else { SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); slotDesc.setIsMaterialized(true); - // same as ISSUE A + // ISSUE A: src slot should be nullable even if the column is not nullable. + // because src slot is what we read from file, not represent to real column value. + // If column is not nullable, error will be thrown when filling the dest slot, + // which is not nullable. slotDesc.setIsNullable(true); params.addToSrc_slot_ids(slotDesc.getId().asInt()); + slotDescByName.put(realColName, slotDesc); + } + } - slotDescByName.put(column.getName(), slotDesc); + // analyze all exprs + for (Map.Entry entry : exprsByName.entrySet()) { + ExprSubstitutionMap smap = new ExprSubstitutionMap(); + List slots = Lists.newArrayList(); + entry.getValue().collect(SlotRef.class, slots); + for (SlotRef slot : slots) { + SlotDescriptor slotDesc = slotDescByName.get(slot.getColumnName()); + if (slotDesc == null) { + throw new UserException("unknown reference column, column=" + entry.getKey() + + ", reference=" + slot.getColumnName()); + } + smap.getLhs().add(slot); + smap.getRhs().add(new SlotRef(slotDesc)); + } + Expr expr = entry.getValue().clone(smap); + expr.analyze(analyzer); + + // check if contain aggregation + List funcs = Lists.newArrayList(); + expr.collect(FunctionCallExpr.class, funcs); + for (FunctionCallExpr fn : funcs) { + if (fn.isAggregateFunction()) { + throw new AnalysisException("Don't support aggregation function in load expression"); + } } + exprsByName.put(entry.getKey(), expr); } // analyze where statement diff --git a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java index c2cf514fd57f58..0aa45445fc012e 100644 --- a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -17,6 +17,7 @@ package org.apache.doris.task; +import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.common.MarkedCountDownLatch; @@ -126,6 +127,11 @@ public TCreateTabletReq toThrift() { if (bfColumns != null && bfColumns.contains(column.getName())) { tColumn.setIs_bloom_filter_column(true); } + // when doing schema change, some modified column has a prefix in name. + // this prefix is only used in FE, not visible to BE, so we should remove this prefix. + if(column.getName().startsWith(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + tColumn.setColumn_name(column.getName().substring(SchemaChangeHandler.SHADOW_NAME_PRFIX.length())); + } tColumns.add(tColumn); } tSchema.setColumns(tColumns); diff --git a/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java b/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java index 4d8334b08020ef..fb11b3c407ce62 100644 --- a/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java +++ b/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java @@ -204,7 +204,7 @@ private Map createEtlIndices(OlapTable table, long partitionId } else { aggregation = aggregateType.name(); } - } else if ("UNIQUE_KEYS" == table.getKeysType().name()) { + } else if (table.getKeysType().name().equalsIgnoreCase("UNIQUE_KEYS")) { aggregation = "REPLACE"; } dppColumn.put("aggregation_method", aggregation); diff --git a/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java b/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java index c9b65736815b4b..ed3d76c0b96531 100644 --- a/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java +++ b/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java @@ -82,7 +82,7 @@ public void plan(List> fileStatusesList, int filesAdded) // Generate tuple descriptor List slotRefs = Lists.newArrayList(); TupleDescriptor tupleDesc = descTable.createTupleDescriptor(); - for (Column col : table.getBaseSchema()) { + for (Column col : table.getFullSchema()) { SlotDescriptor slotDesc = descTable.addSlotDescriptor(tupleDesc); slotDesc.setIsMaterialized(true); slotDesc.setColumn(col); diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index 6feb27e417c5f2..94cbf926ff4383 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -34,6 +34,7 @@ import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Joiner; +import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -51,7 +52,7 @@ public class StreamLoadTask { private TFileFormatType formatType; // optional - private List columnExprDesc; + private List columnExprDescs; private Expr whereExpr; private ColumnSeparator columnSeparator; private String partitions; @@ -83,8 +84,15 @@ public TFileFormatType getFormatType() { return formatType; } - public List getColumnExprDesc() { - return columnExprDesc; + public List getColumnExprDescs() { + return columnExprDescs; + } + + public void addColumnExprDesc(ImportColumnDesc columnExprDesc) { + if (columnExprDesc == null) { + columnExprDescs = Lists.newArrayList(); + } + this.columnExprDescs.add(columnExprDesc); } public Expr getWhereExpr() { @@ -162,7 +170,7 @@ public static StreamLoadTask fromRoutineLoadJob(RoutineLoadJob routineLoadJob) { } private void setOptionalFromRoutineLoadJob(RoutineLoadJob routineLoadJob) { - columnExprDesc = routineLoadJob.getColumnDescs(); + columnExprDescs = routineLoadJob.getColumnDescs(); whereExpr = routineLoadJob.getWhereExpr(); columnSeparator = routineLoadJob.getColumnSeparator(); partitions = routineLoadJob.getPartitions() == null ? null : Joiner.on(",").join(routineLoadJob.getPartitions()); @@ -193,7 +201,7 @@ private void setColumnToColumnExpr(String columns) throws UserException { } if (columnsStmt.getColumns() != null && !columnsStmt.getColumns().isEmpty()) { - columnExprDesc = columnsStmt.getColumns(); + columnExprDescs = columnsStmt.getColumns(); } } From f2c1431264a460b4899ff5005da8fd750e321b57 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 31 Jul 2019 10:56:31 +0800 Subject: [PATCH 23/79] fix replay job bug --- .../org/apache/doris/alter/AlterHandler.java | 11 ++- .../org/apache/doris/alter/AlterJobV2.java | 2 +- .../org/apache/doris/alter/RollupJobV2.java | 32 +++++-- .../apache/doris/alter/SchemaChangeJobV2.java | 32 ++++--- .../org/apache/doris/analysis/InsertStmt.java | 90 +++++++++++++++++-- .../org/apache/doris/qe/ShowExecutor.java | 16 ++++ .../transaction/GlobalTransactionMgr.java | 2 +- 7 files changed, 152 insertions(+), 33 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 7715dcdb37a97d..e1adba0cd89f3b 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -410,8 +410,13 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce // replay the alter job v2 public void replayAlterJobV2(AlterJobV2 alterJob) { - // always replace the alter job with the newly replayed one - alterJobsV2.put(alterJob.getJobId(), alterJob); - alterJob.replay(); + AlterJobV2 existingJob = alterJobsV2.get(alterJob.getJobId()); + if (existingJob == null) { + // This is the first time to replay the alter job, so just using the replayed alterJob to call replay(); + alterJob.replay(alterJob); + alterJobsV2.put(alterJob.getJobId(), alterJob); + } else { + existingJob.replay(alterJob); + } } } diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 83aac9026eab3c..e0534a97053d74 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -168,7 +168,7 @@ protected void getInfo(List> infos) { throw new NotImplementedException(); } - public void replay() { + public void replay(AlterJobV2 replayedJob) { throw new NotImplementedException(); } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 0641b9e7dfc04e..f466074fe81b14 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -568,7 +568,7 @@ public void readFields(DataInput in) throws IOException { * Should replay all changes before this job's state transfer to PENDING. * These changes should be same as changes in RollupHander.processAddRollup() */ - private void replayPending() { + private void replayPending(RollupJobV2 replayedJob) { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return @@ -602,6 +602,9 @@ private void replayPending() { } finally { db.writeUnlock(); } + + this.watershedTxnId = replayedJob.watershedTxnId; + this.jobState = JobState.WAITING_TXN; LOG.info("replay pending rollup job: {}", jobId); } @@ -609,7 +612,7 @@ private void replayPending() { * Replay job in WAITING_TXN state. * Should replay all changes in runPendingJob() */ - private void replayWaitingTxn() { + private void replayWaitingTxn(RollupJobV2 replayedJob) { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return @@ -627,6 +630,9 @@ private void replayWaitingTxn() { } finally { db.writeUnlock(); } + + this.jobState = JobState.RUNNING; + LOG.info("replay waiting txn rollup job: {}", jobId); } @@ -634,7 +640,7 @@ private void replayWaitingTxn() { * Replay job in FINISHED state. * Should replay all changes in runRuningJob() */ - private void replayFinished() { + private void replayFinished(RollupJobV2 replayedJob) { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db != null) { db.writeLock(); @@ -648,31 +654,39 @@ private void replayFinished() { db.writeUnlock(); } } + + this.jobState = JobState.FINISHED; + this.finishedTimeMs = replayedJob.finishedTimeMs; + LOG.info("replay finished rollup job: {}", jobId); } /* * Replay job in CANCELLED state. */ - private void replayCancelled() { + private void replayCancelled(RollupJobV2 replayedJob) { cancelInternal(); + this.jobState = JobState.CANCELLED; + this.finishedTimeMs = replayedJob.finishedTimeMs; + this.errMsg = replayedJob.errMsg; LOG.info("replay cancelled rollup job: {}", jobId); } @Override - public void replay() { + public void replay(AlterJobV2 replayedJob) { + RollupJobV2 replayedRollupJob = (RollupJobV2) replayedJob; switch (jobState) { case PENDING: - replayPending(); + replayPending(replayedRollupJob); break; case WAITING_TXN: - replayWaitingTxn(); + replayWaitingTxn(replayedRollupJob); break; case FINISHED: - replayFinished(); + replayFinished(replayedRollupJob); break; case CANCELLED: - replayCancelled(); + replayCancelled(replayedRollupJob); break; default: break; diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 40d28ce74e32b0..e993bd7f26efd4 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -513,7 +513,6 @@ public synchronized boolean cancel(String errMsg) { cancelInternal(); - jobState = JobState.CANCELLED; this.errMsg = errMsg; this.finishedTimeMs = System.currentTimeMillis(); LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg); @@ -554,6 +553,8 @@ private void cancelInternal() { db.writeUnlock(); } } + + jobState = JobState.CANCELLED; } // Check whether transactions of the given database which txnId is less than 'watershedTxnId' are finished. @@ -572,7 +573,7 @@ public static SchemaChangeJobV2 read(DataInput in) throws IOException { * Should replay all changes before this job's state transfer to PENDING. * These changes should be same as changes in SchemaChangeHandler.createJob() */ - private void replayPending() { + private void replayPending(SchemaChangeJobV2 replayedJob) { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return @@ -610,6 +611,9 @@ private void replayPending() { } finally { db.writeUnlock(); } + + this.watershedTxnId = replayedJob.watershedTxnId; + jobState = JobState.WAITING_TXN; LOG.info("replay pending schema change job: {}", jobId); } @@ -617,7 +621,7 @@ private void replayPending() { * Replay job in WAITING_TXN state. * Should replay all changes in runPendingJob() */ - private void replayWaitingTxn() { + private void replayWaitingTxn(SchemaChangeJobV2 replayedJob) { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { // database may be dropped before replaying this log. just return @@ -635,6 +639,8 @@ private void replayWaitingTxn() { } finally { db.writeUnlock(); } + + jobState = JobState.RUNNING; LOG.info("replay waiting txn schema change job: {}", jobId); } @@ -642,7 +648,7 @@ private void replayWaitingTxn() { * Replay job in FINISHED state. * Should replay all changes in runRuningJob() */ - private void replayFinished() { + private void replayFinished(SchemaChangeJobV2 replayedJob) { Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db != null) { db.writeLock(); @@ -655,31 +661,37 @@ private void replayFinished() { db.writeUnlock(); } } + jobState = JobState.FINISHED; + this.finishedTimeMs = replayedJob.finishedTimeMs; LOG.info("replay finished schema change job: {}", jobId); } /* * Replay job in CANCELLED state. */ - private void replayCancelled() { + private void replayCancelled(SchemaChangeJobV2 replayedJob) { cancelInternal(); + this.jobState = JobState.CANCELLED; + this.finishedTimeMs = replayedJob.finishedTimeMs; + this.errMsg = replayedJob.errMsg; LOG.info("replay cancelled schema change job: {}", jobId); } @Override - public void replay() { + public void replay(AlterJobV2 replayedJob) { + SchemaChangeJobV2 replayedSchemaChangeJob = (SchemaChangeJobV2) replayedJob; switch (jobState) { case PENDING: - replayPending(); + replayPending(replayedSchemaChangeJob); break; case WAITING_TXN: - replayWaitingTxn(); + replayWaitingTxn(replayedSchemaChangeJob); break; case FINISHED: - replayFinished(); + replayFinished(replayedSchemaChangeJob); break; case CANCELLED: - replayCancelled(); + replayCancelled(replayedSchemaChangeJob); break; default: break; diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java index ff942473991f5f..04d893c475f1c7 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.BrokerTable; import org.apache.doris.catalog.Catalog; @@ -377,6 +378,29 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { } } + /* + * When doing schema change, there may be some shadow columns. we should add them add the end of + * targetColumns. And use 'origColIdxsForShadowCols' to save the index of column in 'targetColumns' + * which the shadow column related to. + * eg: + * origin targetColumns: (A,B,C), shadow column: __doris_shadow_B + * after processing, targetColumns: (A, B, C, __doris_shadow_B), + * and origColIdxsForShadowCols has 1 element: "1", which is the index of column B in targetColumns. + */ + List origColIdxsForShadowCols = Lists.newArrayList(); + for (Column column : targetTable.getFullSchema()) { + if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + String origName = Column.removeNamePrefix(column.getName()); + for (int i = 0; i < targetColumns.size(); i++) { + if (targetColumns.get(i).nameEquals(origName, false)) { + origColIdxsForShadowCols.add(i); + break; + } + } + targetColumns.add(column); + } + } + // parse query statement queryStmt.setFromInsert(true); queryStmt.analyze(analyzer); @@ -388,22 +412,45 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { // Check if all columns mentioned is enough checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ; + + // handle VALUES() or SELECT list if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) { SelectStmt selectStmt = (SelectStmt) queryStmt; if (selectStmt.getValueList() != null) { + // INSERT INTO VALUES(...) List> rows = selectStmt.getValueList().getRows(); for (int rowIdx = 0; rowIdx < rows.size(); ++rowIdx) { - analyzeRow(analyzer, targetColumns, rows.get(rowIdx), rowIdx + 1); + analyzeRow(analyzer, targetColumns, rows, rowIdx, origColIdxsForShadowCols); } - for (int i = 0; i < selectStmt.getResultExprs().size(); ++i) { - selectStmt.getResultExprs().set(i, selectStmt.getValueList().getFirstRow().get(i)); - selectStmt.getBaseTblResultExprs().set(i, selectStmt.getValueList().getFirstRow().get(i)); + + // clear these 2 structures, rebuild them using VALUES exprs + selectStmt.getResultExprs().clear(); + selectStmt.getBaseTblResultExprs().clear(); + + for (int i = 0; i < selectStmt.getValueList().getFirstRow().size(); ++i) { + selectStmt.getResultExprs().add(selectStmt.getValueList().getFirstRow().get(i)); + selectStmt.getBaseTblResultExprs().add(selectStmt.getValueList().getFirstRow().get(i)); } } else { - analyzeRow(analyzer, targetColumns, selectStmt.getResultExprs(), 1); + // INSERT INTO SELECT ... + List> rows = Lists.newArrayList(); + rows.add(selectStmt.getResultExprs()); + analyzeRow(analyzer, targetColumns, rows, 0, origColIdxsForShadowCols); + // rows may be changed in analyzeRow(), so rebuild the result exprs + selectStmt.getResultExprs().clear(); + for (Expr expr : rows.get(0)) { + selectStmt.getResultExprs().add(expr); + } } isStreaming = true; } else { + if (!origColIdxsForShadowCols.isEmpty()) { + // extend the result expr by duplicating the related exprs + for (Integer idx : origColIdxsForShadowCols) { + queryStmt.getResultExprs().add(queryStmt.getResultExprs().get(idx)); + } + } + // check compatibility for (int i = 0; i < targetColumns.size(); ++i) { Column column = targetColumns.get(i); if (column.getType().isHllType()) { @@ -419,12 +466,37 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { } } - private void analyzeRow(Analyzer analyzer, List targetColumns, ArrayList row, int rowIdx) - throws AnalysisException { + private void analyzeRow(Analyzer analyzer, List targetColumns, List> rows, + int rowIdx, List origColIdxsForShadowCols) throws AnalysisException { // 1. check number of fields if equal with first row - if (row.size() != targetColumns.size()) { - throw new AnalysisException("Column count doesn't match value count at row " + rowIdx); + // targetColumns contains some shadow columns, which is added by system, + // so we should minus this + if (rows.get(rowIdx).size() != targetColumns.size() - origColIdxsForShadowCols.size()) { + throw new AnalysisException("Column count doesn't match value count at row " + (rowIdx + 1)); + } + + ArrayList row = rows.get(rowIdx); + if (!origColIdxsForShadowCols.isEmpty()) { + /* + * we should extends the row for shadow columns. + * eg: + * the origin row has exprs: (expr1, expr2, expr3), and targetColumns is (A, B, C, __doris_shadow_b) + * after processing, extentedRow is (expr1, expr2, expr3, expr2) + */ + ArrayList extentedRow = Lists.newArrayList(); + for (Expr expr : row) { + extentedRow.add(expr); + } + + for (Integer idx : origColIdxsForShadowCols) { + extentedRow.add(extentedRow.get(idx)); + } + + row = extentedRow; + rows.set(rowIdx, row); } + + // check the compatibility of expr in row and column in targetColumns for (int i = 0; i < row.size(); ++i) { Expr expr = row.get(i); Column col = targetColumns.get(i); diff --git a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java index c95c4964727dab..365b9b076b7b0c 100644 --- a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -74,6 +74,7 @@ import org.apache.doris.catalog.MetadataViewer; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; @@ -1120,6 +1121,21 @@ private void handleShowTablet() throws AnalysisException { isSync = false; break; } + + List replicas = tablet.getReplicas(); + for (Replica replica : replicas) { + Replica tmp = invertedIndex.getReplica(tabletId, replica.getBackendId()); + if (tmp == null) { + isSync = false; + break; + } + // use !=, not equals(), because this should be the same object. + if (tmp != replica) { + isSync = false; + break; + } + } + } finally { db.readUnlock(); } diff --git a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index e911311bb04268..3747258e545067 100644 --- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -1038,7 +1038,7 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat newVersion = replica.getVersion(); newVersionHash = replica.getVersionHash(); } else if (!replica.checkVersionCatchUp(partition.getVisibleVersion(), - partition.getVisibleVersionHash(), false)) { + partition.getVisibleVersionHash(), true)) { // this means the replica has error in the past, but we did not observe it // during upgrade, one job maybe in quorum finished state, for example, A,B,C 3 replica // A,B 's version is 10, C's version is 10 but C' 10 is abnormal should be rollback From b74d493c931a1fbc038bc9f71655ce4e8bb2b094 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 31 Jul 2019 15:13:25 +0800 Subject: [PATCH 24/79] fix replay bug1 --- fe/src/main/java/org/apache/doris/alter/RollupJobV2.java | 6 ++++-- .../main/java/org/apache/doris/alter/SchemaChangeJobV2.java | 3 ++- fe/src/main/java/org/apache/doris/task/StreamLoadTask.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index f466074fe81b14..1cdc58840f556e 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -603,8 +603,9 @@ private void replayPending(RollupJobV2 replayedJob) { db.writeUnlock(); } - this.watershedTxnId = replayedJob.watershedTxnId; this.jobState = JobState.WAITING_TXN; + this.watershedTxnId = replayedJob.watershedTxnId; + LOG.info("replay pending rollup job: {}", jobId); } @@ -632,6 +633,7 @@ private void replayWaitingTxn(RollupJobV2 replayedJob) { } this.jobState = JobState.RUNNING; + this.watershedTxnId = replayedJob.watershedTxnId; LOG.info("replay waiting txn rollup job: {}", jobId); } @@ -675,7 +677,7 @@ private void replayCancelled(RollupJobV2 replayedJob) { @Override public void replay(AlterJobV2 replayedJob) { RollupJobV2 replayedRollupJob = (RollupJobV2) replayedJob; - switch (jobState) { + switch (replayedJob.jobState) { case PENDING: replayPending(replayedRollupJob); break; diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index e993bd7f26efd4..4212735b2c9b15 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -641,6 +641,7 @@ private void replayWaitingTxn(SchemaChangeJobV2 replayedJob) { } jobState = JobState.RUNNING; + this.watershedTxnId = replayedJob.watershedTxnId; LOG.info("replay waiting txn schema change job: {}", jobId); } @@ -680,7 +681,7 @@ private void replayCancelled(SchemaChangeJobV2 replayedJob) { @Override public void replay(AlterJobV2 replayedJob) { SchemaChangeJobV2 replayedSchemaChangeJob = (SchemaChangeJobV2) replayedJob; - switch (jobState) { + switch (replayedJob.jobState) { case PENDING: replayPending(replayedSchemaChangeJob); break; diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index 94cbf926ff4383..3e9373d1edc549 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -89,7 +89,7 @@ public List getColumnExprDescs() { } public void addColumnExprDesc(ImportColumnDesc columnExprDesc) { - if (columnExprDesc == null) { + if (columnExprDescs == null) { columnExprDescs = Lists.newArrayList(); } this.columnExprDescs.add(columnExprDesc); From 5b2ed9259fc8dcf964a9649e762c6a015f956876 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 31 Jul 2019 16:58:16 +0800 Subject: [PATCH 25/79] fix update replica version in task --- fe/src/main/cup/sql_parser.cup | 2 +- .../java/org/apache/doris/alter/AlterHandler.java | 14 ++++++++++++++ .../org/apache/doris/analysis/ShowTabletStmt.java | 4 +++- .../apache/doris/planner/StreamLoadScanNode.java | 6 ++++-- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/fe/src/main/cup/sql_parser.cup b/fe/src/main/cup/sql_parser.cup index 9b720b0fb065d5..c8620b07a20249 100644 --- a/fe/src/main/cup/sql_parser.cup +++ b/fe/src/main/cup/sql_parser.cup @@ -1868,7 +1868,7 @@ show_param ::= :} | KW_TABLET INTEGER_LITERAL:tabletId {: - RESULT = new ShowTabletStmt(null, tabletId); + RESULT = new ShowTabletStmt(null, tabletId, null); :} | KW_TABLET KW_FROM table_name:dbTblName opt_partitions:partitionNames opt_wild_where order_by_clause:orderByClause limit_clause:limitClause {: diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index e1adba0cd89f3b..748a5022d81bda 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -33,6 +33,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.Daemon; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.persist.ReplicaPersistInfo; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.thrift.TTabletInfo; @@ -389,19 +390,32 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce LOG.info("before handle alter task replica: {}, task version: {}-{}", replica, task.getVersion(), task.getVersionHash()); + boolean versionChanged = false; if (replica.getVersion() > task.getVersion()) { // Case 2.2, do nothing } else { if (replica.getLastFailedVersion() > task.getVersion()) { // Case 2.1 replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); + versionChanged = true; } else { // Case 1 Preconditions.checkState(replica.getLastFailedVersion() == -1, replica.getLastFailedVersion()); replica.updateVersionInfo(task.getVersion(), task.getVersionHash(), replica.getDataSize(), replica.getRowCount()); + versionChanged = true; } } + if (versionChanged) { + ReplicaPersistInfo info = ReplicaPersistInfo.createForClone(task.getDbId(), task.getTableId(), + task.getPartitionId(), task.getIndexId(), task.getTabletId(), task.getBackendId(), + replica.getId(), replica.getVersion(), replica.getVersionHash(), -1, + replica.getDataSize(), replica.getRowCount(), + replica.getLastFailedVersion(), replica.getLastFailedVersionHash(), + replica.getLastSuccessVersion(), replica.getLastSuccessVersionHash()); + Catalog.getInstance().getEditLog().logUpdateReplica(info); + } + LOG.info("after handle alter task replica: {}", replica); } finally { db.writeUnlock(); diff --git a/fe/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java b/fe/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java index 37b2d6c7179be6..c3cbd8d7f14f39 100644 --- a/fe/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java @@ -63,10 +63,12 @@ public ShowTabletStmt(TableName dbTableName, long tabletId, List partiti this.dbName = null; this.tableName = null; this.isShowSingleTablet = true; + this.indexName = null; } else { this.dbName = dbTableName.getDb(); this.tableName = dbTableName.getTbl(); this.isShowSingleTablet = false; + this.indexName = Strings.emptyToNull(indexName); } this.tabletId = tabletId; this.partitionNames = partitionNames; @@ -251,7 +253,7 @@ public String toSql() { if (isShowSingleTablet) { sb.append(tabletId); } else { - sb.append(" from ").append("`").append(dbName).append("`.`").append(tableName).append("`"); + sb.append(" FROM ").append("`").append(dbName).append("`.`").append(tableName).append("`"); } if (limitElement != null) { if (limitElement.hasOffset() && limitElement.hasLimit()) { diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 06d5f47cda42ea..738514ef2d38d9 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -131,10 +131,9 @@ public void init(Analyzer analyzer) throws UserException { // So that the following process can be unified if (streamLoadTask.getColumnExprDescs() == null || streamLoadTask.getColumnExprDescs().isEmpty()) { List columns = dstTable.getBaseSchema(); - List columnDescs = Lists.newArrayList(); for (Column column : columns) { ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName()); - columnDescs.add(columnDesc); + LOG.debug("add base column {} to stream load task", column.getName()); streamLoadTask.addColumnExprDesc(columnDesc); } } @@ -150,6 +149,7 @@ public void init(Analyzer analyzer) throws UserException { if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { String baseColName = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, baseColName)); + LOG.debug("add shadow column {} to stream load task, base name: {}", column.getName(), baseColName); streamLoadTask.addColumnExprDesc(columnDesc); } } @@ -177,6 +177,8 @@ public void init(Analyzer analyzer) throws UserException { } } + LOG.debug("slotDescByName: {}, exprsByName: {}", slotDescByName, exprsByName); + // analyze all exprs for (Map.Entry entry : exprsByName.entrySet()) { ExprSubstitutionMap smap = new ExprSubstitutionMap(); From 024898f645a057a34bfa6bb53458c92a059c2e65 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 31 Jul 2019 22:51:54 +0800 Subject: [PATCH 26/79] remove name compare in memtable --- be/src/olap/delta_writer.cpp | 14 ++------------ be/src/olap/delta_writer.h | 3 ++- be/src/olap/memtable.cpp | 8 ++++---- be/src/olap/memtable.h | 5 +++-- be/src/runtime/tablet_writer_mgr.cpp | 7 ++++--- .../org/apache/doris/alter/SchemaChangeJobV2.java | 4 ++-- .../main/java/org/apache/doris/catalog/Table.java | 9 ++++++--- .../org/apache/doris/planner/OlapTableSink.java | 5 +++-- .../apache/doris/service/FrontendServiceImpl.java | 3 ++- 9 files changed, 28 insertions(+), 30 deletions(-) diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index 7a78a48c659981..6499981607d681 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -130,19 +130,9 @@ OLAPStatus DeltaWriter::init() { writer_context.load_id = _req.load_id; RETURN_NOT_OK(RowsetFactory::create_rowset_writer(writer_context, &_rowset_writer)); - const std::vector& slots = _req.tuple_desc->slots(); - const TabletSchema& schema = _tablet->tablet_schema(); - for (size_t col_id = 0; col_id < schema.num_columns(); ++col_id) { - const TabletColumn& column = schema.column(col_id); - for (size_t i = 0; i < slots.size(); ++i) { - if (slots[i]->col_name() == column.name()) { - _col_ids.push_back(i); - } - } - } _tablet_schema = &(_tablet->tablet_schema()); _schema = new Schema(*_tablet_schema); - _mem_table = new MemTable(_schema, _tablet_schema, &_col_ids, + _mem_table = new MemTable(_schema, _tablet_schema, _req.slots, _req.tuple_desc, _tablet->keys_type()); _is_init = true; return OLAP_SUCCESS; @@ -161,7 +151,7 @@ OLAPStatus DeltaWriter::write(Tuple* tuple) { RETURN_NOT_OK(_mem_table->flush(_rowset_writer.get())); SAFE_DELETE(_mem_table); - _mem_table = new MemTable(_schema, _tablet_schema, &_col_ids, + _mem_table = new MemTable(_schema, _tablet_schema, _req.slots, _req.tuple_desc, _tablet->keys_type()); } return OLAP_SUCCESS; diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h index 86f8adb8a4cd52..ad3798c7a984df 100644 --- a/be/src/olap/delta_writer.h +++ b/be/src/olap/delta_writer.h @@ -47,6 +47,8 @@ struct WriteRequest { PUniqueId load_id; bool need_gen_rollup; TupleDescriptor* tuple_desc; + // slots are in order of tablet's schema + std::vector* slots; }; class DeltaWriter { @@ -76,7 +78,6 @@ class DeltaWriter { MemTable* _mem_table; Schema* _schema; const TabletSchema* _tablet_schema; - std::vector _col_ids; bool _delta_written_success; }; diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 8aedd664a78f9d..def9e9e95aae83 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -27,12 +27,12 @@ namespace doris { MemTable::MemTable(Schema* schema, const TabletSchema* tablet_schema, - std::vector* col_ids, TupleDescriptor* tuple_desc, + std::vector* slot_descs, TupleDescriptor* tuple_desc, KeysType keys_type) : _schema(schema), _tablet_schema(tablet_schema), _tuple_desc(tuple_desc), - _col_ids(col_ids), + _slot_descs(slot_descs), _keys_type(keys_type), _row_comparator(_schema) { _schema_size = _schema->schema_size(); @@ -60,9 +60,9 @@ size_t MemTable::memory_usage() { void MemTable::insert(Tuple* tuple) { const std::vector& slots = _tuple_desc->slots(); ContiguousRow row(_schema, _tuple_buf); - for (size_t i = 0; i < _col_ids->size(); ++i) { + for (size_t i = 0; i < _slot_descs->size(); ++i) { auto cell = row.cell(i); - const SlotDescriptor* slot = slots[(*_col_ids)[i]]; + const SlotDescriptor* slot = _slot_descs[i]; bool is_null = tuple->is_null(slot->null_indicator_offset()); void* value = tuple->get_slot(slot->tuple_offset()); diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 79aac7ed40c0ec..69a0b823d7d848 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -32,7 +32,7 @@ class RowCursor; class MemTable { public: MemTable(Schema* schema, const TabletSchema* tablet_schema, - std::vector* col_ids, TupleDescriptor* tuple_desc, + std::vector* slot_descs, TupleDescriptor* tuple_desc, KeysType keys_type); ~MemTable(); size_t memory_usage(); @@ -43,7 +43,8 @@ class MemTable { Schema* _schema; const TabletSchema* _tablet_schema; TupleDescriptor* _tuple_desc; - std::vector* _col_ids; + // the slot in _slot_descs are in order of tablet's schema + std::vector* _slot_descs; KeysType _keys_type; struct RowCursorComparator { diff --git a/be/src/runtime/tablet_writer_mgr.cpp b/be/src/runtime/tablet_writer_mgr.cpp index 8b9bf6487fdcb8..0d30dbbab64fe7 100644 --- a/be/src/runtime/tablet_writer_mgr.cpp +++ b/be/src/runtime/tablet_writer_mgr.cpp @@ -207,16 +207,16 @@ Status TabletsChannel::close(int sender_id, bool* finished, } Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& params) { - std::vector* columns = nullptr; + std::vector* index_slots = nullptr; int32_t schema_hash = 0; for (auto& index : _schema->indexes()) { if (index->index_id == _index_id) { - columns = &index->slots; + index_slots = &index->slots; schema_hash = index->schema_hash; break; } } - if (columns == nullptr) { + if (index_slots == nullptr) { std::stringstream ss; ss << "unknown index id, key=" << _key; return Status::InternalError(ss.str()); @@ -231,6 +231,7 @@ Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& params) request.load_id = params.id(); request.need_gen_rollup = params.need_gen_rollup(); request.tuple_desc = _tuple_desc; + request.slots = index_slots; DeltaWriter* writer = nullptr; auto st = DeltaWriter::open(&request, &writer); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 4212735b2c9b15..d4111975db29fe 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -288,9 +288,9 @@ private void addShadowIndexToCatalog(OlapTable tbl) { indexSchemaVersionAndHashMap.get(shadowIdxId).second, indexShortKeyMap.get(shadowIdxId)); tbl.setStorageTypeToIndex(shadowIdxId, TStorageType.COLUMN); - - tbl.rebuildFullSchema(); } + + tbl.rebuildFullSchema(); } /* diff --git a/fe/src/main/java/org/apache/doris/catalog/Table.java b/fe/src/main/java/org/apache/doris/catalog/Table.java index e7c8469f893eac..69e1ba2c176bc8 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/src/main/java/org/apache/doris/catalog/Table.java @@ -83,10 +83,13 @@ public Table(long id, String tableName, TableType type, List fullSchema) this.id = id; this.name = tableName; this.type = type; - this.fullSchema = fullSchema; - this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + // must copy the list, it should not be the same object as in indexIdToSchmea if (fullSchema != null) { - for (Column col : fullSchema) { + this.fullSchema = Lists.newArrayList(fullSchema); + } + this.nameToColumn = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + if (this.fullSchema != null) { + for (Column col : this.fullSchema) { nameToColumn.put(col.getName(), col); } } else { diff --git a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java index a9ce89734797ba..f330b0403ada1d 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -179,8 +179,9 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) { for (Map.Entry> pair : table.getIndexIdToSchema().entrySet()) { List columns = Lists.newArrayList(); columns.addAll(pair.getValue().stream().map(Column::getName).collect(Collectors.toList())); - schemaParam.addToIndexes(new TOlapTableIndexSchema(pair.getKey(), columns, - table.getSchemaHashByIndexId(pair.getKey()))); + TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, + table.getSchemaHashByIndexId(pair.getKey())); + schemaParam.addToIndexes(indexSchema); } return schemaParam; } diff --git a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 072a5b834386ad..12e27354ef0eb3 100644 --- a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -812,7 +812,8 @@ private TExecPlanFragmentParams streamLoadPutImpl(TStreamLoadPutRequest request) } StreamLoadTask streamLoadTask = StreamLoadTask.fromTStreamLoadPutRequest(request); StreamLoadPlanner planner = new StreamLoadPlanner(db, (OlapTable) table, streamLoadTask); - return planner.plan(streamLoadTask.getId()); + TExecPlanFragmentParams plan = planner.plan(streamLoadTask.getId()); + return plan; } finally { db.readUnlock(); } From 0c6d1606ae38527872cf919f1c7a7796863a7c48 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 31 Jul 2019 22:59:41 +0800 Subject: [PATCH 27/79] fix compile bug --- be/src/olap/memtable.cpp | 3 +-- be/src/olap/memtable.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index def9e9e95aae83..cd74adde7dac56 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -58,11 +58,10 @@ size_t MemTable::memory_usage() { } void MemTable::insert(Tuple* tuple) { - const std::vector& slots = _tuple_desc->slots(); ContiguousRow row(_schema, _tuple_buf); for (size_t i = 0; i < _slot_descs->size(); ++i) { auto cell = row.cell(i); - const SlotDescriptor* slot = _slot_descs[i]; + const SlotDescriptor* slot = (*_slot_descs)[i]; bool is_null = tuple->is_null(slot->null_indicator_offset()); void* value = tuple->get_slot(slot->tuple_offset()); diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 69a0b823d7d848..bc14ce7c22928c 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -32,7 +32,7 @@ class RowCursor; class MemTable { public: MemTable(Schema* schema, const TabletSchema* tablet_schema, - std::vector* slot_descs, TupleDescriptor* tuple_desc, + std::vector* slot_descs, TupleDescriptor* tuple_desc, KeysType keys_type); ~MemTable(); size_t memory_usage(); From 9ff164f86b3e9d2e44017ffc71bae4e0a2f6c4dc Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 5 Aug 2019 18:00:36 +0800 Subject: [PATCH 28/79] refactor insert stmt --- .../org/apache/doris/analysis/InsertStmt.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java index 04d893c475f1c7..f6c66ea5850336 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -309,7 +309,7 @@ private void analyzeTargetTable(Analyzer analyzer) throws AnalysisException { // need a descriptor DescriptorTable descTable = analyzer.getDescTbl(); olapTuple = descTable.createTupleDescriptor(); - for (Column col : olapTable.getBaseSchema()) { + for (Column col : olapTable.getFullSchema()) { SlotDescriptor slotDesc = descTable.addSlotDescriptor(olapTuple); slotDesc.setIsMaterialized(true); slotDesc.setType(col.getType()); @@ -357,7 +357,7 @@ private void checkColumnCoverage(Set mentionedCols, List baseCol } } - public void analyzeSubquery(Analyzer analyzer) throws UserException { + private void analyzeSubquery(Analyzer analyzer) throws UserException { // Analyze columns mentioned in the statement. Set mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); if (targetColumnNames == null) { @@ -386,6 +386,9 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { * origin targetColumns: (A,B,C), shadow column: __doris_shadow_B * after processing, targetColumns: (A, B, C, __doris_shadow_B), * and origColIdxsForShadowCols has 1 element: "1", which is the index of column B in targetColumns. + * + * If the column which the shadow column related to is not mentioned, then do not add the shadow + * column to targetColumns. They will be filled by null or default value when loading. */ List origColIdxsForShadowCols = Lists.newArrayList(); for (Column column : targetTable.getFullSchema()) { @@ -394,10 +397,10 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { for (int i = 0; i < targetColumns.size(); i++) { if (targetColumns.get(i).nameEquals(origName, false)) { origColIdxsForShadowCols.add(i); + targetColumns.add(column); break; } } - targetColumns.add(column); } } @@ -413,7 +416,7 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { // Check if all columns mentioned is enough checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ; - // handle VALUES() or SELECT list + // handle VALUES() or SELECT constant list if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) { SelectStmt selectStmt = (SelectStmt) queryStmt; if (selectStmt.getValueList() != null) { @@ -432,7 +435,7 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { selectStmt.getBaseTblResultExprs().add(selectStmt.getValueList().getFirstRow().get(i)); } } else { - // INSERT INTO SELECT ... + // INSERT INTO SELECT 1,2,3 ... List> rows = Lists.newArrayList(); rows.add(selectStmt.getResultExprs()); analyzeRow(analyzer, targetColumns, rows, 0, origColIdxsForShadowCols); @@ -444,6 +447,7 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { } isStreaming = true; } else { + // INSERT INTO SELECT ... FROM tbl if (!origColIdxsForShadowCols.isEmpty()) { // extend the result expr by duplicating the related exprs for (Integer idx : origColIdxsForShadowCols) { @@ -464,6 +468,25 @@ public void analyzeSubquery(Analyzer analyzer) throws UserException { } } } + + // expand base table ref result exprs in QueryStmt + if (!origColIdxsForShadowCols.isEmpty()) { + if (queryStmt.getResultExprs().size() != queryStmt.getBaseTblResultExprs().size()) { + for (Integer idx : origColIdxsForShadowCols) { + queryStmt.getBaseTblResultExprs().add(queryStmt.getBaseTblResultExprs().get(idx)); + } + } + } + + if (LOG.isDebugEnabled()) { + for (Expr expr : queryStmt.getResultExprs()) { + LOG.debug("final result expr: {}, {}", expr, System.identityHashCode(expr)); + } + + for (Expr expr : queryStmt.getBaseTblResultExprs()) { + LOG.debug("final base table result expr: {}, {}", expr, System.identityHashCode(expr)); + } + } } private void analyzeRow(Analyzer analyzer, List targetColumns, List> rows, From 5ebd6f2229bd631ee0f25e062e223bffd464e498 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 5 Aug 2019 23:15:58 +0800 Subject: [PATCH 29/79] fix insert bugs --- .../org/apache/doris/analysis/InsertStmt.java | 38 ++++++++++++------- .../org/apache/doris/planner/Planner.java | 1 - .../doris/planner/StreamLoadScanNode.java | 2 +- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java index f6c66ea5850336..221deae2c23cdf 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -361,6 +361,8 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { // Analyze columns mentioned in the statement. Set mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); if (targetColumnNames == null) { + // the mentioned columns are columns which are visible to user, so here we use + // getBaseSchema(), not getFullSchema() for (Column col : targetTable.getBaseSchema()) { mentionedColumns.add(col.getName()); targetColumns.add(col); @@ -379,16 +381,17 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { } /* - * When doing schema change, there may be some shadow columns. we should add them add the end of - * targetColumns. And use 'origColIdxsForShadowCols' to save the index of column in 'targetColumns' - * which the shadow column related to. - * eg: - * origin targetColumns: (A,B,C), shadow column: __doris_shadow_B - * after processing, targetColumns: (A, B, C, __doris_shadow_B), - * and origColIdxsForShadowCols has 1 element: "1", which is the index of column B in targetColumns. - * - * If the column which the shadow column related to is not mentioned, then do not add the shadow - * column to targetColumns. They will be filled by null or default value when loading. + * When doing schema change, there may be some shadow columns. we should add + * them to the end of targetColumns. And use 'origColIdxsForShadowCols' to save + * the index of column in 'targetColumns' which the shadow column related to. + * eg: origin targetColumns: (A,B,C), shadow column: __doris_shadow_B after + * processing, targetColumns: (A, B, C, __doris_shadow_B), and + * origColIdxsForShadowCols has 1 element: "1", which is the index of column B + * in targetColumns. + * + * Rule A: If the column which the shadow column related to is not mentioned, + * then do not add the shadow column to targetColumns. They will be filled by + * null or default value when loading. */ List origColIdxsForShadowCols = Lists.newArrayList(); for (Column column : targetTable.getFullSchema()) { @@ -396,6 +399,7 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { String origName = Column.removeNamePrefix(column.getName()); for (int i = 0; i < targetColumns.size(); i++) { if (targetColumns.get(i).nameEquals(origName, false)) { + // Rule A origColIdxsForShadowCols.add(i); targetColumns.add(column); break; @@ -469,23 +473,31 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { } } - // expand base table ref result exprs in QueryStmt + // expand baseTblResultExprs and colLabels in QueryStmt if (!origColIdxsForShadowCols.isEmpty()) { if (queryStmt.getResultExprs().size() != queryStmt.getBaseTblResultExprs().size()) { for (Integer idx : origColIdxsForShadowCols) { queryStmt.getBaseTblResultExprs().add(queryStmt.getBaseTblResultExprs().get(idx)); } } + + if (queryStmt.getResultExprs().size() != queryStmt.getColLabels().size()) { + for (Integer idx : origColIdxsForShadowCols) { + queryStmt.getColLabels().add(queryStmt.getColLabels().get(idx)); + } + } } if (LOG.isDebugEnabled()) { for (Expr expr : queryStmt.getResultExprs()) { LOG.debug("final result expr: {}, {}", expr, System.identityHashCode(expr)); } - for (Expr expr : queryStmt.getBaseTblResultExprs()) { LOG.debug("final base table result expr: {}, {}", expr, System.identityHashCode(expr)); } + for (String colLabel : queryStmt.getColLabels()) { + LOG.debug("final col label: {}", colLabel); + } } } @@ -640,7 +652,7 @@ public void prepareExpressions() throws UserException { exprByName.put(col.getName(), expr); } // reorder resultExprs in table column order - for (Column col : targetTable.getBaseSchema()) { + for (Column col : targetTable.getFullSchema()) { if (exprByName.containsKey(col.getName())) { resultExprs.add(exprByName.get(col.getName())); } else { diff --git a/fe/src/main/java/org/apache/doris/planner/Planner.java b/fe/src/main/java/org/apache/doris/planner/Planner.java index 857fc9e923c570..3b34fc8989a2bd 100644 --- a/fe/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/src/main/java/org/apache/doris/planner/Planner.java @@ -145,7 +145,6 @@ public void createPlanFragments(StatementBase statment, Analyzer analyzer, TQuer singleNodePlanner = new SingleNodePlanner(plannerContext); PlanNode singleNodePlan = singleNodePlanner.createSingleNodePlan(); - List resultExprs = queryStmt.getResultExprs(); if (statment instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) statment; insertStmt.prepareExpressions(); diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 738514ef2d38d9..6eeec812e74b28 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -127,7 +127,7 @@ public void init(Analyzer analyzer) throws UserException { // this means that there are three columns(k1, k2, v1) in source file, // and v2 is derived from (k1 + k2) - // If user does not specify the column expr desc, generate it by using base schema of table. + // If user does not specify the column expr descs, generate it by using base schema of table. // So that the following process can be unified if (streamLoadTask.getColumnExprDescs() == null || streamLoadTask.getColumnExprDescs().isEmpty()) { List columns = dstTable.getBaseSchema(); From 00ef9ccf6dd6854ef52c1967320180e49a9a976e Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 6 Aug 2019 09:58:26 +0800 Subject: [PATCH 30/79] change dpp to 3.2 --- fe/src/main/java/org/apache/doris/common/FeConstants.java | 2 +- fe/src/main/java/org/apache/doris/load/Load.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/common/FeConstants.java b/fe/src/main/java/org/apache/doris/common/FeConstants.java index 9174ec4c56aa79..fd2d3d8676ae2f 100644 --- a/fe/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/src/main/java/org/apache/doris/common/FeConstants.java @@ -28,7 +28,7 @@ public class FeConstants { public static int checkpoint_interval_second = 60; // 1 minutes // dpp version - public static String dpp_version = "3_1_0"; + public static String dpp_version = "3_2_0"; // bloom filter false positive probability public static double default_bloom_filter_fpp = 0.05; diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 8ba271eb7808b1..9afd12cd46ece1 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -723,8 +723,7 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // eg: // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = B); - List fullSchema = table.getFullSchema(); - for (Column column : fullSchema) { + for (Column column : table.getFullSchema()) { if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { if (assignColumnToFunction == null) { assignColumnToFunction = Maps.newHashMap(); @@ -743,11 +742,11 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip columnToFunction = Maps.newHashMap(); for (Entry>> entry : columnToHadoopFunction.entrySet()) { String mappingColumnName = entry.getKey(); - if (table.getColumn(mappingColumnName) == null) { + Column mappingColumn = table.getColumn(mappingColumnName); + if (mappingColumn == null) { throw new DdlException("Mapping column is not in table. column: " + mappingColumnName); } - Column mappingColumn = table.getColumn(mappingColumnName); Pair> function = entry.getValue(); try { DataDescription.validateMappingFunction(function.first, function.second, columnNameMap, From 02959abd0362638606f8567ec06f2f9e41807bb9 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 9 Aug 2019 10:52:40 +0800 Subject: [PATCH 31/79] modify create table timeout --- .../org/apache/doris/alter/RollupJobV2.java | 23 ++++++----- .../apache/doris/alter/SchemaChangeJobV2.java | 17 +++++--- .../org/apache/doris/catalog/Catalog.java | 21 ++++++---- .../java/org/apache/doris/common/Config.java | 5 +++ .../doris/common/MarkedCountDownLatch.java | 10 ++++- .../org/apache/doris/master/MasterImpl.java | 40 +++++++++++-------- .../apache/doris/task/CreateReplicaTask.java | 10 +++++ 7 files changed, 86 insertions(+), 40 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 1cdc58840f556e..f7471a6634f506 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -207,8 +207,8 @@ protected void runPendingJob() { // send all tasks and wait them finished AgentTaskQueue.addBatchTask(batchTask); AgentTaskExecutor.submit(batchTask); - // max timeout is 30 seconds - long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, 30000); + long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, + Config.max_create_table_timeout_second * 1000L); boolean ok = false; try { ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); @@ -217,16 +217,21 @@ protected void runPendingJob() { ok = false; } - if (!ok) { + if (!ok || !countDownLatch.getStatus().ok()) { // create rollup replicas failed. just cancel the job // clear tasks and show the failed replicas to user AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); - List> unfinishedMarks = countDownLatch.getLeftMarks(); - // only show at most 10 results - List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 10)); - String idStr = Joiner.on(", ").join(subList); - LOG.warn("failed to create rollup replicas for job: {}, {}", jobId, idStr); - cancel("Create rollup replicas failed. Error replicas: " + idStr); + String errMsg = null; + if (!countDownLatch.getStatus().ok()) { + errMsg = countDownLatch.getStatus().getErrorMsg(); + } else { + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 3 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); + errMsg = "Error replicas:" + Joiner.on(", ").join(subList); + } + LOG.warn("failed to create rollup replicas for job: {}, {}", jobId, errMsg); + cancel("Create rollup replicas failed. Error: " + errMsg); return; } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index d4111975db29fe..3f1a4169a99d16 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -237,12 +237,17 @@ protected void runPendingJob() { // create replicas failed. just cancel the job // clear tasks and show the failed replicas to user AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); - List> unfinishedMarks = countDownLatch.getLeftMarks(); - // only show at most 10 results - List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 10)); - String idStr = Joiner.on(", ").join(subList); - LOG.warn("failed to create replicas for job: {}, {}", jobId, idStr); - cancel("Create replicas failed. Error replicas: " + idStr); + String errMsg = null; + if (!countDownLatch.getStatus().ok()) { + errMsg = countDownLatch.getStatus().getErrorMsg(); + } else { + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 3 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); + errMsg = "Error replicas:" + Joiner.on(", ").join(subList); + } + LOG.warn("failed to create replicas for job: {}, {}", jobId, errMsg); + cancel("Create replicas failed. Error: " + errMsg); return; } diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 6240b76e48c7de..2789543d78141f 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -3290,6 +3290,7 @@ private Partition createPartitionWithIndices(String clusterName, long dbId, long // estimate timeout long timeout = Config.tablet_create_timeout_second * 1000L * totalTaskNum; + timeout = Math.min(timeout, Config.max_create_table_timeout_second * 1000); try { ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { @@ -3297,19 +3298,25 @@ private Partition createPartitionWithIndices(String clusterName, long dbId, long ok = false; } - if (!ok) { - errMsg = "Failed to create partition[" + partitionName + "]. Timeout"; + if (!ok || !countDownLatch.getStatus().ok()) { + errMsg = "Failed to create partition[" + partitionName + "]. Timeout."; // clear tasks List tasks = batchTask.getAllTasks(); for (AgentTask task : tasks) { AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.CREATE, task.getSignature()); } - List> unfinishedMarks = countDownLatch.getLeftMarks(); - // only show at most 10 results - List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 10)); - String idStr = Joiner.on(", ").join(subList); - LOG.warn("{}. unfinished marks: {}", errMsg, idStr); + if (!countDownLatch.getStatus().ok()) { + errMsg += " Error: " + countDownLatch.getStatus().getErrorMsg(); + } else { + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 3 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); + if (!subList.isEmpty()) { + errMsg += " Unfinished mark: " + Joiner.on(", ").join(subList); + } + } + LOG.warn(errMsg); throw new DdlException(errMsg); } } else { diff --git a/fe/src/main/java/org/apache/doris/common/Config.java b/fe/src/main/java/org/apache/doris/common/Config.java index 92f9bc290d8847..ab289927c2b85b 100644 --- a/fe/src/main/java/org/apache/doris/common/Config.java +++ b/fe/src/main/java/org/apache/doris/common/Config.java @@ -263,6 +263,11 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true, masterOnly = true) public static int tablet_create_timeout_second = 1; + /* + * In order not to wait too long for create table(index), set a max timeout. + */ + @ConfField(mutable = true, masterOnly = true) + public static int max_create_table_timeout_second = 60; /* * Maximal waiting time for all publish version tasks of one transaction to be finished diff --git a/fe/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java b/fe/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java index bc25b3877f2b8f..74a42ee6c76b8b 100644 --- a/fe/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java +++ b/fe/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java @@ -28,6 +28,7 @@ public class MarkedCountDownLatch extends CountDownLatch { private Multimap marks; + private Status st = Status.OK; public MarkedCountDownLatch(int count) { super(count); @@ -50,9 +51,16 @@ public synchronized List> getLeftMarks() { return Lists.newArrayList(marks.entries()); } - public synchronized void countDownToZero() { + public Status getStatus() { + return st; + } + + public synchronized void countDownToZero(Status status) { while(getCount() > 0) { super.countDown(); } + if (st.ok()) { + st = status; + } } } diff --git a/fe/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/src/main/java/org/apache/doris/master/MasterImpl.java index 75d29f0b156f40..7943f99f68d8fd 100644 --- a/fe/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/src/main/java/org/apache/doris/master/MasterImpl.java @@ -134,7 +134,8 @@ public TMasterResult finishTask(TFinishTaskRequest request) throws TException { // We start to let FE perceive the task's error msg if (taskType != TTaskType.MAKE_SNAPSHOT && taskType != TTaskType.UPLOAD && taskType != TTaskType.DOWNLOAD && taskType != TTaskType.MOVE - && taskType != TTaskType.CLONE && taskType != TTaskType.PUBLISH_VERSION) { + && taskType != TTaskType.CLONE && taskType != TTaskType.PUBLISH_VERSION + && taskType != TTaskType.CREATE) { return result; } } @@ -233,24 +234,29 @@ private void finishCreateReplica(AgentTask task, TFinishTaskRequest request) { // if we get here, this task will be removed from AgentTaskQueue for certain. // because in this function, the only problem that cause failure is meta missing. // and if meta is missing, we no longer need to resend this task + try { + CreateReplicaTask createReplicaTask = (CreateReplicaTask) task; + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + createReplicaTask.countDownToZero(task.getBackendId() + ": " + request.getTask_status().getError_msgs().toString()); + } else { + long tabletId = createReplicaTask.getTabletId(); - CreateReplicaTask createReplicaTask = (CreateReplicaTask) task; - long tabletId = createReplicaTask.getTabletId(); - - if (request.isSetFinish_tablet_infos()) { - Replica replica = Catalog.getCurrentInvertedIndex().getReplica(createReplicaTask.getTabletId(), - createReplicaTask.getBackendId()); - replica.setPathHash(request.getFinish_tablet_infos().get(0).getPath_hash()); + if (request.isSetFinish_tablet_infos()) { + Replica replica = Catalog.getCurrentInvertedIndex().getReplica(createReplicaTask.getTabletId(), + createReplicaTask.getBackendId()); + replica.setPathHash(request.getFinish_tablet_infos().get(0).getPath_hash()); + } + + // this should be called before 'countDownLatch()' + Catalog.getCurrentSystemInfo().updateBackendReportVersion(task.getBackendId(), request.getReport_version(), task.getDbId()); + + createReplicaTask.countDownLatch(task.getBackendId(), task.getSignature()); + LOG.debug("finish create replica. tablet id: {}, be: {}, report version: {}", + tabletId, task.getBackendId(), request.getReport_version()); + } + } finally { + AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.CREATE, task.getSignature()); } - - // this should be called before 'countDownLatch()' - Catalog.getCurrentSystemInfo().updateBackendReportVersion(task.getBackendId(), request.getReport_version(), - task.getDbId()); - - createReplicaTask.countDownLatch(task.getBackendId(), task.getSignature()); - LOG.debug("finish create replica. tablet id: {}, be: {}, report version: {}", - tabletId, task.getBackendId(), request.getReport_version()); - AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.CREATE, task.getSignature()); } private void finishRealtimePush(AgentTask task, TFinishTaskRequest request) { diff --git a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 0aa45445fc012e..9da5f4f66af7fb 100644 --- a/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -21,8 +21,10 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.common.MarkedCountDownLatch; +import org.apache.doris.common.Status; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TCreateTabletReq; +import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTabletSchema; @@ -97,6 +99,14 @@ public void countDownLatch(long backendId, long tabletId) { } } + // call this always means one of tasks is failed. count down to zero to finish entire task + public void countDownToZero(String errMsg) { + if (this.latch != null) { + latch.countDownToZero(new Status(TStatusCode.CANCELLED, errMsg)); + LOG.debug("CreateReplicaTask download to zero. error msg: {}", errMsg); + } + } + public void setLatch(MarkedCountDownLatch latch) { this.latch = latch; } From 4e2c6354c7dac8dd3c68d18974151e51617be9a4 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 9 Aug 2019 17:54:37 +0800 Subject: [PATCH 32/79] modify broker load shadow column --- .../doris/analysis/DataDescription.java | 38 ++++++++------- .../apache/doris/clone/TabletScheduler.java | 2 +- .../main/java/org/apache/doris/load/Load.java | 48 +++++++++---------- .../org/apache/doris/load/LoadChecker.java | 2 - .../java/org/apache/doris/load/LoadJob.java | 17 ++----- .../doris/load/loadv2/BrokerLoadJob.java | 8 +--- .../org/apache/doris/load/loadv2/LoadJob.java | 2 +- .../apache/doris/load/loadv2/LoadManager.java | 3 +- .../apache/doris/planner/BrokerScanNode.java | 4 +- .../java/org/apache/doris/qe/Coordinator.java | 2 +- .../java/org/apache/doris/qe/DdlExecutor.java | 2 +- .../doris/service/FrontendServiceImpl.java | 1 + 12 files changed, 58 insertions(+), 71 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index 9bff9c8e9f8003..dea3215d541597 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -35,7 +35,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -66,11 +65,16 @@ public class DataDescription { private static final Logger LOG = LogManager.getLogger(DataDescription.class); public static String FUNCTION_HASH_HLL = "hll_hash"; - private static final List hadoopSupportFunctionName = Arrays.asList("strftime", "time_format", + private static final List HADOOP_SUPPORT_FUNCTION_NAMES = Arrays.asList( + "strftime", + "time_format", "alignment_timestamp", - "default_value", "md5sum", - "replace_value", "now", - "hll_hash"); + "default_value", + "md5sum", + "replace_value", + "now", "hll_hash", + "substitute"); + private final String tableName; private final List partitionNames; private final List filePaths; @@ -80,20 +84,24 @@ public class DataDescription { private final String fileFormat; private final List columnsFromPath; private final boolean isNegative; + // save column mapping in SET(xxx = xxx) clause private final List columnMappingList; // Used for mini load private TNetworkAddress beAddr; private String lineDelimiter; - // This param only include the hadoop function which need to be checked in the future. - // For hadoop load, this param is also used to persistence. - private Map>> columnToHadoopFunction; - /** + /* * Merged from columns and columnMappingList * ImportColumnDesc: column name to expr or null - **/ + */ private List parsedColumnExprList = Lists.newArrayList(); + /* + * This param only include the hadoop function which need to be checked in the future. + * For hadoop load, this param is also used to persistence. + * The function in this param is copied from 'parsedColumnExprList' + */ + private Map>> columnToHadoopFunction = Maps.newHashMap(); private boolean isHadoopLoad = false; @@ -184,13 +192,9 @@ public void setLineDelimiter(String lineDelimiter) { } public void addColumnMapping(String functionName, Pair> pair) { - if (Strings.isNullOrEmpty(functionName) || pair == null) { return; } - if (columnToHadoopFunction == null) { - columnToHadoopFunction = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - } columnToHadoopFunction.put(functionName, pair); } @@ -243,16 +247,14 @@ private void analyzeColumns() throws AnalysisException { parsedColumnExprList.add(importColumnDesc); } - if (columnMappingList == null || columnMappingList.isEmpty()) { return; } + // Step2: analyze column mapping // the column expr only support the SlotRef or eq binary predicate which's child(0) must be a SloRef. // the duplicate column name of SloRef is forbidden. - columnToHadoopFunction = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); for (Expr columnExpr : columnMappingList) { - if (!(columnExpr instanceof BinaryPredicate)) { throw new AnalysisException("Mapping function expr only support the column or eq binary predicate. " + "Expr: " + columnExpr.toSql()); @@ -289,7 +291,7 @@ private void analyzeColumnToHadoopFunction(String columnName, Expr child1) throw Preconditions.checkState(child1 instanceof FunctionCallExpr); FunctionCallExpr functionCallExpr = (FunctionCallExpr) child1; String functionName = functionCallExpr.getFnName().getFunction(); - if (!hadoopSupportFunctionName.contains(functionName.toLowerCase())) { + if (!HADOOP_SUPPORT_FUNCTION_NAMES.contains(functionName.toLowerCase())) { return; } List paramExprs = functionCallExpr.getParams().exprs(); diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java index 5c62f885ccd7be..680a22e142f5d7 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -875,7 +875,7 @@ private void deleteReplicaInternal(TabletSchedCtx tabletCtx, Replica replica, St * 2. Wait for any txns before the watermark txn id to be finished. If all are finished, which means this replica is * safe to be deleted. */ - if (!force && replica.getState().isLoadable() && replica.getWatermarkTxnId() == -1) { + if (!force && replica.getState().canLoad() && replica.getWatermarkTxnId() == -1) { long nextTxnId = Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId(); replica.setWatermarkTxnId(nextTxnId); replica.setState(ReplicaState.DECOMMISSION); diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 9afd12cd46ece1..c70d00d5066a71 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -234,7 +234,8 @@ private void writeUnlock() { // return true if we truly add the load job // return false otherwise (eg: a retry request) - public boolean addLoadJob(TMiniLoadRequest request) throws DdlException { + @Deprecated + public boolean addMiniLoadJob(TMiniLoadRequest request) throws DdlException { // get params String fullDbName = request.getDb(); String tableName = request.getTbl(); @@ -297,8 +298,7 @@ public boolean addLoadJob(TMiniLoadRequest request) throws DdlException { } DataDescription dataDescription = new DataDescription(tableName, partitionNames, filePaths, - columnNames, - columnSeparator, formatType, false, null); + columnNames, columnSeparator, formatType, false, null); dataDescription.setLineDelimiter(lineDelimiter); dataDescription.setBeAddr(beAddr); // parse hll param pair @@ -446,12 +446,7 @@ private LoadJob createLoadJob(LoadStmt stmt, EtlJobType etlJobType, } if (properties.containsKey(LoadStmt.LOAD_DELETE_FLAG_PROPERTY)) { - String flag = properties.get(LoadStmt.LOAD_DELETE_FLAG_PROPERTY); - if (flag.equalsIgnoreCase("true") || flag.equalsIgnoreCase("false")) { - job.setDeleteFlag(Boolean.parseBoolean(flag)); - } else { - throw new DdlException("Value of delete flag is invalid"); - } + throw new DdlException("Do not support load_delete_flag"); } if (properties.containsKey(LoadStmt.EXEC_MEM_LIMIT)) { @@ -469,7 +464,7 @@ private LoadJob createLoadJob(LoadStmt stmt, EtlJobType etlJobType, Map>> tableToPartitionSources = Maps.newHashMap(); for (DataDescription dataDescription : dataDescriptions) { // create source - checkAndCreateSource(db, dataDescription, tableToPartitionSources, job.getDeleteFlag(), etlJobType); + checkAndCreateSource(db, dataDescription, tableToPartitionSources, etlJobType); job.addTableName(dataDescription.getTableName()); } for (Entry>> tableEntry : tableToPartitionSources.entrySet()) { @@ -596,10 +591,11 @@ private LoadJob createLoadJob(LoadStmt stmt, EtlJobType etlJobType, return job; } + /* + * This is used for both hadoop load and broker load v2 + */ public static void checkAndCreateSource(Database db, DataDescription dataDescription, - Map>> tableToPartitionSources, - boolean deleteFlag, EtlJobType jobType) - throws DdlException { + Map>> tableToPartitionSources, EtlJobType jobType) throws DdlException { Source source = new Source(dataDescription.getFilePaths()); long tableId = -1; Set sourcePartitionIds = Sets.newHashSet(); @@ -638,10 +634,6 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip throw new DdlException("Load for AGG_KEYS table should not specify NEGATIVE"); } - if (((OlapTable) table).getKeysType() != KeysType.UNIQUE_KEYS && deleteFlag) { - throw new DdlException("Delete flag can only be used for UNIQUE_KEYS table"); - } - // get table schema List baseSchema = table.getBaseSchema(); @@ -678,7 +670,7 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { parsedColumnExprMap.put(importColumnDesc.getColumnName(), importColumnDesc.getExpr()); } - for (Column column : tableSchema) { + for (Column column : baseSchema) { String columnName = column.getName(); if (columnNames.contains(columnName)) { continue; @@ -718,19 +710,25 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // convert mapping column and func arg columns to schema format // When doing schema change, there may have some 'shadow' columns, with prefix '__doris_shadow_' in - // their names. These columns are visible to user, but we need to generate data for these columns. + // their names. These columns are invisible to user, but we need to generate data for these columns. // So we add column mappings for these column. // eg: // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' - // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = B); + // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = substitute(B)); for (Column column : table.getFullSchema()) { if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { - if (assignColumnToFunction == null) { - assignColumnToFunction = Maps.newHashMap(); - dataDescription.setColumnToFunction(assignColumnToFunction); + /* + * There is a case that if user does not specify the related origin column, eg: + * COLUMNS (A, C), and B is not specified, but B is being modified so there is a shadow column '__doris_shadow_B'. + * We can not just add a mapping function "__doris_shadow_B = substitute(B)", because Doris can not find column B. + * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping + */ + String originCol = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); + if (columnNames.contains(originCol)) { + assignColumnToFunction.put(column.getName(), Pair.create("substitute", Lists.newArrayList(originCol))); + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, originCol)); + parsedColumnExprList.add(importColumnDesc); } - assignColumnToFunction.put(column.getName(), Pair.create("substitute", - Lists.newArrayList(column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)))); } } diff --git a/fe/src/main/java/org/apache/doris/load/LoadChecker.java b/fe/src/main/java/org/apache/doris/load/LoadChecker.java index 486c32c9cdb81c..4950246eb0be06 100644 --- a/fe/src/main/java/org/apache/doris/load/LoadChecker.java +++ b/fe/src/main/java/org/apache/doris/load/LoadChecker.java @@ -452,8 +452,6 @@ private Set submitPushTasks(LoadJob job, Database db) { TPushType type = TPushType.LOAD; if (job.isSyncDeleteJob()) { type = TPushType.DELETE; - } else if (job.getDeleteFlag()) { - type = TPushType.LOAD_DELETE; } // add task to batchTask diff --git a/fe/src/main/java/org/apache/doris/load/LoadJob.java b/fe/src/main/java/org/apache/doris/load/LoadJob.java index cb788a48af0600..d8c288f7f2a1b5 100644 --- a/fe/src/main/java/org/apache/doris/load/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/LoadJob.java @@ -83,7 +83,6 @@ public enum JobState { long timestamp; private int timeoutSecond; private double maxFilterRatio; - private boolean deleteFlag; private JobState state; private BrokerDesc brokerDesc; @@ -148,7 +147,6 @@ public LoadJob(long id, long dbId, long tableId, long partitionId, String label, this.transactionId = -1; this.timestamp = -1; this.timeoutSecond = DEFAULT_TIMEOUT_S; - this.deleteFlag = true; this.state = JobState.LOADING; this.progress = 0; this.createTimeMs = System.currentTimeMillis(); @@ -200,7 +198,6 @@ public LoadJob(String label, int timeoutSecond, double maxFilterRatio) { this.timestamp = -1; this.timeoutSecond = timeoutSecond; this.maxFilterRatio = maxFilterRatio; - this.deleteFlag = false; this.state = JobState.PENDING; this.progress = 0; this.createTimeMs = System.currentTimeMillis(); @@ -284,14 +281,6 @@ public void setMaxFilterRatio(double maxFilterRatio) { public double getMaxFilterRatio() { return maxFilterRatio; } - - public void setDeleteFlag(boolean deleteFlag) { - this.deleteFlag = deleteFlag; - } - - public boolean getDeleteFlag() { - return deleteFlag; - } public JobState getState() { return state; @@ -647,7 +636,7 @@ public long getDeleteJobTimeout() { @Override public String toString() { return "LoadJob [id=" + id + ", dbId=" + dbId + ", label=" + label + ", timeoutSecond=" + timeoutSecond - + ", maxFilterRatio=" + maxFilterRatio + ", deleteFlag=" + deleteFlag + ", state=" + state + + ", maxFilterRatio=" + maxFilterRatio + ", state=" + state + ", progress=" + progress + ", createTimeMs=" + createTimeMs + ", etlStartTimeMs=" + etlStartTimeMs + ", etlFinishTimeMs=" + etlFinishTimeMs + ", loadStartTimeMs=" + loadStartTimeMs + ", loadFinishTimeMs=" + loadFinishTimeMs + ", failMsg=" + failMsg + ", etlJobType=" + etlJobType @@ -706,7 +695,7 @@ public void write(DataOutput out) throws IOException { out.writeLong(timestamp); out.writeInt(timeoutSecond); out.writeDouble(maxFilterRatio); - out.writeBoolean(deleteFlag); + out.writeBoolean(true); // delete flag, does not use anymore Text.writeString(out, state.name()); out.writeInt(progress); out.writeLong(createTimeMs); @@ -853,7 +842,7 @@ public void readFields(DataInput in) throws IOException { timeoutSecond = in.readInt(); maxFilterRatio = in.readDouble(); - deleteFlag = false; + boolean deleteFlag = false; if (version >= FeMetaVersion.VERSION_30) { deleteFlag = in.readBoolean(); } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index acee0b8b7f7ac5..10e02cf63be210 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -76,7 +76,6 @@ public class BrokerLoadJob extends LoadJob { private static final Logger LOG = LogManager.getLogger(BrokerLoadJob.class); // input params - private List dataDescriptions = Lists.newArrayList(); private BrokerDesc brokerDesc; // this param is used to persist the expr of columns // the origin stmt is persisted instead of columns expr @@ -93,12 +92,10 @@ public BrokerLoadJob() { this.jobType = EtlJobType.BROKER; } - public BrokerLoadJob(long dbId, String label, BrokerDesc brokerDesc, List dataDescriptions, - String originStmt) + private BrokerLoadJob(long dbId, String label, BrokerDesc brokerDesc, String originStmt) throws MetaNotFoundException { super(dbId, label); this.timeoutSecond = Config.broker_load_default_timeout_second; - this.dataDescriptions = dataDescriptions; this.brokerDesc = brokerDesc; this.originStmt = originStmt; this.jobType = EtlJobType.BROKER; @@ -118,8 +115,7 @@ public static BrokerLoadJob fromLoadStmt(LoadStmt stmt, String originStmt) throw // create job try { BrokerLoadJob brokerLoadJob = new BrokerLoadJob(db.getId(), stmt.getLabel().getLabelName(), - stmt.getBrokerDesc(), stmt.getDataDescriptions(), - originStmt); + stmt.getBrokerDesc(), originStmt); brokerLoadJob.setJobProperties(stmt.getProperties()); brokerLoadJob.setDataSourceInfo(db, stmt.getDataDescriptions()); return brokerLoadJob; diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index bc3c66e0f50991..eca39a7bf5ca43 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -313,7 +313,7 @@ protected static void checkDataSourceInfo(Database db, List dat // >> Map>> loadInfo = Maps.newHashMap(); // only support broker load now - Load.checkAndCreateSource(db, dataDescription, loadInfo, false, jobType); + Load.checkAndCreateSource(db, dataDescription, loadInfo, jobType); } } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadManager.java index 36f6d929e3ce64..fb76b393cdc325 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadManager.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadManager.java @@ -202,6 +202,7 @@ public void createLoadJobV1FromStmt(LoadStmt stmt, EtlJobType jobType, long time * else: return true. * @throws DdlException */ + @Deprecated public boolean createLoadJobV1FromRequest(TMiniLoadRequest request) throws DdlException { String cluster = SystemInfoService.DEFAULT_CLUSTER; if (request.isSetCluster()) { @@ -211,7 +212,7 @@ public boolean createLoadJobV1FromRequest(TMiniLoadRequest request) throws DdlEx writeLock(); try { checkLabelUsed(database.getId(), request.getLabel(), null); - return Catalog.getCurrentCatalog().getLoadInstance().addLoadJob(request); + return Catalog.getCurrentCatalog().getLoadInstance().addMiniLoadJob(request); } finally { writeUnlock(); } diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index ef342ec767d588..a1aafb2197ad1b 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -235,7 +235,7 @@ private void initColumns(ParamCreateContext context) throws UserException { // there are no columns transform List originColumnNameToExprList = context.fileGroup.getColumnExprList(); if (originColumnNameToExprList == null || originColumnNameToExprList.isEmpty()) { - for (Column column : targetTable.getBaseSchema()) { + for (Column column : targetTable.getFullSchema()) { SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); slotDesc.setIsMaterialized(true); @@ -406,6 +406,8 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr FunctionName nowFunctionName = new FunctionName("NOW"); FunctionCallExpr newFunc = new FunctionCallExpr(nowFunctionName, new FunctionParams(null)); return newFunc; + } else if (funcName.equalsIgnoreCase("substitute")) { + return funcExpr.getChild(0); } } return originExpr; diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index c583ad18b77d5f..53c7e2deebba78 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -676,7 +676,7 @@ private void cancelInternal(PPlanFragmentCancelReason cancelReason) { cancelRemoteFragmentsAsync(cancelReason); if (profileDoneSignal != null) { // count down to zero to notify all objects waiting for this - profileDoneSignal.countDownToZero(); + profileDoneSignal.countDownToZero(new Status()); LOG.info("unfinished instance: {}", profileDoneSignal.getLeftMarks().stream().map(e->DebugUtil.printId(e.getKey())).toArray()); } } diff --git a/fe/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/src/main/java/org/apache/doris/qe/DdlExecutor.java index e6364302e63bec..56137aeadcb58c 100644 --- a/fe/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -115,7 +115,7 @@ public static void execute(Catalog catalog, DdlStmt ddlStmt, String origStmt) th } jobType = EtlJobType.HADOOP; } - if (loadStmt.getVersion().equals(Load.VERSION) || loadStmt.getBrokerDesc() == null) { + if (loadStmt.getVersion().equals(Load.VERSION) || jobType == EtlJobType.HADOOP) { catalog.getLoadManager().createLoadJobV1FromStmt(loadStmt, jobType, System.currentTimeMillis()); } else { catalog.getLoadManager().createLoadJobFromStmt(loadStmt, origStmt); diff --git a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 12e27354ef0eb3..25c8297e3ebc64 100644 --- a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -335,6 +335,7 @@ public TFetchResourceResult fetchResource() throws TException { return masterImpl.fetchResource(); } + @Deprecated @Override public TFeResult miniLoad(TMiniLoadRequest request) throws TException { LOG.info("receive mini load request: label: {}, db: {}, tbl: {}, backend: {}", From ae86683a68a96cbe824217ca43541b3af36b64fa Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 10 Aug 2019 21:13:02 +0800 Subject: [PATCH 33/79] fill the data description columns --- .../doris/analysis/DataDescription.java | 23 +++++++++++++++++++ .../main/java/org/apache/doris/load/Load.java | 2 ++ .../apache/doris/planner/BrokerScanNode.java | 4 ++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index dea3215d541597..52598adc21a067 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.Pair; @@ -35,6 +36,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -524,6 +526,27 @@ public void analyzeWithoutCheckPriv() throws AnalysisException { analyzeColumns(); } + public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlException { + if (columns != null && !columns.isEmpty()) { + return; + } + + Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { + mappingColNames.add(importColumnDesc.getColumnName()); + } + + int placeholder = 0; + for (Column column : baseSchema) { + if (!mappingColNames.contains(column.getName())) { + columns.add(column.getName()); + parsedColumnExprList.add(new ImportColumnDesc(column.getName(), null)); + } else { + columns.add("__doris_tmp_" + (placeholder++)); + } + } + } + public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("DATA INFILE ("); diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index c70d00d5066a71..8e25ce204c41d6 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -637,6 +637,8 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // get table schema List baseSchema = table.getBaseSchema(); + dataDescription.fillColumnInfoIfNotSpecified(baseSchema); + // source columns List columnNames = Lists.newArrayList(); List assignColumnNames = Lists.newArrayList(); diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index a1aafb2197ad1b..c1b5341f73905d 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -232,9 +232,10 @@ private void initColumns(ParamCreateContext context) throws UserException { context.slotDescByName = slotDescByName; TBrokerScanRangeParams params = context.params; - // there are no columns transform + List originColumnNameToExprList = context.fileGroup.getColumnExprList(); if (originColumnNameToExprList == null || originColumnNameToExprList.isEmpty()) { + // there are no columns transform for (Column column : targetTable.getFullSchema()) { SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); @@ -303,7 +304,6 @@ private void initColumns(ParamCreateContext context) throws UserException { columnNameToExpr.put(entry.getKey(), expr); } params.setSrc_tuple_id(srcTupleDesc.getId().asInt()); - } /** From 40f54a9552b69e00a558a6b7f34a898ffa8994a7 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 10 Aug 2019 22:48:08 +0800 Subject: [PATCH 34/79] change meta version to 59 --- .../doris/analysis/DataDescription.java | 50 +++++++------ .../org/apache/doris/analysis/LoadStmt.java | 4 +- .../org/apache/doris/catalog/Catalog.java | 2 +- .../apache/doris/load/BrokerFileGroup.java | 7 +- .../doris/load/loadv2/BrokerLoadJob.java | 30 ++++---- .../apache/doris/planner/BrokerScanNode.java | 72 ++++++++----------- 6 files changed, 82 insertions(+), 83 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index 52598adc21a067..b220aaa853935b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -81,7 +81,7 @@ public class DataDescription { private final List partitionNames; private final List filePaths; // the column name list of data desc - private final List columns; + private List columns; private final ColumnSeparator columnSeparator; private final String fileFormat; private final List columnsFromPath; @@ -223,32 +223,23 @@ public boolean isHadoopLoad() { * "col2": "tmp_col2+1", "col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} */ private void analyzeColumns() throws AnalysisException { - if (columns == null && columnsFromPath != null) { - throw new AnalysisException("Can not specify columns_from_path without column_list"); - } - List columnList = Lists.newArrayList(); - if (columns != null) { - columnList.addAll(columns); - if (columnsFromPath != null) { - columnList.addAll(columnsFromPath); - } - } - if (columnList.isEmpty()) { - return; - } - // merge columns exprs from columns and columnMappingList // used to check duplicated column name Set columnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - // Order of parsedColumnExprList: columns(fileFieldNames) + columnsFromPath + + // merge columns exprs from columns and columnMappingList // Step1: analyze columns - for (String columnName : columnList) { - if (!columnNames.add(columnName)) { - throw new AnalysisException("Duplicate column : " + columnName); + if (columns != null && !columns.isEmpty()) { + // Step1: analyze columns + for (String columnName : columns) { + if (!columnNames.add(columnName)) { + throw new AnalysisException("Duplicate column : " + columnName); + } + ImportColumnDesc importColumnDesc = new ImportColumnDesc(columnName, null); + parsedColumnExprList.add(importColumnDesc); } - ImportColumnDesc importColumnDesc = new ImportColumnDesc(columnName, null); - parsedColumnExprList.add(importColumnDesc); } + // Step2: analyze column mapping if (columnMappingList == null || columnMappingList.isEmpty()) { return; } @@ -526,11 +517,25 @@ public void analyzeWithoutCheckPriv() throws AnalysisException { analyzeColumns(); } + /* + * If user does not specify COLUMNS in load stmt, we fill it here. + * eg1: + * both COLUMNS and SET clause is empty. after fill: + * (k1,k2,k3) + * eg2: + * COLUMNS is empty, SET is not empty + * SET ( k2 = default_value("2") ) + * after fill: + * (k1, __doris_tmp_0, k3) + * SET ( k2 = default_value("2") ) + * + * __doris_tmp_0 is a placeholder, which mean load process should skip that column in source file. + */ public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlException { if (columns != null && !columns.isEmpty()) { return; } - + columns = Lists.newArrayList(); Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { mappingColNames.add(importColumnDesc.getColumnName()); @@ -545,6 +550,7 @@ public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlExce columns.add("__doris_tmp_" + (placeholder++)); } } + LOG.debug("after fill column info. columns: {}, parsed column exprs: {}", columns, parsedColumnExprList); } public String toSql() { diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java index 99a2923549bc1a..021b15448c9253 100644 --- a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -197,13 +197,13 @@ public static void checkProperties(Map properties) throws DdlExc } - private void analyzeVersion() { + private void analyzeVersion() throws AnalysisException { if (properties == null) { return; } final String versionProperty = properties.get(VERSION); if (versionProperty != null) { - version = Load.VERSION; + throw new AnalysisException("Do not support VERSION property"); } } diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 2789543d78141f..424aa62ccd1f6e 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1617,7 +1617,7 @@ public long loadAlterJob(DataInputStream dis, long checksum, JobType type) throw } // alter job v2 - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_58) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { size = dis.readInt(); newChecksum ^= size; for (int i = 0; i < size; i++) { diff --git a/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java b/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java index bf69f4a76cdaed..b0ce4e21ca36b5 100644 --- a/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java +++ b/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java @@ -64,11 +64,11 @@ public class BrokerFileGroup implements Writable { private List columnsFromPath; private boolean isNegative; private List partitionIds; - // this is a compatible param which only happens before the function of broker has been supported. private List fileFieldNames; private List filePaths; // this is a compatible param which only happens before the function of broker has been supported. + @Deprecated private Map exprColumnMap; // this param will be recreated by data desc when the log replay private List columnExprList; @@ -91,6 +91,7 @@ public BrokerFileGroup(DataDescription dataDescription) { this.dataDescription = dataDescription; this.columnsFromPath = dataDescription.getColumnsFromPath(); this.exprColumnMap = null; + this.fileFieldNames = dataDescription.getColumnNames(); this.columnExprList = dataDescription.getParsedColumnExprList(); } @@ -179,6 +180,10 @@ public List getFilePaths() { return filePaths; } + public List getFileFieldNames() { + return fileFieldNames; + } + public List getColumnExprList() { return columnExprList; } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 10e02cf63be210..e627038e88bf12 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -109,26 +109,31 @@ public static BrokerLoadJob fromLoadStmt(LoadStmt stmt, String originStmt) throw if (db == null) { throw new DdlException("Database[" + dbName + "] does not exist"); } - // check data source info - LoadJob.checkDataSourceInfo(db, stmt.getDataDescriptions(), EtlJobType.BROKER); // create job try { BrokerLoadJob brokerLoadJob = new BrokerLoadJob(db.getId(), stmt.getLabel().getLabelName(), stmt.getBrokerDesc(), originStmt); brokerLoadJob.setJobProperties(stmt.getProperties()); - brokerLoadJob.setDataSourceInfo(db, stmt.getDataDescriptions()); + brokerLoadJob.checkAndDataSourceInfo(db, stmt.getDataDescriptions()); return brokerLoadJob; } catch (MetaNotFoundException e) { throw new DdlException(e.getMessage()); } } - private void setDataSourceInfo(Database db, List dataDescriptions) throws DdlException { - for (DataDescription dataDescription : dataDescriptions) { - BrokerFileGroup fileGroup = new BrokerFileGroup(dataDescription); - fileGroup.parse(db); - dataSourceInfo.addFileGroup(fileGroup); + private void checkAndDataSourceInfo(Database db, List dataDescriptions) throws DdlException { + // check data source info + db.readLock(); + try { + LoadJob.checkDataSourceInfo(db, dataDescriptions, EtlJobType.BROKER); + for (DataDescription dataDescription : dataDescriptions) { + BrokerFileGroup fileGroup = new BrokerFileGroup(dataDescription); + fileGroup.parse(db); + dataSourceInfo.addFileGroup(fileGroup); + } + } finally { + db.readUnlock(); } } @@ -273,7 +278,7 @@ public void analyze() { if (db == null) { throw new DdlException("Database[" + dbId + "] does not exist"); } - setDataSourceInfo(db, stmt.getDataDescriptions()); + checkAndDataSourceInfo(db, stmt.getDataDescriptions()); } catch (Exception e) { LOG.info(new LogBuilder(LogKey.LOAD_JOB, id) .add("origin_stmt", originStmt) @@ -488,7 +493,6 @@ protected void executeReplayOnVisible(TransactionState txnState) { public void write(DataOutput out) throws IOException { super.write(out); brokerDesc.write(out); - dataSourceInfo.write(out); Text.writeString(out, originStmt); } @@ -496,9 +500,9 @@ public void write(DataOutput out) throws IOException { public void readFields(DataInput in) throws IOException { super.readFields(in); brokerDesc = BrokerDesc.read(in); - // The data source info also need to be replayed - // because the load properties of old broker load has been saved in here. - dataSourceInfo.readFields(in); + if (Catalog.getCurrentCatalogJournalVersion() <= FeMetaVersion.VERSION_58) { + dataSourceInfo.readFields(in); + } if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_58) { originStmt = Text.readString(in); diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index c1b5341f73905d..ea2e3f752fa299 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -225,63 +225,49 @@ private void initParams(ParamCreateContext context) throws AnalysisException, Us * @throws UserException */ private void initColumns(ParamCreateContext context) throws UserException { - // This tuple descriptor is used for origin file - TupleDescriptor srcTupleDesc = analyzer.getDescTbl().createTupleDescriptor(); - context.tupleDescriptor = srcTupleDesc; - Map slotDescByName = Maps.newHashMap(); - context.slotDescByName = slotDescByName; - - TBrokerScanRangeParams params = context.params; - + List sourceFileColumns = context.fileGroup.getFileFieldNames(); List originColumnNameToExprList = context.fileGroup.getColumnExprList(); - if (originColumnNameToExprList == null || originColumnNameToExprList.isEmpty()) { - // there are no columns transform - for (Column column : targetTable.getFullSchema()) { - SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); - slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); - slotDesc.setIsMaterialized(true); - // ISSUE A: src slot should be nullable even if the column is not nullable. - // because src slot is what we read from file, not represent to real column value. - // If column is not nullable, error will be thrown when filling the dest slot, - // which is not nullable - slotDesc.setIsNullable(true); - slotDescByName.put(column.getName(), slotDesc); - params.addToSrc_slot_ids(slotDesc.getId().asInt()); - } - params.setSrc_tuple_id(srcTupleDesc.getId().asInt()); - return; + // originColumnNameToExprList must has emlements. because it is always filled by user or by system + Preconditions.checkState(originColumnNameToExprList != null && !originColumnNameToExprList.isEmpty()); + + // 1. create source slots + context.tupleDescriptor = analyzer.getDescTbl().createTupleDescriptor(); + context.slotDescByName = Maps.newHashMap(); + for (String sourceColName : sourceFileColumns) { + SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(context.tupleDescriptor); + slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); + slotDesc.setIsMaterialized(true); + // src slot should be nullable even if the column is not nullable. + // because src slot is what we read from file, not represent to real column value. + // If column is not nullable, error will be thrown when filling the dest slot, + // which is not nullable + slotDesc.setIsNullable(true); + context.params.addToSrc_slot_ids(slotDesc.getId().asInt()); + String realColName = targetTable.getColumn(sourceColName) == null ? sourceColName + : targetTable.getColumn(sourceColName).getName(); + context.slotDescByName.put(realColName, slotDesc); } + context.params.setSrc_tuple_id(context.tupleDescriptor.getId().asInt()); - // there are columns expr which belong to load - Map columnNameToExpr = Maps.newHashMap(); - context.exprMap = columnNameToExpr; + // 2. handle column mapping exprs + context.exprMap = Maps.newHashMap(); for (ImportColumnDesc originColumnNameToExpr : originColumnNameToExprList) { - // make column name case match with real column name String columnName = originColumnNameToExpr.getColumnName(); Expr columnExpr = originColumnNameToExpr.getExpr(); String realColName = targetTable.getColumn(columnName) == null ? columnName : targetTable.getColumn(columnName).getName(); if (columnExpr != null) { columnExpr = transformHadoopFunctionExpr(columnName, columnExpr); - columnNameToExpr.put(realColName, columnExpr); - } else { - SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); - slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); - slotDesc.setIsMaterialized(true); - // same as ISSUE A - slotDesc.setIsNullable(true); - slotDesc.setColumn(new Column(realColName, PrimitiveType.VARCHAR)); - params.addToSrc_slot_ids(slotDesc.getId().asInt()); - slotDescByName.put(realColName, slotDesc); + context.exprMap.put(realColName, columnExpr); } } - // analyze all exprs - for (Map.Entry entry : columnNameToExpr.entrySet()) { + // analyze all column mapping exprs + for (Map.Entry entry : context.exprMap.entrySet()) { ExprSubstitutionMap smap = new ExprSubstitutionMap(); List slots = Lists.newArrayList(); entry.getValue().collect(SlotRef.class, slots); for (SlotRef slot : slots) { - SlotDescriptor slotDesc = slotDescByName.get(slot.getColumnName()); + SlotDescriptor slotDesc = context.slotDescByName.get(slot.getColumnName()); if (slotDesc == null) { throw new UserException("unknown reference column, column=" + entry.getKey() + ", reference=" + slot.getColumnName()); @@ -300,10 +286,8 @@ private void initColumns(ParamCreateContext context) throws UserException { throw new AnalysisException("Don't support aggregation function in load expression"); } } - - columnNameToExpr.put(entry.getKey(), expr); + context.exprMap.put(entry.getKey(), expr); } - params.setSrc_tuple_id(srcTupleDesc.getId().asInt()); } /** From 2ef23e906434fdfacc9621f99ac2e70311310810 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 11 Aug 2019 20:27:42 +0800 Subject: [PATCH 35/79] Fix schema change bug --- .../doris/analysis/DataDescription.java | 30 ++++++++-------- .../org/apache/doris/catalog/OlapTable.java | 1 + .../org/apache/doris/catalog/Partition.java | 13 +++++++ .../main/java/org/apache/doris/load/Load.java | 35 +++++++++++++++---- .../apache/doris/planner/BrokerScanNode.java | 19 ++++++++-- 5 files changed, 75 insertions(+), 23 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index b220aaa853935b..d6b4ef6839d2dc 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -532,24 +532,24 @@ public void analyzeWithoutCheckPriv() throws AnalysisException { * __doris_tmp_0 is a placeholder, which mean load process should skip that column in source file. */ public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlException { - if (columns != null && !columns.isEmpty()) { - return; - } - columns = Lists.newArrayList(); - Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { - mappingColNames.add(importColumnDesc.getColumnName()); - } + if (columns == null || columns.isEmpty()) { + columns = Lists.newArrayList(); + Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { + mappingColNames.add(importColumnDesc.getColumnName()); + } - int placeholder = 0; - for (Column column : baseSchema) { - if (!mappingColNames.contains(column.getName())) { - columns.add(column.getName()); - parsedColumnExprList.add(new ImportColumnDesc(column.getName(), null)); - } else { - columns.add("__doris_tmp_" + (placeholder++)); + int placeholder = 0; + for (Column column : baseSchema) { + if (!mappingColNames.contains(column.getName())) { + columns.add(column.getName()); + parsedColumnExprList.add(new ImportColumnDesc(column.getName(), null)); + } else { + columns.add("__doris_tmp_" + (placeholder++)); + } } } + LOG.debug("after fill column info. columns: {}, parsed column exprs: {}", columns, parsedColumnExprList); } diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index bf2ab130a74bb0..617ee8f13162b7 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -249,6 +249,7 @@ public void rebuildFullSchema() { } } } + LOG.debug("after rebuild full schema. table {}, schema: {}", id, fullSchema); } public boolean deleteIndexInfo(String indexName) { diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 8ad13fc702e124..f14c3fb3d661a8 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -312,6 +312,11 @@ public void write(DataOutput out) throws IOException { } } + out.writeInt(idToShadowIndex.size()); + for (MaterializedIndex shadowIndex : idToShadowIndex.values()) { + shadowIndex.write(out); + } + out.writeLong(visibleVersion); out.writeLong(visibleVersionHash); @@ -338,6 +343,14 @@ public void readFields(DataInput in) throws IOException { MaterializedIndex rollupTable = MaterializedIndex.read(in); idToVisibleRollupIndex.put(rollupTable.getId(), rollupTable); } + + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + int shadowIndexCount = in.readInt(); + for (int i = 0; i < shadowIndexCount; i++) { + MaterializedIndex shadowIndex = MaterializedIndex.read(in); + idToShadowIndex.put(shadowIndex.getId(), shadowIndex); + } + } visibleVersion = in.readLong(); visibleVersionHash = in.readLong(); diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 8e25ce204c41d6..dfe57c592450ad 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -636,7 +636,7 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // get table schema List baseSchema = table.getBaseSchema(); - + // fill the column info if user does not specify them dataDescription.fillColumnInfoIfNotSpecified(baseSchema); // source columns @@ -714,7 +714,7 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // When doing schema change, there may have some 'shadow' columns, with prefix '__doris_shadow_' in // their names. These columns are invisible to user, but we need to generate data for these columns. // So we add column mappings for these column. - // eg: + // eg1: // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = substitute(B)); for (Column column : table.getFullSchema()) { @@ -726,18 +726,43 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping */ String originCol = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); - if (columnNames.contains(originCol)) { - assignColumnToFunction.put(column.getName(), Pair.create("substitute", Lists.newArrayList(originCol))); + Expr mappingExpr = parsedColumnExprMap.get(originCol); + if (mappingExpr != null) { + /* + * eg: + * (A, C) SET (B = func(xx)) + * -> + * (A, C) SET (B = func(xx), __doris_shadow_B = func(xxx)) + */ + if (columnToHadoopFunction.containsKey(originCol)) { + columnToHadoopFunction.put(column.getName(), columnToHadoopFunction.get(originCol)); + } + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), mappingExpr); + parsedColumnExprList.add(importColumnDesc); + } else { + /* + * eg: + * (A, B, C) + * -> + * (A, B, C) SET (__doris_shadow_B = substitute(B)) + */ + columnToHadoopFunction.put(column.getName(), Pair.create("substitute", Lists.newArrayList(originCol))); ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, originCol)); parsedColumnExprList.add(importColumnDesc); } + } } + LOG.debug("after add shadow column. parsedColumnExprList: {}, columnToHadoopFunction: {}", + parsedColumnExprList, columnToHadoopFunction); + Map columnNameMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); for (String columnName : columnNames) { columnNameMap.put(columnName, columnName); } + + // validate hadoop functions if (columnToHadoopFunction != null) { columnToFunction = Maps.newHashMap(); for (Entry>> entry : columnToHadoopFunction.entrySet()) { @@ -3264,5 +3289,3 @@ public Integer getLoadJobNumByTypeAndState(EtlJobType type, JobState state) { return num; } } - - diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index ea2e3f752fa299..d93e2dcac85fa4 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -17,6 +17,7 @@ package org.apache.doris.planner; +import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.BinaryPredicate; @@ -254,8 +255,21 @@ private void initColumns(ParamCreateContext context) throws UserException { for (ImportColumnDesc originColumnNameToExpr : originColumnNameToExprList) { String columnName = originColumnNameToExpr.getColumnName(); Expr columnExpr = originColumnNameToExpr.getExpr(); - String realColName = targetTable.getColumn(columnName) == null ? columnName - : targetTable.getColumn(columnName).getName(); + Column col = targetTable.getColumn(columnName); + if (col == null) { + if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + /* + * The shadow column mapping expr is added when creating load job. + * But the load job is being actually scheduled, the schema change may already finished. + * So the shadow column may not be found here. + * We can just ignore this shadow column's mapping expr, like it does not exist. + */ + continue; + } + throw new UserException("Unknown column(" + columnName + ")"); + } + + String realColName = col.getName(); if (columnExpr != null) { columnExpr = transformHadoopFunctionExpr(columnName, columnExpr); context.exprMap.put(realColName, columnExpr); @@ -288,6 +302,7 @@ private void initColumns(ParamCreateContext context) throws UserException { } context.exprMap.put(entry.getKey(), expr); } + LOG.debug("after init column, exprMap: {}", context.exprMap); } /** From 85ecd396ddf59c9de013eab4c088933dc7688ceb Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 11 Aug 2019 20:42:23 +0800 Subject: [PATCH 36/79] Fix unknown xxx bg --- .../java/org/apache/doris/planner/BrokerScanNode.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index d93e2dcac85fa4..b35cf16406935c 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -257,6 +257,7 @@ private void initColumns(ParamCreateContext context) throws UserException { Expr columnExpr = originColumnNameToExpr.getExpr(); Column col = targetTable.getColumn(columnName); if (col == null) { + // maybe 1) shadow column, 2) unknown column if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { /* * The shadow column mapping expr is added when creating load job. @@ -265,10 +266,15 @@ private void initColumns(ParamCreateContext context) throws UserException { * We can just ignore this shadow column's mapping expr, like it does not exist. */ continue; + } else if (columnExpr == null) { + // this is a unknown column, but the column expr is null, so we just consider it as + // a placeholder column, ignore it + continue; } + // unknown column but has column expr, which is not allowed. throw new UserException("Unknown column(" + columnName + ")"); } - + Preconditions.checkNotNull(col, columnName); String realColName = col.getName(); if (columnExpr != null) { columnExpr = transformHadoopFunctionExpr(columnName, columnExpr); From 0bfd424bc8c48b68f7ccc2709767a42c2050ffdf Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 12 Aug 2019 17:57:47 +0800 Subject: [PATCH 37/79] support duplicate column mapping --- .../org/apache/doris/alter/RollupHandler.java | 15 ++++- .../doris/alter/SchemaChangeHandler.java | 16 ++++- .../doris/analysis/DataDescription.java | 59 +++++++++++-------- .../apache/doris/analysis/ShowAlterStmt.java | 1 - 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index fccb7de7e7aabe..3be5d2faa1052a 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -44,8 +44,10 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Preconditions; @@ -563,7 +565,7 @@ public List> getAlterJobInfosByDb(Database db) { List> rollupJobInfos = new LinkedList>(); getOldAlterJobInfos(db, rollupJobInfos); - getAlterJobV2Infos(rollupJobInfos); + getAlterJobV2Infos(db, rollupJobInfos); // sort by // "JobId", "TableName", "CreateTime", "FinishedTime", "BaseIndexName", "RollupIndexName" @@ -573,8 +575,17 @@ public List> getAlterJobInfosByDb(Database db) { return rollupJobInfos; } - private void getAlterJobV2Infos(List> rollupJobInfos) { + private void getAlterJobV2Infos(Database db, List> rollupJobInfos) { + ConnectContext ctx = ConnectContext.get(); for (AlterJobV2 alterJob : alterJobsV2.values()) { + if (alterJob.getDbId() != db.getId()) { + continue; + } + if (ctx != null) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ctx, db.getFullName(), alterJob.getTableName(), PrivPredicate.ALTER)) { + continue; + } + } alterJob.getInfo(rollupJobInfos); } } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index de12c9c0efa58f..13c9da3b55c722 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -56,6 +56,8 @@ import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Preconditions; @@ -1207,7 +1209,7 @@ private void runOldAlterJob() { public List> getAlterJobInfosByDb(Database db) { List> schemaChangeJobInfos = new LinkedList>(); getOldAlterJobInfos(db, schemaChangeJobInfos); - getAlterJobV2Infos(schemaChangeJobInfos); + getAlterJobV2Infos(db, schemaChangeJobInfos); // sort by "JobId", "PartitionName", "CreateTime", "FinishTime", "IndexName", "IndexState" ListComparator> comparator = new ListComparator>(0, 1, 2, 3, 4, 5); @@ -1215,12 +1217,22 @@ public List> getAlterJobInfosByDb(Database db) { return schemaChangeJobInfos; } - private void getAlterJobV2Infos(List> schemaChangeJobInfos) { + private void getAlterJobV2Infos(Database db, List> schemaChangeJobInfos) { + ConnectContext ctx = ConnectContext.get(); for (AlterJobV2 alterJob : alterJobsV2.values()) { + if (alterJob.getDbId() != db.getId()) { + continue; + } + if (ctx != null) { + if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(ctx, db.getFullName(), alterJob.getTableName(), PrivPredicate.ALTER)) { + continue; + } + } alterJob.getInfo(schemaChangeJobInfos); } } + @Deprecated private void getOldAlterJobInfos(Database db, List> schemaChangeJobInfos) { List selectedJobs = Lists.newArrayList(); diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index d6b4ef6839d2dc..e97079e4c74ea1 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -223,7 +223,7 @@ public boolean isHadoopLoad() { * "col2": "tmp_col2+1", "col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} */ private void analyzeColumns() throws AnalysisException { - // used to check duplicated column name + // used to check duplicated column name in COLUMNS Set columnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); // merge columns exprs from columns and columnMappingList @@ -232,7 +232,7 @@ private void analyzeColumns() throws AnalysisException { // Step1: analyze columns for (String columnName : columns) { if (!columnNames.add(columnName)) { - throw new AnalysisException("Duplicate column : " + columnName); + throw new AnalysisException("Duplicate column: " + columnName); } ImportColumnDesc importColumnDesc = new ImportColumnDesc(columnName, null); parsedColumnExprList.add(importColumnDesc); @@ -244,6 +244,8 @@ private void analyzeColumns() throws AnalysisException { return; } + // used to check duplicated column name in SET clause + Set columnMappingNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); // Step2: analyze column mapping // the column expr only support the SlotRef or eq binary predicate which's child(0) must be a SloRef. // the duplicate column name of SloRef is forbidden. @@ -263,7 +265,7 @@ private void analyzeColumns() throws AnalysisException { + "The mapping column error. column: " + child0.toSql()); } String column = ((SlotRef) child0).getColumnName(); - if (!columnNames.add(column)) { + if (!columnMappingNames.add(column)) { throw new AnalysisException("Duplicate column mapping: " + column); } // hadoop load only supports the FunctionCallExpr @@ -520,34 +522,41 @@ public void analyzeWithoutCheckPriv() throws AnalysisException { /* * If user does not specify COLUMNS in load stmt, we fill it here. * eg1: - * both COLUMNS and SET clause is empty. after fill: - * (k1,k2,k3) + * both COLUMNS and SET clause is empty. after fill: + * (k1,k2,k3) + * * eg2: - * COLUMNS is empty, SET is not empty - * SET ( k2 = default_value("2") ) - * after fill: - * (k1, __doris_tmp_0, k3) - * SET ( k2 = default_value("2") ) + * COLUMNS is empty, SET is not empty + * SET ( k2 = default_value("2") ) + * after fill: + * (k1, k2, k3) + * SET ( k2 = default_value("2") ) + * + * eg3: + * COLUMNS is empty, SET is not empty + * SET (k2 = strftime("%Y-%m-%d %H:%M:%S", k2) + * after fill: + * (k1,k2,k3) + * SET (k2 = strftime("%Y-%m-%d %H:%M:%S", k2) * - * __doris_tmp_0 is a placeholder, which mean load process should skip that column in source file. */ public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlException { - if (columns == null || columns.isEmpty()) { - columns = Lists.newArrayList(); - Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { - mappingColNames.add(importColumnDesc.getColumnName()); - } + if (columns != null && !columns.isEmpty()) { + return; + } - int placeholder = 0; - for (Column column : baseSchema) { - if (!mappingColNames.contains(column.getName())) { - columns.add(column.getName()); - parsedColumnExprList.add(new ImportColumnDesc(column.getName(), null)); - } else { - columns.add("__doris_tmp_" + (placeholder++)); - } + columns = Lists.newArrayList(); + + Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { + mappingColNames.add(importColumnDesc.getColumnName()); + } + + for (Column column : baseSchema) { + if (!mappingColNames.contains(column.getName())) { + parsedColumnExprList.add(new ImportColumnDesc(column.getName(), null)); } + columns.add(column.getName()); } LOG.debug("after fill column info. columns: {}, parsed column exprs: {}", columns, parsedColumnExprList); diff --git a/fe/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java b/fe/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java index 1bba28f30e2a6c..c7fe7880716bbc 100644 --- a/fe/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java @@ -87,7 +87,6 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { Preconditions.checkNotNull(type); // check auth when get job info - handleShowAlterTable(analyzer); } From 59796d154495c063291acd0d48f3e0585d34ba58 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 13 Aug 2019 14:45:32 +0800 Subject: [PATCH 38/79] abort timeout task --- .../java/org/apache/doris/catalog/OlapTable.java | 4 ++-- .../doris/transaction/GlobalTransactionMgr.java | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 617ee8f13162b7..122da79f88ae49 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1091,11 +1091,11 @@ public long proximateRowCount() { for (Partition partition : getPartitions()) { long version = partition.getVisibleVersion(); long versionHash = partition.getVisibleVersionHash(); - for (MaterializedIndex index : partition.getMaterializedIndices()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : index.getTablets()) { long tabletRowCount = 0L; for (Replica replica : tablet.getReplicas()) { - if (replica.checkVersionCatchUp(version, versionHash) + if (replica.checkVersionCatchUp(version, versionHash, false) && replica.getRowCount() > tabletRowCount) { tabletRowCount = replica.getRowCount(); } diff --git a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index 3747258e545067..c42d58d0320b06 100644 --- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -788,6 +788,7 @@ public void removeOldTransactions() { } // 3. use dbIdToTxnIds to remove old transactions, without holding load locks again + List abortedTxns = Lists.newArrayList(); writeLock(); try { List transactionsToDelete = Lists.newArrayList(); @@ -814,6 +815,7 @@ public void removeOldTransactions() { transactionState.setFinishTime(System.currentTimeMillis()); transactionState.setReason("transaction is timeout and is cancelled automatically"); unprotectUpsertTransactionState(transactionState); + abortedTxns.add(transactionState); } } } @@ -826,10 +828,19 @@ public void removeOldTransactions() { } finally { writeUnlock(); } + + for (TransactionState abortedTxn : abortedTxns) { + try { + abortedTxn.afterStateTransform(TransactionStatus.ABORTED, true, abortedTxn.getReason()); + } catch (UserException e) { + // just print a log, it does not matter. + LOG.warn("after abort timeout txn failed. txn id: {}", abortedTxn.getTransactionId(), e); + } + } } private boolean checkTxnHasRelatedJob(TransactionState txnState, Map> dbIdToTxnIds) { - // TODO: put checkTxnHasRelaredJob into Load + // TODO: put checkTxnHasRelatedJob into Load Set txnIds = dbIdToTxnIds.get(txnState.getDbId()); if (txnIds == null) { // We can't find the related load job of this database. From 1f661735f0e4627455d774f4b2ebd1ad8f6c25a0 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 13 Aug 2019 19:40:42 +0800 Subject: [PATCH 39/79] fix dead log bug --- fe/src/main/java/org/apache/doris/alter/AlterJobV2.java | 8 ++++++-- .../java/org/apache/doris/alter/SchemaChangeJobV2.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index e0534a97053d74..8cc6369b7ff543 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -126,6 +126,10 @@ public boolean isDone() { * run() and cancel() * Only these 2 methods can be visited by different thread(internal working thread and user connection thread) * So using 'synchronized' to make sure only one thread can run the job at one time. + * + * lock order: + * synchronized + * db lock */ public synchronized void run() { if (isTimeout()) { @@ -186,7 +190,7 @@ public static AlterJobV2 read(DataInput in) throws IOException { } @Override - public synchronized void write(DataOutput out) throws IOException { + public void write(DataOutput out) throws IOException { Text.writeString(out, type.name()); Text.writeString(out, jobState.name()); @@ -202,7 +206,7 @@ public synchronized void write(DataOutput out) throws IOException { } @Override - public synchronized void readFields(DataInput in) throws IOException { + public void readFields(DataInput in) throws IOException { // read common members as write in AlterJobV2.write(). // except 'type' member, which is read in AlterJobV2.read() jobState = JobState.valueOf(Text.readString(in)); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 3f1a4169a99d16..0c7596a85eb5e3 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -751,7 +751,7 @@ public List> getUnfinishedTasks(int limit) { } @Override - public synchronized void write(DataOutput out) throws IOException { + public void write(DataOutput out) throws IOException { super.write(out); out.writeInt(partitionIndexTabletMap.rowKeySet().size()); From da5bbe5a7b7e64f38124a56891ed8e4660bf768a Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 14 Aug 2019 14:31:19 +0800 Subject: [PATCH 40/79] fix dead lock --- .../org/apache/doris/alter/RollupHandler.java | 21 ++++--- .../doris/alter/SchemaChangeHandler.java | 55 +++++++++++++------ 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 3be5d2faa1052a..136e95db1ae188 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -673,14 +673,12 @@ public void cancel(CancelStmt stmt) throws DdlException { // find from new alter jobs first rollupJobV2 = getAlterJobV2(olapTable.getId()); - if (rollupJobV2 != null) { - if (!rollupJobV2.cancel("user cancelled")) { - throw new DdlException("Job can not be cancelled. State: " + rollupJobV2.getJobState()); - } - } else { + if (rollupJobV2 == null) { rollupJob = getAlterJob(olapTable.getId()); Preconditions.checkNotNull(rollupJob, olapTable.getId()); - if (rollupJob.getState() == JobState.FINISHED || rollupJob.getState() == JobState.CANCELLED) { + if (rollupJob.getState() == JobState.FINISHED + || rollupJob.getState() == JobState.FINISHING + || rollupJob.getState() == JobState.CANCELLED) { throw new DdlException("job is already " + rollupJob.getState().name() + ", can not cancel it"); } rollupJob.cancel(olapTable, "user cancelled"); @@ -689,7 +687,16 @@ public void cancel(CancelStmt stmt) throws DdlException { db.writeUnlock(); } - if (rollupJob != null) { + // alter job v2's cancel must be called outside the database lock + if (rollupJobV2 != null) { + if (!rollupJobV2.cancel("user cancelled")) { + throw new DdlException("Job can not be cancelled. State: " + rollupJobV2.getJobState()); + } + return; + } + + // handle old alter job + if (rollupJob != null && rollupJob.getState() == JobState.CANCELLED) { jobDone(rollupJob); } } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 13c9da3b55c722..fdf672f3cedeae 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -46,11 +46,14 @@ import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.common.util.ListComparator; @@ -1340,33 +1343,49 @@ public void cancel(CancelStmt stmt) throws DdlException { throw new DdlException("Database[" + dbName + "] does not exist"); } - AlterJob alterJob = null; + AlterJob schemaChangeJob = null; + AlterJobV2 schemaChangeJobV2 = null; db.writeLock(); try { - // 1. get table - OlapTable olapTable = (OlapTable) db.getTable(tableName); - if (olapTable == null) { - throw new DdlException("Table[" + tableName + "] does not exist"); + Table table = db.getTable(tableName); + if (table == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName); } - - // 2. find schema change job - alterJob = alterJobs.get(olapTable.getId()); - if (alterJob == null) { - throw new DdlException("Table[" + tableName + "] is not under SCHEMA CHANGE"); + if (!(table instanceof OlapTable)) { + ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, tableName); } - - if (alterJob.getState() == JobState.FINISHING || - alterJob.getState() == JobState.FINISHED || - alterJob.getState() == JobState.CANCELLED) { - throw new DdlException("job is already " + alterJob.getState().name() + ", can not cancel it"); + OlapTable olapTable = (OlapTable) table; + if (olapTable.getState() != OlapTableState.SCHEMA_CHANGE) { + throw new DdlException("Table[" + tableName + "] is not under SCHEMA_CHANGE."); } - // 3. cancel schema change job - alterJob.cancel(olapTable, "user cancelled"); + // find from new alter jobs first + schemaChangeJobV2 = getAlterJobV2(olapTable.getId()); + if (schemaChangeJobV2 == null) { + schemaChangeJob = getAlterJob(olapTable.getId()); + Preconditions.checkNotNull(schemaChangeJob, olapTable.getId()); + if (schemaChangeJob.getState() == JobState.FINISHING + || schemaChangeJob.getState() == JobState.FINISHED + || schemaChangeJob.getState() == JobState.CANCELLED) { + throw new DdlException("job is already " + schemaChangeJob.getState().name() + ", can not cancel it"); + } + schemaChangeJob.cancel(olapTable, "user cancelled"); + } } finally { db.writeUnlock(); } - jobDone(alterJob); + // alter job v2's cancel must be called outside the database lock + if (schemaChangeJobV2 != null) { + if (!schemaChangeJobV2.cancel("user cancelled")) { + throw new DdlException("Job can not be cancelled. State: " + schemaChangeJobV2.getJobState()); + } + return; + } + + // handle old alter job + if (schemaChangeJob != null && schemaChangeJob.getState() == JobState.CANCELLED) { + jobDone(schemaChangeJob); + } } } From fc69416be88ed0486d57269b280952b28b496404 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Aug 2019 09:21:53 +0800 Subject: [PATCH 41/79] Save table and index in transaction state --- .../Data Manipulation/ROUTINE LOAD.md | 1 + .../org/apache/doris/alter/AlterHandler.java | 6 ++-- .../org/apache/doris/alter/RollupHandler.java | 2 +- .../doris/alter/SchemaChangeHandler.java | 2 +- .../org/apache/doris/analysis/InsertStmt.java | 8 +++++ .../doris/common/proc/RollupProcDir.java | 6 ++-- .../doris/load/loadv2/BrokerLoadJob.java | 16 +++++++--- .../doris/load/routineload/KafkaTaskInfo.java | 4 +-- .../load/routineload/RoutineLoadJob.java | 14 +++++++-- .../doris/planner/StreamLoadPlanner.java | 4 +++ .../org/apache/doris/qe/ShowExecutor.java | 2 +- .../doris/service/FrontendServiceImpl.java | 9 +++++- .../transaction/GlobalTransactionMgr.java | 31 +++++++++++++++++-- .../doris/transaction/TransactionState.java | 23 ++++++++++++++ 14 files changed, 107 insertions(+), 21 deletions(-) diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md index f618e824ad9b9d..20315ad1080983 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/ROUTINE LOAD.md @@ -258,5 +258,6 @@ ); ## keyword + CREATE,ROUTINE,LOAD diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 748a5022d81bda..23baa70a7f8a2d 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -90,9 +90,11 @@ protected void addAlterJobV2(AlterJobV2 alterJob) { LOG.info("add {} job {}", alterJob.getType(), alterJob.getJobId()); } - public AlterJobV2 getAlterJobV2(long tblId) { + public AlterJobV2 getUnfinishedAlterJobV2(long tblId) { for (AlterJobV2 alterJob : alterJobsV2.values()) { - if (alterJob.getTableId() == tblId && alterJob.getJobState() == AlterJobV2.JobState.RUNNING) { + if (alterJob.getTableId() == tblId + && alterJob.getJobState() != AlterJobV2.JobState.FINISHED + && alterJob.getJobState() != AlterJobV2.JobState.CANCELLED) { return alterJob; } } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 136e95db1ae188..1bdfdc56b59997 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -672,7 +672,7 @@ public void cancel(CancelStmt stmt) throws DdlException { } // find from new alter jobs first - rollupJobV2 = getAlterJobV2(olapTable.getId()); + rollupJobV2 = getUnfinishedAlterJobV2(olapTable.getId()); if (rollupJobV2 == null) { rollupJob = getAlterJob(olapTable.getId()); Preconditions.checkNotNull(rollupJob, olapTable.getId()); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index fdf672f3cedeae..a606514af76f1f 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1360,7 +1360,7 @@ public void cancel(CancelStmt stmt) throws DdlException { } // find from new alter jobs first - schemaChangeJobV2 = getAlterJobV2(olapTable.getId()); + schemaChangeJobV2 = getUnfinishedAlterJobV2(olapTable.getId()); if (schemaChangeJobV2 == null) { schemaChangeJob = getAlterJob(olapTable.getId()); Preconditions.checkNotNull(schemaChangeJob, olapTable.getId()); diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java index 221deae2c23cdf..9f3e4f0b31ae3a 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -31,6 +31,7 @@ import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -43,6 +44,7 @@ import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TUniqueId; +import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionState.LoadJobSourceType; import com.google.common.base.Joiner; @@ -702,6 +704,12 @@ public DataSink createDataSink() throws AnalysisException { public void finalize() throws UserException { if (targetTable instanceof OlapTable) { ((OlapTableSink) dataSink).finalize(); + // add table indexes to transaction state + TransactionState txnState = Catalog.getCurrentGlobalTransactionMgr().getTransactionState(transactionId); + if (txnState == null) { + throw new DdlException("txn does not exist: " + transactionId); + } + txnState.addTableIndexes((OlapTable) targetTable); } } diff --git a/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java index c290f55f0010ca..520939e1b2d828 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/RollupProcDir.java @@ -72,18 +72,18 @@ public boolean register(String name, ProcNodeInterface node) { @Override public ProcNodeInterface lookup(String jobIdStr) throws AnalysisException { if (Strings.isNullOrEmpty(jobIdStr)) { - throw new AnalysisException("Table id is null"); + throw new AnalysisException("Job id is null"); } long jobId = -1L; try { jobId = Long.valueOf(jobIdStr); } catch (Exception e) { - throw new AnalysisException("Table id is invalid"); + throw new AnalysisException("Job id is invalid"); } Preconditions.checkState(jobId != -1L); - AlterJobV2 job = rollupHandler.getAlterJobV2(jobId); + AlterJobV2 job = rollupHandler.getUnfinishedAlterJobV2(jobId); if (job == null) { return null; } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index e627038e88bf12..c75dc9155e54e3 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -360,12 +360,20 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme strictMode, transactionId, this); UUID uuid = UUID.randomUUID(); TUniqueId loadId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); - task.init(loadId, attachment.getFileStatusByTable(tableId), - attachment.getFileNumByTable(tableId)); - // Add tasks into list and pool + task.init(loadId, attachment.getFileStatusByTable(tableId), attachment.getFileNumByTable(tableId)); idToTasks.put(task.getSignature(), task); loadStatistic.numLoadedRowsMap.put(loadId, new AtomicLong(0)); - Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(task); + + // save all related tables and rollups in transaction state + TransactionState txnState = Catalog.getCurrentGlobalTransactionMgr().getTransactionState(transactionId); + if (txnState == null) { + throw new UserException("txn does not exist: " + transactionId); + } + txnState.addTableIndexes(table); + } + // submit all tasks together + for (LoadTask loadTask : idToTasks.values()) { + Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); } } finally { db.readUnlock(); diff --git a/fe/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java b/fe/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java index 1d35f9b1066f89..9b4c0a90ab2b12 100644 --- a/fe/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java +++ b/fe/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java @@ -101,9 +101,9 @@ protected String getTaskDataSourceProperties() { private TExecPlanFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws UserException { TUniqueId loadId = new TUniqueId(id.getMostSignificantBits(), id.getLeastSignificantBits()); // plan for each task, in case table has change(rollup or schema change) - TExecPlanFragmentParams tExecPlanFragmentParams = routineLoadJob.plan(loadId); + TExecPlanFragmentParams tExecPlanFragmentParams = routineLoadJob.plan(loadId, txnId); TPlanFragment tPlanFragment = tExecPlanFragmentParams.getFragment(); - tPlanFragment.getOutput_sink().getOlap_table_sink().setTxn_id(this.txnId); + tPlanFragment.getOutput_sink().getOlap_table_sink().setTxn_id(txnId); return tExecPlanFragmentParams; } } diff --git a/fe/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 801f63156ac4e8..29d4c6ed1fa4a6 100644 --- a/fe/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -196,7 +196,7 @@ public boolean isFinalState() { protected List routineLoadTaskInfoList = Lists.newArrayList(); // stream load planer will be initialized during job schedule - StreamLoadPlanner planner; + protected StreamLoadPlanner planner; // this is the origin stmt of CreateRoutineLoadStmt, we use it to persist the RoutineLoadJob, // because we can not serialize the Expressions contained in job. @@ -574,7 +574,7 @@ private void initPlanner() throws UserException { planner = new StreamLoadPlanner(db, (OlapTable) db.getTable(this.tableId), streamLoadTask); } - public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { + public TExecPlanFragmentParams plan(TUniqueId loadId, long txnId) throws UserException { Preconditions.checkNotNull(planner); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { @@ -582,7 +582,15 @@ public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { } db.readLock(); try { - return planner.plan(loadId); + TExecPlanFragmentParams planParams = planner.plan(loadId); + // add table indexes to transaction state + TransactionState txnState = Catalog.getCurrentGlobalTransactionMgr().getTransactionState(txnId); + if (txnState == null) { + throw new MetaNotFoundException("txn does not exist: " + txnId); + } + txnState.addTableIndexes(planner.getDestTable()); + + return planParams; } finally { db.readUnlock(); } diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index e38d660d62714d..8d5dfa5cb9082d 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -80,6 +80,10 @@ public StreamLoadPlanner(Database db, OlapTable destTable, StreamLoadTask stream descTable = analyzer.getDescTbl(); } + public OlapTable getDestTable() { + return destTable; + } + // create the plan. the plan's query id and load id are same, using the parameter 'loadId' public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { // construct tuple descriptor, used for scanNode and dataSink diff --git a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java index 365b9b076b7b0c..3ecd5e71e3ec10 100644 --- a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -932,7 +932,7 @@ private void handleShowRoutineLoad() throws AnalysisException { // if the jobName has been specified throw new AnalysisException("There is no job named " + showRoutineLoadStmt.getName() + " in db " + showRoutineLoadStmt.getDbFullName() - + " include history " + showRoutineLoadStmt.isIncludeHistory()); + + ". Include history? " + showRoutineLoadStmt.isIncludeHistory()); } resultSet = new ShowResultSet(showRoutineLoadStmt.getMetaData(), rows); } diff --git a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 25c8297e3ebc64..7a82187bda4769 100644 --- a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -791,7 +791,7 @@ private TExecPlanFragmentParams streamLoadPutImpl(TStreamLoadPutRequest request) cluster = SystemInfoService.DEFAULT_CLUSTER; } - Catalog catalog = Catalog.getInstance(); + Catalog catalog = Catalog.getCurrentCatalog(); String fullDbName = ClusterNamespace.getFullName(cluster, request.getDb()); Database db = catalog.getDb(fullDbName); if (db == null) { @@ -814,6 +814,13 @@ private TExecPlanFragmentParams streamLoadPutImpl(TStreamLoadPutRequest request) StreamLoadTask streamLoadTask = StreamLoadTask.fromTStreamLoadPutRequest(request); StreamLoadPlanner planner = new StreamLoadPlanner(db, (OlapTable) table, streamLoadTask); TExecPlanFragmentParams plan = planner.plan(streamLoadTask.getId()); + // add table indexes to transaction state + TransactionState txnState = Catalog.getCurrentGlobalTransactionMgr().getTransactionState(request.getTxnId()); + if (txnState == null) { + throw new UserException("txn does not exist: " + request.getTxnId()); + } + txnState.addTableIndexes((OlapTable) table); + return plan; } finally { db.readUnlock(); diff --git a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index c42d58d0320b06..49c3ae38da4707 100644 --- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -328,7 +328,20 @@ public void commitTransaction(long dbId, long transactionId, List allIndices = partition.getMaterializedIndices(IndexExtState.ALL); + + List allIndices = null; + if (transactionState.getLoadedTblIndexes().isEmpty()) { + allIndices = partition.getMaterializedIndices(IndexExtState.ALL); + } else { + allIndices = Lists.newArrayList(); + for (long indexId : transactionState.getLoadedTblIndexes().get(tableId)) { + MaterializedIndex index = partition.getIndex(indexId); + if (index != null) { + allIndices.add(index); + } + } + } + if (table.getState() == OlapTableState.ROLLUP || table.getState() == OlapTableState.SCHEMA_CHANGE) { /* * This is just a optimization that do our best to not let publish version tasks @@ -640,8 +653,20 @@ public void finishTransaction(long transactionId, Set errorReplicaIds) thr } int quorumReplicaNum = partitionInfo.getReplicationNum(partitionId) / 2 + 1; - List allInices = partition.getMaterializedIndices(IndexExtState.ALL); - for (MaterializedIndex index : allInices) { + List allIndices = null; + if (transactionState.getLoadedTblIndexes().isEmpty()) { + allIndices = partition.getMaterializedIndices(IndexExtState.ALL); + } else { + allIndices = Lists.newArrayList(); + for (long indexId : transactionState.getLoadedTblIndexes().get(tableId)) { + MaterializedIndex index = partition.getIndex(indexId); + if (index != null) { + allIndices.add(index); + } + } + } + + for (MaterializedIndex index : allIndices) { for (Tablet tablet : index.getTablets()) { int healthReplicaNum = 0; for (Replica replica : tablet.getReplicas()) { diff --git a/fe/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/src/main/java/org/apache/doris/transaction/TransactionState.java index b44a81a1142f90..c1f2f1d768f934 100644 --- a/fe/src/main/java/org/apache/doris/transaction/TransactionState.java +++ b/fe/src/main/java/org/apache/doris/transaction/TransactionState.java @@ -18,6 +18,7 @@ package org.apache.doris.transaction; import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Config; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; @@ -141,6 +142,11 @@ public String toString() { // optional private TxnCommitAttachment txnCommitAttachment; + // this map should be set when load execution begin, so that when the txn commit, it will know + // which tables and rollups it loaded. + // tbl id -> (index ids) + private Map> loadedTblIndexes = Maps.newHashMap(); + private String errorLogUrl = null; public TransactionState() { @@ -408,6 +414,23 @@ public void setTxnCommitAttachment(TxnCommitAttachment txnCommitAttachment) { this.txnCommitAttachment = txnCommitAttachment; } + public void addTableIndexes(OlapTable table) { + Set indexIds = loadedTblIndexes.get(table.getId()); + if (indexIds == null) { + indexIds = Sets.newHashSet(); + loadedTblIndexes.put(table.getId(), indexIds); + } + // always rewrite the index ids + indexIds.clear(); + for (Long indexId : table.getIndexIdToSchema().keySet()) { + indexIds.add(indexId); + } + } + + public Map> getLoadedTblIndexes() { + return loadedTblIndexes; + } + @Override public String toString() { StringBuilder sb = new StringBuilder("TransactionState. "); From 245e554697d5d1bceb7c12a5061504b5cd8b0b10 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Aug 2019 17:11:04 +0800 Subject: [PATCH 42/79] fix replace_value --- .../doris/analysis/DataDescription.java | 7 +-- .../apache/doris/planner/BrokerScanNode.java | 56 +++++++++++++++---- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index e97079e4c74ea1..c5cf56b1697419 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -291,6 +291,7 @@ private void analyzeColumnToHadoopFunction(String columnName, Expr child1) throw } List paramExprs = functionCallExpr.getParams().exprs(); List args = Lists.newArrayList(); + for (Expr paramExpr : paramExprs) { if (paramExpr instanceof SlotRef) { SlotRef slot = (SlotRef) paramExpr; @@ -301,10 +302,8 @@ private void analyzeColumnToHadoopFunction(String columnName, Expr child1) throw } else if (paramExpr instanceof NullLiteral) { args.add(null); } else { - if (isHadoopLoad) { - throw new AnalysisException("Mapping function args error, arg: " + paramExpr.toSql()); - } - continue; + // hadoop function only support slot, string and null parameters + throw new AnalysisException("Mapping function args error, arg: " + paramExpr.toSql()); } } diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index b35cf16406935c..759c4ff580f546 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -29,6 +29,7 @@ import org.apache.doris.analysis.FunctionParams; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; @@ -337,22 +338,57 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr SlotRef slotRef = new SlotRef(null, columnName); // We will convert this to IF(`col` != child0, `col`, child1), // because we need the if return type equal to `col`, we use NE - // - exprs.add(new BinaryPredicate(BinaryPredicate.Operator.NE, slotRef, funcExpr.getChild(0))); - exprs.add(slotRef); - if (funcExpr.hasChild(1)) { - exprs.add(funcExpr.getChild(1)); + + /* + * We will convert this based on different cases: + * case 1: k1 = replace_value(null, anyval); + * to: k1 = if (k1 is not null, k1, anyval); + * + * case 2: k1 = replace_value(anyval1, anyval2); + * to: k1 = if (k1 is not null, if(k1 != anyval1, k1, anyval2), null); + */ + if (funcExpr.getChild(0) instanceof NullLiteral) { + // case 1 + exprs.add(new IsNullPredicate(slotRef, true)); + exprs.add(slotRef); + if (funcExpr.hasChild(1)) { + exprs.add(funcExpr.getChild(1)); + } else { + if (column.getDefaultValue() != null) { + exprs.add(new StringLiteral(column.getDefaultValue())); + } else { + if (column.isAllowNull()) { + exprs.add(NullLiteral.create(Type.VARCHAR)); + } else { + throw new UserException("Column(" + columnName + ") has no default value."); + } + } + } } else { - if (column.getDefaultValue() != null) { - exprs.add(new StringLiteral(column.getDefaultValue())); + // case 2 + exprs.add(new IsNullPredicate(slotRef, true)); + List innerIfExprs = Lists.newArrayList(); + innerIfExprs.add(new BinaryPredicate(BinaryPredicate.Operator.NE, slotRef, funcExpr.getChild(0))); + innerIfExprs.add(slotRef); + if (funcExpr.hasChild(1)) { + innerIfExprs.add(funcExpr.getChild(1)); } else { - if (column.isAllowNull()) { - exprs.add(NullLiteral.create(Type.VARCHAR)); + if (column.getDefaultValue() != null) { + innerIfExprs.add(new StringLiteral(column.getDefaultValue())); } else { - throw new UserException("Column(" + columnName + ") has no default value."); + if (column.isAllowNull()) { + innerIfExprs.add(NullLiteral.create(Type.VARCHAR)); + } else { + throw new UserException("Column(" + columnName + ") has no default value."); + } } } + FunctionCallExpr innerIfFn = new FunctionCallExpr("if", innerIfExprs); + exprs.add(innerIfFn); + exprs.add(NullLiteral.create(Type.VARCHAR)); } + + LOG.debug("replace_value expr: {}", exprs); FunctionCallExpr newFn = new FunctionCallExpr("if", exprs); return newFn; } else if (funcName.equalsIgnoreCase("strftime")) { From b26bf6a9c14f658e2834a35fa59506f983daa6a7 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 16 Aug 2019 22:20:53 +0800 Subject: [PATCH 43/79] copy the column expr descs in routine load job --- fe/src/main/java/org/apache/doris/task/StreamLoadTask.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index 3e9373d1edc549..362c1a1e115f38 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -170,7 +170,9 @@ public static StreamLoadTask fromRoutineLoadJob(RoutineLoadJob routineLoadJob) { } private void setOptionalFromRoutineLoadJob(RoutineLoadJob routineLoadJob) { - columnExprDescs = routineLoadJob.getColumnDescs(); + // copy the columnExprDescs, cause it may be changed when planning. + // so we keep the columnExprDescs in routine load job as origin. + columnExprDescs = Lists.newArrayList(routineLoadJob.getColumnDescs()); whereExpr = routineLoadJob.getWhereExpr(); columnSeparator = routineLoadJob.getColumnSeparator(); partitions = routineLoadJob.getPartitions() == null ? null : Joiner.on(",").join(routineLoadJob.getPartitions()); From 89c9325b12d6c9019c7acdccaa4bb34016d29e21 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 17 Aug 2019 11:17:41 +0800 Subject: [PATCH 44/79] fix stream load task null pointer exception --- .../main/java/org/apache/doris/planner/BrokerScanNode.java | 5 +++++ fe/src/main/java/org/apache/doris/task/StreamLoadTask.java | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 759c4ff580f546..235839e96ae50d 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -411,6 +411,11 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr return dateFormatFunc; } else if (funcName.equalsIgnoreCase("alignment_timestamp")) { + /* + * change to: + * UNIX_TIMESTAMP(DATE_FORMAT(FROM_UNIXTIME(value), "%Y-01-01 00:00:00")) + */ + FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); FunctionCallExpr fromUnixFunc = new FunctionCallExpr( diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index 362c1a1e115f38..999cddb4afbf49 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -172,7 +172,11 @@ public static StreamLoadTask fromRoutineLoadJob(RoutineLoadJob routineLoadJob) { private void setOptionalFromRoutineLoadJob(RoutineLoadJob routineLoadJob) { // copy the columnExprDescs, cause it may be changed when planning. // so we keep the columnExprDescs in routine load job as origin. - columnExprDescs = Lists.newArrayList(routineLoadJob.getColumnDescs()); + if (routineLoadJob.getColumnDescs() != null) { + columnExprDescs = Lists.newArrayList(routineLoadJob.getColumnDescs()); + } else { + columnExprDescs = null; + } whereExpr = routineLoadJob.getWhereExpr(); columnSeparator = routineLoadJob.getColumnSeparator(); partitions = routineLoadJob.getPartitions() == null ? null : Joiner.on(",").join(routineLoadJob.getPartitions()); From 221b9f24393fe2dca2b7f57ccb0834e371322296 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 17 Aug 2019 20:33:37 +0800 Subject: [PATCH 45/79] Fix bug 1. LoadChecker submittask NPE 2. backup job should not backup invisible index 3. cancel all old alter job --- .../org/apache/doris/alter/RollupHandler.java | 8 ++++++++ .../apache/doris/alter/SchemaChangeHandler.java | 7 +++++++ .../org/apache/doris/backup/BackupHandler.java | 3 ++- .../java/org/apache/doris/backup/BackupJob.java | 3 ++- .../org/apache/doris/backup/RestoreJob.java | 2 +- .../java/org/apache/doris/catalog/Catalog.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 4 ++-- .../java/org/apache/doris/load/LoadChecker.java | 17 +++++++++++------ 8 files changed, 34 insertions(+), 12 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 1bdfdc56b59997..03f04d86d663f9 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -459,6 +459,14 @@ private void runOldAlterJob() { for (AlterJob alterJob : alterJobs.values()) { RollupJob rollupJob = (RollupJob) alterJob; + if (rollupJob.getState() != JobState.FINISHING + && rollupJob.getState() != JobState.FINISHED + && rollupJob.getState() != JobState.CANCELLED) { + // cancel the old alter table job + cancelledJobs.add(rollupJob); + continue; + } + if (rollupJob.getTransactionId() < 0) { // it means this is an old type job and current version is real time load version // then kill this job diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index a606514af76f1f..35b16182c5b434 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1108,6 +1108,13 @@ private void runOldAlterJob() { for (AlterJob alterJob : alterJobs.values()) { SchemaChangeJob schemaChangeJob = (SchemaChangeJob) alterJob; + if (schemaChangeJob.getState() != JobState.FINISHING + && schemaChangeJob.getState() != JobState.FINISHED + && schemaChangeJob.getState() != JobState.CANCELLED) { + // cancel the old alter table job + cancelledJobs.add(schemaChangeJob); + continue; + } // it means this is an old type job and current version is real time load version // then kill this job if (alterJob.getTransactionId() < 0) { diff --git a/fe/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/src/main/java/org/apache/doris/backup/BackupHandler.java index 9014d6c4782d9b..24d8a0f0896d5d 100644 --- a/fe/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -30,6 +30,7 @@ import org.apache.doris.backup.BackupJobInfo.BackupTableInfo; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; @@ -298,7 +299,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws } // copy a table with selected partitions for calculating the signature - OlapTable copiedTbl = olapTbl.selectiveCopy(tblRef.getPartitions(), true); + OlapTable copiedTbl = olapTbl.selectiveCopy(tblRef.getPartitions(), true, IndexExtState.VISIBLE); if (copiedTbl == null) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Failed to copy table " + tblName + " with selected partitions"); diff --git a/fe/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/src/main/java/org/apache/doris/backup/BackupJob.java index a11b91f53b5ba9..9c411a81e27162 100644 --- a/fe/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/src/main/java/org/apache/doris/backup/BackupJob.java @@ -420,7 +420,8 @@ private void prepareAndSendSnapshotTask() { for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); OlapTable tbl = (OlapTable) db.getTable(tblName); - OlapTable copiedTbl = tbl.selectiveCopy(tableRef.getPartitions(), true); + // only copy visible indexes + OlapTable copiedTbl = tbl.selectiveCopy(tableRef.getPartitions(), true, IndexExtState.VISIBLE); if (copiedTbl == null) { status = new Status(ErrCode.COMMON_ERROR, "faild to copy table: " + tblName); return; diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java index a0ef11a00b275b..0910ded5d72447 100644 --- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -875,7 +875,7 @@ private void genFileMapping(OlapTable localTbl, Partition localPartition, Long r BackupPartitionInfo backupPartInfo, boolean overwrite) { for (MaterializedIndex localIdx : localPartition.getMaterializedIndices(IndexExtState.VISIBLE)) { LOG.debug("get index id: {}, index name: {}", localIdx.getId(), - localTbl.getIndexNameById(localIdx.getId())); + localTbl.getIndexNameById(localIdx.getId())); BackupIndexInfo backupIdxInfo = backupPartInfo.getIdx(localTbl.getIndexNameById(localIdx.getId())); Preconditions.checkState(backupIdxInfo.tablets.size() == localIdx.getTablets().size()); for (int i = 0; i < localIdx.getTablets().size(); i++) { diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 424aa62ccd1f6e..62f6f27ec1cbc0 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -5939,7 +5939,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti } } - copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), true); + copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), true, IndexExtState.VISIBLE); } finally { db.readUnlock(); diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 122da79f88ae49..accf4b540e1dac 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -958,7 +958,7 @@ public boolean equals(Table table) { return true; } - public OlapTable selectiveCopy(Collection reservedPartNames, boolean resetState) { + public OlapTable selectiveCopy(Collection reservedPartNames, boolean resetState, IndexExtState extState) { OlapTable copied = new OlapTable(); if (!DeepCopy.copy(this, copied)) { LOG.warn("failed to copy olap table: " + getName()); @@ -970,7 +970,7 @@ public OlapTable selectiveCopy(Collection reservedPartNames, boolean res for (Partition partition : copied.getPartitions()) { partition.setState(PartitionState.NORMAL); copied.getPartitionInfo().setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD)); - for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.ALL)) { + for (MaterializedIndex idx : partition.getMaterializedIndices(extState)) { idx.setState(IndexState.NORMAL); for (Tablet tablet : idx.getTablets()) { for (Replica replica : tablet.getReplicas()) { diff --git a/fe/src/main/java/org/apache/doris/load/LoadChecker.java b/fe/src/main/java/org/apache/doris/load/LoadChecker.java index 4950246eb0be06..5f9aafdf99ef71 100644 --- a/fe/src/main/java/org/apache/doris/load/LoadChecker.java +++ b/fe/src/main/java/org/apache/doris/load/LoadChecker.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; -import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Replica; @@ -399,16 +398,22 @@ private Set submitPushTasks(LoadJob job, Database db) { List indices = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : indices) { long indexId = index.getId(); - // if index is in rollup, then not load into it, be will automatically convert the data - if (index.getState() == IndexState.ROLLUP) { - LOG.error("skip table under rollup[{}]", indexId); - continue; - } + // 1. the load job's etl is started before rollup finished // 2. rollup job comes into finishing state, add rollup index to catalog // 3. load job's etl finished, begin to load // 4. load will send data to new rollup index, but could not get schema hash, load will failed + /* + * new: + * 1. load job is started before alter table, and etl task does not contains new indexes + * 2. just send push tasks to indexes which it contains, ignore others + */ if (!tableLoadInfo.containsIndex(indexId)) { + if (rollupJob == null) { + // new process, just continue + continue; + } + if (rollupJob.getRollupIndexId() == indexId) { continue; } else { From 09f4e29c43d502dce6b1a5e7ba7bc0f1426466a6 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 18 Aug 2019 22:53:25 +0800 Subject: [PATCH 46/79] Use "Africa/Abidjan" as default pull load job time zone --- fe/src/main/java/org/apache/doris/qe/Coordinator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index 53c7e2deebba78..954096a65bc23b 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -226,7 +226,8 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, this.queryOptions = new TQueryOptions(); this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); this.queryGlobals.setTimestamp_ms(new Date().getTime()); - this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); + // this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); + this.queryGlobals.setTime_zone("Africa/Abidjan"); this.tResourceInfo = new TResourceInfo("", ""); this.needReport = true; this.clusterName = cluster; From 6c14e3aa5042e391445ce2c04bae8a47a752a006 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 19 Aug 2019 09:43:07 +0800 Subject: [PATCH 47/79] fix timezone bug --- be/src/exprs/timestamp_functions.cpp | 4 ++++ .../cn/administrator-guide/time-zone.md | 14 +++++++++++++- .../org/apache/doris/planner/BrokerScanNode.java | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index 2b62c3258a8d67..115f1544fe0a65 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -391,6 +391,7 @@ DateTimeVal TimestampFunctions::timestamp( return val; } +// FROM_UNIXTIME() StringVal TimestampFunctions::from_unix( FunctionContext* context, const IntVal& unix_time) { if (unix_time.is_null) { @@ -405,6 +406,7 @@ StringVal TimestampFunctions::from_unix( return AnyValUtil::from_string_temp(context, buf); } +// FROM_UNIXTIME() StringVal TimestampFunctions::from_unix( FunctionContext* context, const IntVal& unix_time, const StringVal& fmt) { if (unix_time.is_null || fmt.is_null) { @@ -422,10 +424,12 @@ StringVal TimestampFunctions::from_unix( return AnyValUtil::from_string_temp(context, buf); } +// UNIX_TIMESTAMP() IntVal TimestampFunctions::to_unix(FunctionContext* context) { return IntVal(context->impl()->state()->timestamp_ms() / 1000); } +// UNIX_TIMESTAMP() IntVal TimestampFunctions::to_unix( FunctionContext* context, const StringVal& string_val, const StringVal& fmt) { if (string_val.is_null || fmt.is_null) { diff --git a/docs/documentation/cn/administrator-guide/time-zone.md b/docs/documentation/cn/administrator-guide/time-zone.md index a4e42b0365bbea..2a6d727a96441c 100644 --- a/docs/documentation/cn/administrator-guide/time-zone.md +++ b/docs/documentation/cn/administrator-guide/time-zone.md @@ -37,7 +37,15 @@ Doris 内部存在多个时区相关参数 包括NOW()或CURTIME()等时间函数显示的值，也包括show load, show backends中的时间值。 -但不会影响create table 中时间类型分区列的less than值，也不会影响存储为date/datetime类型的值的显示。 +但不会影响 create table 中时间类型分区列的 less than 值，也不会影响存储为 date/datetime 类型的值的显示。 + +受时区影响的函数： + +* `FROM_UNIXTIME`：给定一个 UTC 时间戳，返回指定时区的日期时间：如 `FROM_UNIXTIME(0)`， 返回 CST 时区：`1970-01-01 08:00:00`。 +* `UNIX_TIMESTAMP`：给定一个指定时区日期时间，返回 UTC 时间戳：如 CST 时区 `UNIX_TIMESTAMP('1970-01-01 08:00:00')`，返回 `0`。 +* `CURTIME`：返回指定时区时间。 +* `NOW`：返指定地时区日期时间。 +* `CONVERT_TZ`：将一个日期时间从一个指定时区转换到另一个指定时区。 ## 使用限制 @@ -51,3 +59,7 @@ Doris 内部存在多个时区相关参数 * 为了兼容Doris，支持CST缩写时区，内部会将CST转移为"Asia/Shanghai"的中国标准时区 +## 时区格式列表 + +[List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 235839e96ae50d..079be11eee6539 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -392,6 +392,7 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr FunctionCallExpr newFn = new FunctionCallExpr("if", exprs); return newFn; } else if (funcName.equalsIgnoreCase("strftime")) { + // FROM_UNIXTIME(val) FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); FunctionCallExpr fromUnixFunc = new FunctionCallExpr( @@ -399,6 +400,7 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr return fromUnixFunc; } else if (funcName.equalsIgnoreCase("time_format")) { + // DATE_FORMAT(STR_TO_DATE(dt_str, dt_fmt)) FunctionName strToDateName = new FunctionName("STR_TO_DATE"); List strToDateExprs = Lists.newArrayList(funcExpr.getChild(2), funcExpr.getChild(1)); FunctionCallExpr strToDateFuncExpr = new FunctionCallExpr( From b3cec36df60cb576a8a86a4ed5c0f2c2b4a899dd Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 19 Aug 2019 10:44:20 +0800 Subject: [PATCH 48/79] add timezone to load stmt --- .../Data Manipulation/BROKER LOAD.md | 3 ++- .../java/org/apache/doris/analysis/Analyzer.java | 5 +++++ .../java/org/apache/doris/analysis/LoadStmt.java | 1 + .../org/apache/doris/common/util/TimeUtils.java | 14 +++++++------- .../main/java/org/apache/doris/load/ExportJob.java | 3 ++- .../apache/doris/load/loadv2/BrokerLoadJob.java | 4 ++-- .../java/org/apache/doris/load/loadv2/LoadJob.java | 12 +++++++++++- .../apache/doris/load/loadv2/LoadLoadingTask.java | 8 +++++--- .../doris/load/loadv2/LoadingTaskPlanner.java | 7 ++++++- .../org/apache/doris/planner/BrokerScanNode.java | 12 +++++++++--- .../main/java/org/apache/doris/qe/Coordinator.java | 5 ++--- .../java/org/apache/doris/task/PullLoadTask.java | 3 ++- 12 files changed, 54 insertions(+), 23 deletions(-) diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md index 977320a0d03c4a..4063c28c537c0e 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md @@ -133,7 +133,8 @@ max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 exec_mem_limit: 设置导入使用的内存上限。默认为2G，单位字节。这里是指单个 BE 节点的内存上限。 一个导入可能分布于多个BE。我们假设 1GB 数据在单个节点处理需要最大5GB内存。那么假设1GB文件分布在2个节点处理，那么理论上，每个节点需要内存为2.5GB。则该参数可以设置为 2684354560，即2.5GB - strict mode： 是否对数据进行严格限制。默认为true。 + strict mode： 是否对数据进行严格限制。默认为true。 + timezone: 指定某些受时区影响的函数的时区，如 strftime/alignment_timestamp/from_unixtime 等等，具体请查阅 [时区] 文档。如果不指定，则使用 "Asia/Shanghai" 时区。 5. 导入数据格式样例 diff --git a/fe/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/src/main/java/org/apache/doris/analysis/Analyzer.java index 5d3ec0d05a4894..2b3bae98e8bed0 100644 --- a/fe/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -32,6 +32,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.IdGenerator; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.planner.PlanNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.BetweenToCompoundRule; @@ -138,6 +139,8 @@ public class Analyzer { private TupleId visibleSemiJoinedTupleId_ = null; // for some situation that udf is not allowed. private boolean isUDFAllowed = true; + // timezone specified for some operation, such as broker load + private String timezone = TimeUtils.DEFAULT_TIME_ZONE; public void setIsSubquery() { isSubquery = true; @@ -150,6 +153,8 @@ public void setIsSubquery() { public void setUDFAllowed(boolean val) { this.isUDFAllowed = val; } public boolean isUDFAllowed() { return this.isUDFAllowed; } + public void setTimezone(String timezone) { this.timezone = timezone; } + public String getTimezone() { return timezone; } // state shared between all objects of an Analyzer tree // TODO: Many maps here contain properties about tuples, e.g., whether diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java index 021b15448c9253..17deb5766ebeb9 100644 --- a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -60,6 +60,7 @@ public class LoadStmt extends DdlStmt { public static final String CLUSTER_PROPERTY = "cluster"; private static final String VERSION = "version"; public static final String STRICT_MODE = "strict_mode"; + public static final String TIMEZONE = "timezone"; // for load data from Baidu Object Store(BOS) public static final String BOS_ENDPOINT = "bos_endpoint"; diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java index 4e0c41df71d82b..571ca2b7a5b911 100644 --- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -20,15 +20,15 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; - import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.VariableMgr; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -48,10 +48,12 @@ public class TimeUtils { private static final Logger LOG = LogManager.getLogger(TimeUtils.class); + public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai"; + private static final TimeZone TIME_ZONE; // set CST to +08:00 instead of America/Chicago - public static final ImmutableMap timeZoneAliasMap = ImmutableMap.of("CST", "Asia/Shanghai"); + public static final ImmutableMap timeZoneAliasMap = ImmutableMap.of("CST", DEFAULT_TIME_ZONE); // NOTICE: Date formats are not synchronized. // it must be used as synchronized externally. @@ -79,8 +81,6 @@ public class TimeUtils { public static int MIN_TIME; public static int MAX_TIME; - public static String DEFAULT_TIME_ZONE = "Asia/Shanghai"; - static { TIME_ZONE = new SimpleTimeZone(8 * 3600 * 1000, ""); diff --git a/fe/src/main/java/org/apache/doris/load/ExportJob.java b/fe/src/main/java/org/apache/doris/load/ExportJob.java index 6207ae03ed610c..fbfd965c120aca 100644 --- a/fe/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/src/main/java/org/apache/doris/load/ExportJob.java @@ -352,7 +352,8 @@ private void genCoordinators(List fragments, List nodes) ScanNode scanNode = nodes.get(i); TUniqueId queryId = new TUniqueId(uuid.getMostSignificantBits() + i, uuid.getLeastSignificantBits()); Coordinator coord = new Coordinator( - id, queryId, desc, Lists.newArrayList(fragment), Lists.newArrayList(scanNode), clusterName); + id, queryId, desc, Lists.newArrayList(fragment), Lists.newArrayList(scanNode), clusterName, + TimeUtils.DEFAULT_TIME_ZONE); coord.setExecMemoryLimit(getExecMemLimit()); this.coordList.add(coord); } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index c75dc9155e54e3..36fa1a4b132245 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -356,8 +356,8 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme // Generate loading task and init the plan of task LoadLoadingTask task = new LoadLoadingTask(db, table, brokerDesc, - entry.getValue(), getDeadlineMs(), execMemLimit, - strictMode, transactionId, this); + entry.getValue(), getDeadlineMs(), execMemLimit, + strictMode, transactionId, this, timezone); UUID uuid = UUID.randomUUID(); TUniqueId loadId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); task.init(loadId, attachment.getFileStatusByTable(tableId), attachment.getFileNumByTable(tableId)); diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index eca39a7bf5ca43..b61c7d915c98b7 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -94,9 +94,10 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements protected long timeoutSecond = Config.broker_load_default_timeout_second; protected long execMemLimit = 2147483648L; // 2GB; protected double maxFilterRatio = 0; + protected boolean strictMode = true; + protected String timezone = TimeUtils.DEFAULT_TIME_ZONE; @Deprecated protected boolean deleteFlag = false; - protected boolean strictMode = true; protected long createTimestamp = System.currentTimeMillis(); protected long loadStartTimestamp = -1; @@ -303,6 +304,11 @@ protected void setJobProperties(Map properties) throws DdlExcept if (properties.containsKey(LoadStmt.STRICT_MODE)) { strictMode = Boolean.valueOf(properties.get(LoadStmt.STRICT_MODE)); } + + if (properties.containsKey(LoadStmt.TIMEZONE)) { + timezone = properties.get(LoadStmt.TIMEZONE); + TimeUtils.checkTimeZoneValid(timezone); + } } } @@ -880,6 +886,7 @@ public void write(DataOutput out) throws IOException { out.writeBoolean(true); authorizationInfo.write(out); } + Text.writeString(out, timezone); } @Override @@ -920,5 +927,8 @@ public void readFields(DataInput in) throws IOException { authorizationInfo.readFields(in); } } + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + timezone = Text.readString(in); + } } } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index 4f25a90b12136e..3ee8a437563c51 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -57,13 +57,14 @@ public class LoadLoadingTask extends LoadTask { private final long execMemLimit; private final boolean strictMode; private final long txnId; + private final String timezone; private LoadingTaskPlanner planner; public LoadLoadingTask(Database db, OlapTable table, BrokerDesc brokerDesc, List fileGroups, long jobDeadlineMs, long execMemLimit, boolean strictMode, - long txnId, LoadTaskCallback callback) { + long txnId, LoadTaskCallback callback, String timezone) { super(callback); this.db = db; this.table = table; @@ -75,11 +76,12 @@ public LoadLoadingTask(Database db, OlapTable table, this.txnId = txnId; this.failMsg = new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL); this.retryTime = 2; // 2 times is enough + this.timezone = timezone; } public void init(TUniqueId loadId, List> fileStatusList, int fileNum) throws UserException { this.loadId = loadId; - planner = new LoadingTaskPlanner(callback.getCallbackId(), txnId, db.getId(), table, brokerDesc, fileGroups, strictMode); + planner = new LoadingTaskPlanner(callback.getCallbackId(), txnId, db.getId(), table, brokerDesc, fileGroups, strictMode, timezone); planner.plan(loadId, fileStatusList, fileNum); } @@ -98,7 +100,7 @@ protected void executeTask() throws Exception{ private void executeOnce() throws Exception { // New one query id, Coordinator curCoordinator = new Coordinator(callback.getCallbackId(), loadId, planner.getDescTable(), - planner.getFragments(), planner.getScanNodes(), db.getClusterName()); + planner.getFragments(), planner.getScanNodes(), db.getClusterName(), planner.getTimezone()); curCoordinator.setQueryType(TQueryType.LOAD); curCoordinator.setExecMemoryLimit(execMemLimit); curCoordinator.setTimeout((int) (getLeftTimeMs() / 1000)); diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index d1f8990633fbc3..e0019e68a720cd 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -80,7 +80,7 @@ public class LoadingTaskPlanner { public LoadingTaskPlanner(Long loadJobId, long txnId, long dbId, OlapTable table, BrokerDesc brokerDesc, List brokerFileGroups, - boolean strictMode) { + boolean strictMode, String timezone) { this.loadJobId = loadJobId; this.txnId = txnId; this.dbId = dbId; @@ -88,6 +88,7 @@ public LoadingTaskPlanner(Long loadJobId, long txnId, long dbId, OlapTable table this.brokerDesc = brokerDesc; this.fileGroups = brokerFileGroups; this.strictMode = strictMode; + this.analyzer.setTimezone(timezone); } public void plan(TUniqueId loadId, List> fileStatusesList, int filesAdded) @@ -154,6 +155,10 @@ public List getScanNodes() { return scanNodes; } + public String getTimezone() { + return analyzer.getTimezone(); + } + private String convertBrokerDescPartitionInfo() throws LoadException, MetaNotFoundException { String result = ""; for (BrokerFileGroup brokerFileGroup : fileGroups) { diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 079be11eee6539..7ceb0df2adc04e 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -133,6 +133,7 @@ private static class ParamCreateContext { public TupleDescriptor tupleDescriptor; public Map exprMap; public Map slotDescByName; + public String timezone; } private List paramCreateContexts; @@ -168,6 +169,7 @@ public void init(Analyzer analyzer) throws UserException { for (BrokerFileGroup fileGroup : fileGroups) { ParamCreateContext context = new ParamCreateContext(); context.fileGroup = fileGroup; + context.timezone = analyzer.getTimezone(); try { initParams(context); } catch (AnalysisException e) { @@ -278,7 +280,7 @@ private void initColumns(ParamCreateContext context) throws UserException { Preconditions.checkNotNull(col, columnName); String realColName = col.getName(); if (columnExpr != null) { - columnExpr = transformHadoopFunctionExpr(columnName, columnExpr); + columnExpr = transformHadoopFunctionExpr(columnName, columnExpr, context.timezone); context.exprMap.put(realColName, columnExpr); } } @@ -322,7 +324,7 @@ private void initColumns(ParamCreateContext context) throws UserException { * @return * @throws UserException */ - private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) throws UserException { + private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr, String timezone) throws UserException { Column column = targetTable.getColumn(columnName); if (column == null) { throw new UserException("Unknown column(" + columnName + ")"); @@ -415,14 +417,17 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr } else if (funcName.equalsIgnoreCase("alignment_timestamp")) { /* * change to: - * UNIX_TIMESTAMP(DATE_FORMAT(FROM_UNIXTIME(value), "%Y-01-01 00:00:00")) + * UNIX_TIMESTAMP(DATE_FORMAT(FROM_UNIXTIME(ts), "%Y-01-01 00:00:00")); + * */ + // FROM_UNIXTIME FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); FunctionCallExpr fromUnixFunc = new FunctionCallExpr( fromUnixName, new FunctionParams(false, fromUnixArgs)); + // DATE_FORMAT StringLiteral precision = (StringLiteral) funcExpr.getChild(0); StringLiteral format; if (precision.getStringValue().equalsIgnoreCase("year")) { @@ -441,6 +446,7 @@ private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr) thr FunctionCallExpr dateFormatFunc = new FunctionCallExpr( dateFormatName, new FunctionParams(false, dateFormatArgs)); + // UNIX_TIMESTAMP FunctionName unixTimeName = new FunctionName("UNIX_TIMESTAMP"); List unixTimeArgs = Lists.newArrayList(); unixTimeArgs.add(dateFormatFunc); diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index 954096a65bc23b..4b874e53cf0ead 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -216,7 +216,7 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) { // Used for broker load task/export task coordinator public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, - List fragments, List scanNodes, String cluster) { + List fragments, List scanNodes, String cluster, String timezone) { this.isBlockQuery = true; this.jobId = jobId; this.queryId = queryId; @@ -226,8 +226,7 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, this.queryOptions = new TQueryOptions(); this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); this.queryGlobals.setTimestamp_ms(new Date().getTime()); - // this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); - this.queryGlobals.setTime_zone("Africa/Abidjan"); + this.queryGlobals.setTime_zone(timezone); this.tResourceInfo = new TResourceInfo("", ""); this.needReport = true; this.clusterName = cluster; diff --git a/fe/src/main/java/org/apache/doris/task/PullLoadTask.java b/fe/src/main/java/org/apache/doris/task/PullLoadTask.java index 3ac0ebf96dd3aa..5a44f223cf0d37 100644 --- a/fe/src/main/java/org/apache/doris/task/PullLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/PullLoadTask.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.BrokerFileGroup; import org.apache.doris.qe.Coordinator; import org.apache.doris.qe.QeProcessorImpl; @@ -210,7 +211,7 @@ public void executeOnce() throws UserException { UUID uuid = UUID.randomUUID(); queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); curCoordinator = new Coordinator(jobId, queryId, planner.getDescTable(), - planner.getFragments(), planner.getScanNodes(), db.getClusterName()); + planner.getFragments(), planner.getScanNodes(), db.getClusterName(), TimeUtils.DEFAULT_TIME_ZONE); curCoordinator.setQueryType(TQueryType.LOAD); curCoordinator.setExecMemoryLimit(execMemLimit); curCoordinator.setTimeout((int) (getLeftTimeMs() / 1000)); From 23e57c6266786c60ca531fbd21edca858cee676c Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 19 Aug 2019 16:14:35 +0800 Subject: [PATCH 49/79] fix columns from path after merge --- .../doris/analysis/DataDescription.java | 41 +++++++++++++++---- .../apache/doris/planner/BrokerScanNode.java | 11 +++-- .../doris/common/util/BrokerUtilTest.java | 17 ++++++-- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index c5cf56b1697419..2694dd7d75da71 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -217,19 +217,28 @@ public boolean isHadoopLoad() { } /** - * Analyze parsedExprMap and columnToHadoopFunction from columns and columnMappingList - * Example: columns (col1, tmp_col2, tmp_col3) set (col2=tmp_col2+1, col3=strftime("%Y-%m-%d %H:%M:%S", tmp_col3)) - * Result: parsedExprMap = {"col1": null, "tmp_col2": null, "tmp_col3": null, - * "col2": "tmp_col2+1", "col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} + * Analyze parsedExprMap and columnToHadoopFunction from columns, columns from path and columnMappingList + * Example: + * columns (col1, tmp_col2, tmp_col3) + * columns from path as (col4, col5) + * set (col2=tmp_col2+1, col3=strftime("%Y-%m-%d %H:%M:%S", tmp_col3)) + * Result: + * + * parsedExprMap = {"col1": null, "tmp_col2": null, "tmp_col3": null, "col4": null, "col5": null, + * "col2": "tmp_col2+1", "col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} + * columnToHadoopFunction = {"col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} */ private void analyzeColumns() throws AnalysisException { - // used to check duplicated column name in COLUMNS + if ((columns == null || columns.isEmpty()) && (columnMappingList != null && !columnMappingList.isEmpty())) { + throw new AnalysisException("Can not specify columns_from_path without column_list"); + } + + // used to check duplicated column name in COLUMNS and COLUMNS FROM PATH Set columnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - // merge columns exprs from columns and columnMappingList - // Step1: analyze columns + // merge columns exprs from columns, columns from path and columnMappingList + // 1. analyze columns if (columns != null && !columns.isEmpty()) { - // Step1: analyze columns for (String columnName : columns) { if (!columnNames.add(columnName)) { throw new AnalysisException("Duplicate column: " + columnName); @@ -239,7 +248,21 @@ private void analyzeColumns() throws AnalysisException { } } - // Step2: analyze column mapping + // 2. analyze columns from path + if (columnsFromPath != null && !columnsFromPath.isEmpty()) { + if (isHadoopLoad) { + throw new AnalysisException("Hadoop load does not support specifying columns from path"); + } + for (String columnName : columnsFromPath) { + if (!columnNames.add(columnName)) { + throw new AnalysisException("Duplicate column: " + columnName); + } + ImportColumnDesc importColumnDesc = new ImportColumnDesc(columnName, null); + parsedColumnExprList.add(importColumnDesc); + } + } + + // 3: analyze column mapping if (columnMappingList == null || columnMappingList.isEmpty()) { return; } diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 7ceb0df2adc04e..988491a98a45ee 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -230,14 +230,19 @@ private void initParams(ParamCreateContext context) throws AnalysisException, Us */ private void initColumns(ParamCreateContext context) throws UserException { List sourceFileColumns = context.fileGroup.getFileFieldNames(); + List pathColumns = context.fileGroup.getColumnsFromPath(); List originColumnNameToExprList = context.fileGroup.getColumnExprList(); - // originColumnNameToExprList must has emlements. because it is always filled by user or by system + // originColumnNameToExprList must has elements. because it is always filled by user or by system Preconditions.checkState(originColumnNameToExprList != null && !originColumnNameToExprList.isEmpty()); - // 1. create source slots + // 1. create source slots, from sourceFileColumns and pathColumns context.tupleDescriptor = analyzer.getDescTbl().createTupleDescriptor(); context.slotDescByName = Maps.newHashMap(); - for (String sourceColName : sourceFileColumns) { + List allColumns = Lists.newArrayList(sourceFileColumns); + if (pathColumns != null) { + allColumns.addAll(pathColumns); + } + for (String sourceColName : allColumns) { SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(context.tupleDescriptor); slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); slotDesc.setIsMaterialized(true); diff --git a/fe/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java b/fe/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java index f8f1815142f21f..55a6e00c000d55 100644 --- a/fe/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java +++ b/fe/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java @@ -17,15 +17,18 @@ package org.apache.doris.common.util; -import com.google.common.collect.Lists; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import org.apache.doris.common.UserException; + +import com.google.common.collect.Lists; + import org.junit.Test; import java.util.Collections; import java.util.List; -import static org.junit.Assert.*; - public class BrokerUtilTest { @Test @@ -110,5 +113,13 @@ public void parseColumnsFromPath() { } catch (UserException ignored) { } + path = "/path/to/dir/k1=2/a/xxx.csv"; + try { + List columns = BrokerUtil.parseColumnsFromPath(path, Collections.singletonList("k1")); + fail(); + } catch (UserException ignored) { + ignored.printStackTrace(); + } + } } From d11e493b42c7cd2a30dee3aadf3c87932ce91957 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 20 Aug 2019 22:33:55 +0800 Subject: [PATCH 50/79] modify version to 60 after rebase --- fe/src/main/java/org/apache/doris/catalog/Catalog.java | 2 +- fe/src/main/java/org/apache/doris/catalog/Partition.java | 2 +- .../main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 2 +- fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 62f6f27ec1cbc0..a493f8d88803e4 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1617,7 +1617,7 @@ public long loadAlterJob(DataInputStream dis, long checksum, JobType type) throw } // alter job v2 - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { size = dis.readInt(); newChecksum ^= size; for (int i = 0; i < size; i++) { diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index f14c3fb3d661a8..d91ab91b53d268 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -344,7 +344,7 @@ public void readFields(DataInput in) throws IOException { idToVisibleRollupIndex.put(rollupTable.getId(), rollupTable); } - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { int shadowIndexCount = in.readInt(); for (int i = 0; i < shadowIndexCount; i++) { MaterializedIndex shadowIndex = MaterializedIndex.read(in); diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 36fa1a4b132245..e6b9a4236344ca 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -508,7 +508,7 @@ public void write(DataOutput out) throws IOException { public void readFields(DataInput in) throws IOException { super.readFields(in); brokerDesc = BrokerDesc.read(in); - if (Catalog.getCurrentCatalogJournalVersion() <= FeMetaVersion.VERSION_58) { + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_60) { dataSourceInfo.readFields(in); } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index b61c7d915c98b7..15c2c026187cab 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -927,7 +927,7 @@ public void readFields(DataInput in) throws IOException { authorizationInfo.readFields(in); } } - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { timezone = Text.readString(in); } } From 285f49f94fbbff6fd2be1e1f79bca2594a4f87a3 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 21 Aug 2019 09:09:57 +0800 Subject: [PATCH 51/79] fix column path --- fe/src/main/java/org/apache/doris/analysis/DataDescription.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index 2694dd7d75da71..d1d346f8f9ce04 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -229,7 +229,7 @@ public boolean isHadoopLoad() { * columnToHadoopFunction = {"col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} */ private void analyzeColumns() throws AnalysisException { - if ((columns == null || columns.isEmpty()) && (columnMappingList != null && !columnMappingList.isEmpty())) { + if ((columns == null || columns.isEmpty()) && (columnsFromPath != null && !columnsFromPath.isEmpty())) { throw new AnalysisException("Can not specify columns_from_path without column_list"); } From 08469f58383f16f3acaf1dce2e166e0575581bcb Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 21 Aug 2019 10:04:22 +0800 Subject: [PATCH 52/79] change AlterJobV2 to abstract --- .../org/apache/doris/alter/AlterJobV2.java | 33 ++++++++----------- .../org/apache/doris/alter/RollupJobV2.java | 22 ++++++------- .../apache/doris/alter/SchemaChangeJobV2.java | 2 +- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 8cc6369b7ff543..155a6d670d53bb 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; -import org.apache.commons.lang.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,7 +39,7 @@ * Version 2 of AlterJob, for replacing the old version of AlterJob. * This base class of RollupJob and SchemaChangeJob */ -public class AlterJobV2 implements Writable { +public abstract class AlterJobV2 implements Writable { private static final Logger LOG = LogManager.getLogger(AlterJobV2.class); public enum JobState { @@ -133,7 +132,7 @@ public boolean isDone() { */ public synchronized void run() { if (isTimeout()) { - cancel("Timeout"); + cancelImpl("Timeout"); return; } @@ -152,29 +151,23 @@ public synchronized void run() { } } - protected void runPendingJob() { - throw new NotImplementedException(); + public final boolean cancel(String errMsg) { + synchronized (this) { + return cancelImpl(errMsg); + } } - protected void runWaitingTxnJob() { - throw new NotImplementedException(); - } + protected abstract void runPendingJob(); - protected void runRunningJob() { - throw new NotImplementedException(); - } + protected abstract void runWaitingTxnJob(); - public synchronized boolean cancel(String errMsg) { - throw new NotImplementedException(); - } + protected abstract void runRunningJob(); - protected void getInfo(List> infos) { - throw new NotImplementedException(); - } + protected abstract boolean cancelImpl(String errMsg); - public void replay(AlterJobV2 replayedJob) { - throw new NotImplementedException(); - } + protected abstract void getInfo(List> infos); + + public abstract void replay(AlterJobV2 replayedJob); public static AlterJobV2 read(DataInput in) throws IOException { JobType type = JobType.valueOf(Text.readString(in)); diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index f7471a6634f506..661e587f149d5a 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -146,7 +146,7 @@ protected void runPendingJob() { LOG.info("begin to send create rollup replica tasks. job: {}", jobId); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancel("Databasee " + dbId + " does not exist"); + cancelImpl("Databasee " + dbId + " does not exist"); return; } @@ -164,7 +164,7 @@ protected void runPendingJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); @@ -231,7 +231,7 @@ protected void runPendingJob() { errMsg = "Error replicas:" + Joiner.on(", ").join(subList); } LOG.warn("failed to create rollup replicas for job: {}, {}", jobId, errMsg); - cancel("Create rollup replicas failed. Error: " + errMsg); + cancelImpl("Create rollup replicas failed. Error: " + errMsg); return; } @@ -241,7 +241,7 @@ protected void runPendingJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); @@ -290,7 +290,7 @@ protected void runWaitingTxnJob() { LOG.info("previous transactions are all finished, begin to send rollup tasks. job: {}", jobId); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancel("Databasee " + dbId + " does not exist"); + cancelImpl("Databasee " + dbId + " does not exist"); return; } @@ -298,7 +298,7 @@ protected void runWaitingTxnJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); @@ -362,7 +362,7 @@ protected void runRunningJob() { */ Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancel("Databasee " + dbId + " does not exist"); + cancelImpl("Databasee " + dbId + " does not exist"); return; } @@ -370,7 +370,7 @@ protected void runRunningJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP); @@ -399,7 +399,7 @@ protected void runRunningJob() { if (healthyReplicaNum < expectReplicationNum / 2 + 1) { LOG.warn("rollup tablet {} has few healthy replicas: {}, rollup job: {}", rollupTablet.getId(), replicas, jobId); - cancel("rollup tablet " + rollupTablet.getId() + " has few healthy replicas"); + cancelImpl("rollup tablet " + rollupTablet.getId() + " has few healthy replicas"); return; } } // end for tablets @@ -432,11 +432,11 @@ private void onFinished(OlapTable tbl) { } /* - * cancel() can be called any time any place. + * cancelImpl() can be called any time any place. * We need to clean any possible residual of this job. */ @Override - public synchronized boolean cancel(String errMsg) { + protected boolean cancelImpl(String errMsg) { if (jobState.isFinalState()) { return false; } diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 0c7596a85eb5e3..1d9cfa9041ee8c 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -511,7 +511,7 @@ private void onFinished(OlapTable tbl) { * We need to clean any possible residual of this job. */ @Override - public synchronized boolean cancel(String errMsg) { + protected synchronized boolean cancelImpl(String errMsg) { if (jobState.isFinalState()) { return false; } From 9e8577ecbd7bf2d1f34d10605edb6cbb42d322fe Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 21 Aug 2019 15:03:52 +0800 Subject: [PATCH 53/79] add timezone in load --- be/src/olap/delta_writer.h | 2 +- be/src/olap/memtable.cpp | 2 +- be/src/olap/memtable.h | 4 ++-- be/test/olap/delta_writer_test.cpp | 5 +++-- .../apache/doris/alter/SchemaChangeJobV2.java | 20 +++++++++---------- .../org/apache/doris/analysis/LoadStmt.java | 7 +++++++ .../org/apache/doris/load/loadv2/LoadJob.java | 1 - .../org/apache/doris/catalog/FakeEditLog.java | 6 ++++++ 8 files changed, 30 insertions(+), 17 deletions(-) diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h index ad3798c7a984df..f9d1587f5ee187 100644 --- a/be/src/olap/delta_writer.h +++ b/be/src/olap/delta_writer.h @@ -48,7 +48,7 @@ struct WriteRequest { bool need_gen_rollup; TupleDescriptor* tuple_desc; // slots are in order of tablet's schema - std::vector* slots; + const std::vector* slots; }; class DeltaWriter { diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index cd74adde7dac56..a0054a89deeeda 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -27,7 +27,7 @@ namespace doris { MemTable::MemTable(Schema* schema, const TabletSchema* tablet_schema, - std::vector* slot_descs, TupleDescriptor* tuple_desc, + const std::vector* slot_descs, TupleDescriptor* tuple_desc, KeysType keys_type) : _schema(schema), _tablet_schema(tablet_schema), diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index bc14ce7c22928c..1a262ce3ddb750 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -32,7 +32,7 @@ class RowCursor; class MemTable { public: MemTable(Schema* schema, const TabletSchema* tablet_schema, - std::vector* slot_descs, TupleDescriptor* tuple_desc, + const std::vector* slot_descs, TupleDescriptor* tuple_desc, KeysType keys_type); ~MemTable(); size_t memory_usage(); @@ -44,7 +44,7 @@ class MemTable { const TabletSchema* _tablet_schema; TupleDescriptor* _tuple_desc; // the slot in _slot_descs are in order of tablet's schema - std::vector* _slot_descs; + const std::vector* _slot_descs; KeysType _keys_type; struct RowCursorComparator { diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 435619e1bcf5b7..ab4e0aba36e0b3 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -323,17 +323,18 @@ TEST_F(TestDeltaWriter, write) { DescriptorTbl* desc_tbl = nullptr; DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl); TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0); + const std::vector& slots = tuple_desc->slots(); PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); WriteRequest write_req = {10004, 270068376, WriteType::LOAD, - 20002, 30002, load_id, false, tuple_desc}; + 20002, 30002, load_id, false, tuple_desc, + &(tuple_desc->slots())}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); ASSERT_NE(delta_writer, nullptr); - const std::vector& slots = tuple_desc->slots(); Arena arena; // Tuple 1 { diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 1d9cfa9041ee8c..55d5e02b4c9f76 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -157,7 +157,7 @@ protected void runPendingJob() { LOG.info("begin to send create replica tasks. job: {}", jobId); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancel("Databasee " + dbId + " does not exist"); + cancelImpl("Databasee " + dbId + " does not exist"); return; } @@ -175,7 +175,7 @@ protected void runPendingJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); @@ -247,7 +247,7 @@ protected void runPendingJob() { errMsg = "Error replicas:" + Joiner.on(", ").join(subList); } LOG.warn("failed to create replicas for job: {}, {}", jobId, errMsg); - cancel("Create replicas failed. Error: " + errMsg); + cancelImpl("Create replicas failed. Error: " + errMsg); return; } @@ -257,7 +257,7 @@ protected void runPendingJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); @@ -316,7 +316,7 @@ protected void runWaitingTxnJob() { LOG.info("previous transactions are all finished, begin to send schema change tasks. job: {}", jobId); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancel("Databasee " + dbId + " does not exist"); + cancelImpl("Databasee " + dbId + " does not exist"); return; } @@ -324,7 +324,7 @@ protected void runWaitingTxnJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); @@ -395,7 +395,7 @@ protected void runRunningJob() { */ Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancel("Databasee " + dbId + " does not exist"); + cancelImpl("Databasee " + dbId + " does not exist"); return; } @@ -403,7 +403,7 @@ protected void runRunningJob() { try { OlapTable tbl = (OlapTable) db.getTable(tableId); if (tbl == null) { - cancel("Table " + tableId + " does not exist"); + cancelImpl("Table " + tableId + " does not exist"); return; } Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE); @@ -433,7 +433,7 @@ protected void runRunningJob() { if (healthyReplicaNum < expectReplicationNum / 2 + 1) { LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}", shadowTablet.getId(), replicas, jobId); - cancel("shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); + cancelImpl("shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); return; } } // end for tablets @@ -507,7 +507,7 @@ private void onFinished(OlapTable tbl) { } /* - * cancel() can be called any time any place. + * cancelImpl() can be called any time any place. * We need to clean any possible residual of this job. */ @Override diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java index 17deb5766ebeb9..1cdd20b720025b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -21,6 +21,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.Load; import org.apache.doris.qe.ConnectContext; @@ -93,6 +94,7 @@ public class LoadStmt extends DdlStmt { .add(CLUSTER_PROPERTY) .add(STRICT_MODE) .add(VERSION) + .add(TIMEZONE) .build(); public LoadStmt(LabelName label, List dataDescriptions, @@ -196,6 +198,11 @@ public static void checkProperties(Map properties) throws DdlExc } } + // time zone + final String timezone = properties.get(STRICT_MODE); + if (timezone != null) { + TimeUtils.checkTimeZoneValid(timezone); + } } private void analyzeVersion() throws AnalysisException { diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index 15c2c026187cab..aff28c52d0a6f8 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -307,7 +307,6 @@ protected void setJobProperties(Map properties) throws DdlExcept if (properties.containsKey(LoadStmt.TIMEZONE)) { timezone = properties.get(LoadStmt.TIMEZONE); - TimeUtils.checkTimeZoneValid(timezone); } } } diff --git a/fe/src/test/java/org/apache/doris/catalog/FakeEditLog.java b/fe/src/test/java/org/apache/doris/catalog/FakeEditLog.java index 62d5e3937b07e0..4d45afcb644665 100644 --- a/fe/src/test/java/org/apache/doris/catalog/FakeEditLog.java +++ b/fe/src/test/java/org/apache/doris/catalog/FakeEditLog.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.alter.AlterJobV2; import org.apache.doris.alter.RollupJob; import org.apache.doris.alter.SchemaChangeJob; import org.apache.doris.cluster.Cluster; @@ -91,6 +92,11 @@ public void logOpRoutineLoadJob(RoutineLoadOperation operation) { } + @Mock + public void logAlterJob(AlterJobV2 alterJob) { + + } + public TransactionState getTransaction(long transactionId) { return allTransactionState.get(transactionId); } From 2158b2f4b363a968bbb18e242b86da49d46e3139 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 Aug 2019 08:41:43 +0800 Subject: [PATCH 54/79] add ut --- .../apache/doris/alter/SchemaChangeJobV2.java | 60 ++++++++++--------- .../org/apache/doris/catalog/Partition.java | 3 +- .../org/apache/doris/common/FeConstants.java | 3 + .../org/apache/doris/task/AgentTaskQueue.java | 9 +++ 4 files changed, 46 insertions(+), 29 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 55d5e02b4c9f76..ec25b8d68fda39 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -31,6 +31,7 @@ import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; @@ -220,35 +221,37 @@ protected void runPendingJob() { db.readUnlock(); } - // send all tasks and wait them finished - AgentTaskQueue.addBatchTask(batchTask); - AgentTaskExecutor.submit(batchTask); - // max timeout is 1 min - long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, 60000); - boolean ok = false; - try { - ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - LOG.warn("InterruptedException: ", e); - ok = false; - } - - if (!ok) { - // create replicas failed. just cancel the job - // clear tasks and show the failed replicas to user - AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); - String errMsg = null; - if (!countDownLatch.getStatus().ok()) { - errMsg = countDownLatch.getStatus().getErrorMsg(); - } else { - List> unfinishedMarks = countDownLatch.getLeftMarks(); - // only show at most 3 results - List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); - errMsg = "Error replicas:" + Joiner.on(", ").join(subList); + if (!FeConstants.runningUnitTtest) { + // send all tasks and wait them finished + AgentTaskQueue.addBatchTask(batchTask); + AgentTaskExecutor.submit(batchTask); + // max timeout is 1 min + long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, 60000); + boolean ok = false; + try { + ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOG.warn("InterruptedException: ", e); + ok = false; + } + + if (!ok) { + // create replicas failed. just cancel the job + // clear tasks and show the failed replicas to user + AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); + String errMsg = null; + if (!countDownLatch.getStatus().ok()) { + errMsg = countDownLatch.getStatus().getErrorMsg(); + } else { + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 3 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); + errMsg = "Error replicas:" + Joiner.on(", ").join(subList); + } + LOG.warn("failed to create replicas for job: {}, {}", jobId, errMsg); + cancelImpl("Create replicas failed. Error: " + errMsg); + return; } - LOG.warn("failed to create replicas for job: {}, {}", jobId, errMsg); - cancelImpl("Create replicas failed. Error: " + errMsg); - return; } // create all replicas success. @@ -368,6 +371,7 @@ protected void runWaitingTxnJob() { AgentTaskQueue.addBatchTask(schemaChangeBatchTask); AgentTaskExecutor.submit(schemaChangeBatchTask); + this.jobState = JobState.RUNNING; // DO NOT write edit log here, tasks will be send again if FE restart or master changed. diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index d91ab91b53d268..ff4647276b5363 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -49,10 +49,11 @@ public class Partition extends MetaObject implements Writable { public static final long PARTITION_INIT_VERSION = 1L; public static final long PARTITION_INIT_VERSION_HASH = 0L; - @Deprecated public enum PartitionState { NORMAL, + @Deprecated ROLLUP, + @Deprecated SCHEMA_CHANGE } diff --git a/fe/src/main/java/org/apache/doris/common/FeConstants.java b/fe/src/main/java/org/apache/doris/common/FeConstants.java index fd2d3d8676ae2f..e4f7b606a3b2e2 100644 --- a/fe/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/src/main/java/org/apache/doris/common/FeConstants.java @@ -33,6 +33,9 @@ public class FeConstants { // bloom filter false positive probability public static double default_bloom_filter_fpp = 0.05; + // set to true to skip some step when running FE unit test + public static boolean runningUnitTtest = false; + // general model // Current meta data version. Use this version to write journals and image public static int meta_version = FeMetaVersion.VERSION_60; diff --git a/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java b/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java index 48e02054ebd6bf..6b701caa36b2c8 100644 --- a/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java +++ b/fe/src/main/java/org/apache/doris/task/AgentTaskQueue.java @@ -142,6 +142,15 @@ public static synchronized AgentTask getTask(long backendId, TTaskType type, lon return signatureMap.get(signature); } + // this is just for unit test + public static synchronized List getTask(TTaskType type) { + List res = Lists.newArrayList(); + for (Map agentTasks : tasks.column(TTaskType.ALTER).values()) { + res.addAll(agentTasks.values()); + } + return res; + } + public static synchronized List getDiffTasks(long backendId, Map> runningTasks) { List diffTasks = new ArrayList(); if (!tasks.containsRow(backendId)) { From a71b90f340ca90d1e16b707f230ab2d029b0bf0a Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 Aug 2019 18:54:00 +0800 Subject: [PATCH 55/79] add ut --- .../org/apache/doris/alter/RollupJobV2.java | 57 +-- .../apache/doris/alter/SchemaChangeJobV2.java | 2 +- .../org/apache/doris/common/FeConstants.java | 2 +- .../org/apache/doris/alter/RollupJobTest.java | 352 ------------------ .../apache/doris/alter/RollupJobV2Test.java | 221 +++++++++++ .../doris/alter/SchemaChangeJobTest.java | 280 -------------- .../doris/alter/SchemaChangeJobV2Test.java | 199 ++++++++++ 7 files changed, 452 insertions(+), 661 deletions(-) delete mode 100644 fe/src/test/java/org/apache/doris/alter/RollupJobTest.java create mode 100644 fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java delete mode 100644 fe/src/test/java/org/apache/doris/alter/SchemaChangeJobTest.java create mode 100644 fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 661e587f149d5a..1b837534af90d6 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -32,6 +32,7 @@ import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; @@ -204,35 +205,37 @@ protected void runPendingJob() { db.readUnlock(); } - // send all tasks and wait them finished - AgentTaskQueue.addBatchTask(batchTask); - AgentTaskExecutor.submit(batchTask); - long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, - Config.max_create_table_timeout_second * 1000L); - boolean ok = false; - try { - ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - LOG.warn("InterruptedException: ", e); - ok = false; - } + if (!FeConstants.runningUnitTest) { + // send all tasks and wait them finished + AgentTaskQueue.addBatchTask(batchTask); + AgentTaskExecutor.submit(batchTask); + long timeout = Math.min(Config.tablet_create_timeout_second * 1000L * totalReplicaNum, + Config.max_create_table_timeout_second * 1000L); + boolean ok = false; + try { + ok = countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOG.warn("InterruptedException: ", e); + ok = false; + } - if (!ok || !countDownLatch.getStatus().ok()) { - // create rollup replicas failed. just cancel the job - // clear tasks and show the failed replicas to user - AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); - String errMsg = null; - if (!countDownLatch.getStatus().ok()) { - errMsg = countDownLatch.getStatus().getErrorMsg(); - } else { - List> unfinishedMarks = countDownLatch.getLeftMarks(); - // only show at most 3 results - List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); - errMsg = "Error replicas:" + Joiner.on(", ").join(subList); + if (!ok || !countDownLatch.getStatus().ok()) { + // create rollup replicas failed. just cancel the job + // clear tasks and show the failed replicas to user + AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); + String errMsg = null; + if (!countDownLatch.getStatus().ok()) { + errMsg = countDownLatch.getStatus().getErrorMsg(); + } else { + List> unfinishedMarks = countDownLatch.getLeftMarks(); + // only show at most 3 results + List> subList = unfinishedMarks.subList(0, Math.min(unfinishedMarks.size(), 3)); + errMsg = "Error replicas:" + Joiner.on(", ").join(subList); + } + LOG.warn("failed to create rollup replicas for job: {}, {}", jobId, errMsg); + cancelImpl("Create rollup replicas failed. Error: " + errMsg); + return; } - LOG.warn("failed to create rollup replicas for job: {}, {}", jobId, errMsg); - cancelImpl("Create rollup replicas failed. Error: " + errMsg); - return; } // create all rollup replicas success. diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index ec25b8d68fda39..ba87b502b915dd 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -221,7 +221,7 @@ protected void runPendingJob() { db.readUnlock(); } - if (!FeConstants.runningUnitTtest) { + if (!FeConstants.runningUnitTest) { // send all tasks and wait them finished AgentTaskQueue.addBatchTask(batchTask); AgentTaskExecutor.submit(batchTask); diff --git a/fe/src/main/java/org/apache/doris/common/FeConstants.java b/fe/src/main/java/org/apache/doris/common/FeConstants.java index e4f7b606a3b2e2..7fc7cfc13a0f2b 100644 --- a/fe/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/src/main/java/org/apache/doris/common/FeConstants.java @@ -34,7 +34,7 @@ public class FeConstants { public static double default_bloom_filter_fpp = 0.05; // set to true to skip some step when running FE unit test - public static boolean runningUnitTtest = false; + public static boolean runningUnitTest = false; // general model // Current meta data version. Use this version to write journals and image diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java b/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java deleted file mode 100644 index 59d4c28de454ab..00000000000000 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobTest.java +++ /dev/null @@ -1,352 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.alter; - -import static org.junit.Assert.assertEquals; - -import org.apache.doris.alter.AlterJob.JobState; -import org.apache.doris.analysis.AccessTestUtil; -import org.apache.doris.analysis.AddRollupClause; -import org.apache.doris.analysis.AlterClause; -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.CatalogTestUtil; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.FakeCatalog; -import org.apache.doris.catalog.FakeEditLog; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexExtState; -import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.OlapTable.OlapTableState; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Partition.PartitionState; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.meta.MetaContext; -import org.apache.doris.task.AgentTask; -import org.apache.doris.task.AgentTaskQueue; -import org.apache.doris.thrift.TTabletInfo; -import org.apache.doris.thrift.TTaskType; -import org.apache.doris.transaction.FakeTransactionIDGenerator; -import org.apache.doris.transaction.GlobalTransactionMgr; -import org.apache.doris.transaction.TabletCommitInfo; -import org.apache.doris.transaction.TransactionState; -import org.apache.doris.transaction.TransactionState.LoadJobSourceType; -import org.apache.doris.transaction.TransactionStatus; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import mockit.internal.startup.Startup; - -public class RollupJobTest { - - private static FakeEditLog fakeEditLog; - private static FakeCatalog fakeCatalog; - private static FakeTransactionIDGenerator fakeTransactionIDGenerator; - private static GlobalTransactionMgr masterTransMgr; - private static GlobalTransactionMgr slaveTransMgr; - private static Catalog masterCatalog; - private static Catalog slaveCatalog; - - private String transactionSource = "localfe"; - private static Analyzer analyzer; - private static AddRollupClause clause; - - static { - Startup.initializeIfPossible(); - } - - @Before - public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException, AnalysisException { - fakeEditLog = new FakeEditLog(); - fakeCatalog = new FakeCatalog(); - fakeTransactionIDGenerator = new FakeTransactionIDGenerator(); - masterCatalog = CatalogTestUtil.createTestCatalog(); - slaveCatalog = CatalogTestUtil.createTestCatalog(); - MetaContext metaContext = new MetaContext(); - metaContext.setMetaVersion(FeMetaVersion.VERSION_40); - metaContext.setThreadLocalInfo(); - // masterCatalog.setJournalVersion(FeMetaVersion.VERSION_40); - // slaveCatalog.setJournalVersion(FeMetaVersion.VERSION_40); - masterTransMgr = masterCatalog.getGlobalTransactionMgr(); - masterTransMgr.setEditLog(masterCatalog.getEditLog()); - - slaveTransMgr = slaveCatalog.getGlobalTransactionMgr(); - slaveTransMgr.setEditLog(slaveCatalog.getEditLog()); - analyzer = AccessTestUtil.fetchAdminAnalyzer(false); - clause = new AddRollupClause(CatalogTestUtil.testRollupIndex2, Lists.newArrayList("k1", "v"), null, - CatalogTestUtil.testIndex1, null); - clause.analyze(analyzer); - } - - @Test - public void testAddRollup() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(clause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); - RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); - Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); - Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); - } - - // start a rollup, then finished - @Test - public void testRollup1() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); - - // add a rollup job - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(clause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); - RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); - Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); - Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); - MaterializedIndex rollupIndex = rollupJob.getRollupIndex(CatalogTestUtil.testPartitionId1); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); - assertEquals(IndexState.ROLLUP, rollupIndex.getState()); - assertEquals(IndexState.NORMAL, baseIndex.getState()); - assertEquals(OlapTableState.ROLLUP, olapTable.getState()); - assertEquals(PartitionState.ROLLUP, testPartition.getState()); - Tablet rollupTablet = rollupIndex.getTablets().get(0); - List replicas = rollupTablet.getReplicas(); - Replica rollupReplica1 = replicas.get(0); - Replica rollupReplica2 = replicas.get(1); - Replica rollupReplica3 = replicas.get(2); - - assertEquals(-1, rollupReplica1.getVersion()); - assertEquals(-1, rollupReplica2.getVersion()); - assertEquals(-1, rollupReplica3.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica2.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica3.getLastFailedVersion()); - assertEquals(-1, rollupReplica1.getLastSuccessVersion()); - assertEquals(-1, rollupReplica2.getLastSuccessVersion()); - assertEquals(-1, rollupReplica3.getLastSuccessVersion()); - - // rollup handler run one cycle, agent task is generated and send tasks - rollupHandler.runOneCycle(); - AgentTask task1 = AgentTaskQueue.getTask(rollupReplica1.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - AgentTask task2 = AgentTaskQueue.getTask(rollupReplica2.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - AgentTask task3 = AgentTaskQueue.getTask(rollupReplica3.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - - // be report finishe rollup success - TTabletInfo tTabletInfo = new TTabletInfo(rollupTablet.getId(), CatalogTestUtil.testSchemaHash1, - CatalogTestUtil.testStartVersion, CatalogTestUtil.testStartVersionHash, 0, 0); - rollupHandler.handleFinishedReplica(task1, tTabletInfo, -1); - rollupHandler.handleFinishedReplica(task2, tTabletInfo, -1); - rollupHandler.handleFinishedReplica(task3, tTabletInfo, -1); - - // rollup hander run one cycle again, the rollup job is finishing - rollupHandler.runOneCycle(); - Assert.assertEquals(JobState.FINISHING, rollupJob.getState()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica2.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica3.getVersion()); - assertEquals(-1, rollupReplica1.getLastFailedVersion()); - assertEquals(-1, rollupReplica2.getLastFailedVersion()); - assertEquals(-1, rollupReplica3.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastSuccessVersion()); - } - - // load some data and one replica has errors - // start a rollup and then load data - // load finished and rollup finished - @Test - public void testRollup2() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - // load one transaction with backend 2 has errors - long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, - CatalogTestUtil.testTxnLable1, transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); - // commit a transaction, backend 2 has errors - TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId1); - // TabletCommitInfo tabletCommitInfo2 = new - // TabletCommitInfo(CatalogTestUtil.testTabletId1, - // CatalogTestUtil.testBackendId2); - TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId3); - List transTablets = Lists.newArrayList(); - transTablets.add(tabletCommitInfo1); - // transTablets.add(tabletCommitInfo2); - transTablets.add(tabletCommitInfo3); - masterTransMgr.commitTransaction(CatalogTestUtil.testDbId1, transactionId, transTablets); - TransactionState transactionState = fakeEditLog.getTransaction(transactionId); - assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); - Set errorReplicaIds = Sets.newHashSet(); - errorReplicaIds.add(CatalogTestUtil.testReplicaId2); - masterTransMgr.finishTransaction(transactionId, errorReplicaIds); - transactionState = fakeEditLog.getTransaction(transactionId); - assertEquals(TransactionStatus.VISIBLE, transactionState.getTransactionStatus()); - - // start a rollup - RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); - // add a rollup job - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(clause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); - RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); - Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); - Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); - MaterializedIndex rollupIndex = rollupJob.getRollupIndex(CatalogTestUtil.testPartitionId1); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); - assertEquals(IndexState.ROLLUP, rollupIndex.getState()); - assertEquals(IndexState.NORMAL, baseIndex.getState()); - assertEquals(OlapTableState.ROLLUP, olapTable.getState()); - assertEquals(PartitionState.ROLLUP, testPartition.getState()); - Tablet rollupTablet = rollupIndex.getTablets().get(0); - List replicas = rollupTablet.getReplicas(); - Replica rollupReplica1 = replicas.get(0); - Replica rollupReplica3 = replicas.get(1); - assertEquals(2, rollupTablet.getReplicas().size()); - - assertEquals(-1, rollupReplica1.getVersion()); - assertEquals(-1, rollupReplica3.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica1.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica3.getLastFailedVersion()); - assertEquals(-1, rollupReplica1.getLastSuccessVersion()); - assertEquals(-1, rollupReplica3.getLastSuccessVersion()); - - // rollup handler run one cycle, agent task is generated and send tasks - rollupHandler.runOneCycle(); - AgentTask task1 = AgentTaskQueue.getTask(rollupReplica1.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - AgentTask task3 = AgentTaskQueue.getTask(rollupReplica3.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - - // be report finishe rollup success - TTabletInfo tTabletInfo = new TTabletInfo(rollupTablet.getId(), CatalogTestUtil.testSchemaHash1, - CatalogTestUtil.testStartVersion + 1, CatalogTestUtil.testPartitionNextVersionHash, 0, 0); - rollupHandler.handleFinishedReplica(task1, tTabletInfo, -1); - rollupHandler.handleFinishedReplica(task3, tTabletInfo, -1); - - // rollup hander run one cycle again, the rollup job is finishing - rollupHandler.runOneCycle(); - Assert.assertEquals(JobState.FINISHING, rollupJob.getState()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica1.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica3.getVersion()); - assertEquals(-1, rollupReplica1.getLastFailedVersion()); - assertEquals(-1, rollupReplica3.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica3.getLastSuccessVersion()); - } - - // start a rollup and then load data - // but load to rolluping index failed, then rollup is cancelled - @Test - public void testRollup3() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); - - // add a rollup job - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(clause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); - RollupJob rollupJob = (RollupJob) rollupHandler.getAlterJob(CatalogTestUtil.testTableId1); - Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); - Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); - MaterializedIndex rollupIndex = rollupJob.getRollupIndex(CatalogTestUtil.testPartitionId1); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); - assertEquals(IndexState.ROLLUP, rollupIndex.getState()); - assertEquals(IndexState.NORMAL, baseIndex.getState()); - assertEquals(OlapTableState.ROLLUP, olapTable.getState()); - assertEquals(PartitionState.ROLLUP, testPartition.getState()); - Tablet rollupTablet = rollupIndex.getTablets().get(0); - List replicas = rollupTablet.getReplicas(); - Replica rollupReplica1 = replicas.get(0); - Replica rollupReplica2 = replicas.get(1); - Replica rollupReplica3 = replicas.get(2); - - // rollup handler run one cycle, agent task is generated and send tasks - rollupHandler.runOneCycle(); - AgentTask task1 = AgentTaskQueue.getTask(rollupReplica1.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - AgentTask task2 = AgentTaskQueue.getTask(rollupReplica2.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - AgentTask task3 = AgentTaskQueue.getTask(rollupReplica3.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); - - // load a transaction, but rollup tablet failed, then the rollup job should be - // cancelled - long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, - CatalogTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); - // commit a transaction, backend 2 has errors - TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId1); - TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId2); - TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId3); - List transTablets = Lists.newArrayList(); - transTablets.add(tabletCommitInfo1); - transTablets.add(tabletCommitInfo2); - transTablets.add(tabletCommitInfo3); - masterTransMgr.commitTransaction(CatalogTestUtil.testDbId1, transactionId, transTablets); - TransactionState transactionState = fakeEditLog.getTransaction(transactionId); - assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); - Set errorReplicaIds = Sets.newHashSet(); - errorReplicaIds.add(CatalogTestUtil.testReplicaId2); - masterTransMgr.finishTransaction(transactionId, errorReplicaIds); - transactionState = fakeEditLog.getTransaction(transactionId); - - // rollup replca's last failed version should change to 13 - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica1.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica2.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, rollupReplica3.getLastFailedVersion()); - - // be report finishe rollup success - TTabletInfo tTabletInfo = new TTabletInfo(rollupTablet.getId(), CatalogTestUtil.testSchemaHash1, - CatalogTestUtil.testStartVersion, CatalogTestUtil.testStartVersionHash, 0, 0); - rollupHandler.handleFinishedReplica(task1, tTabletInfo, -1); - rollupHandler.handleFinishedReplica(task2, tTabletInfo, -1); - rollupHandler.handleFinishedReplica(task3, tTabletInfo, -1); - - // rollup hander run one cycle again, the rollup job is finishing - rollupHandler.runOneCycle(); - Assert.assertEquals(JobState.CANCELLED, rollupJob.getState()); - assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.ALL).size()); - } -} diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java new file mode 100644 index 00000000000000..ea2c00e64f3149 --- /dev/null +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.alter.AlterJobV2.JobState; +import org.apache.doris.analysis.AccessTestUtil; +import org.apache.doris.analysis.AddRollupClause; +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.FakeCatalog; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.UserException; +import org.apache.doris.meta.MetaContext; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.transaction.FakeTransactionIDGenerator; +import org.apache.doris.transaction.GlobalTransactionMgr; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/* + * Author: Chenmingyu + * Date: Aug 21, 2019 + */ + +public class RollupJobV2Test { + + private static FakeEditLog fakeEditLog; + private static FakeCatalog fakeCatalog; + private static FakeTransactionIDGenerator fakeTransactionIDGenerator; + private static GlobalTransactionMgr masterTransMgr; + private static GlobalTransactionMgr slaveTransMgr; + private static Catalog masterCatalog; + private static Catalog slaveCatalog; + + private String transactionSource = "localfe"; + private static Analyzer analyzer; + private static AddRollupClause clause; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, AnalysisException { + fakeEditLog = new FakeEditLog(); + fakeCatalog = new FakeCatalog(); + fakeTransactionIDGenerator = new FakeTransactionIDGenerator(); + masterCatalog = CatalogTestUtil.createTestCatalog(); + slaveCatalog = CatalogTestUtil.createTestCatalog(); + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_60); + metaContext.setThreadLocalInfo(); + // masterCatalog.setJournalVersion(FeMetaVersion.VERSION_40); + // slaveCatalog.setJournalVersion(FeMetaVersion.VERSION_40); + masterTransMgr = masterCatalog.getGlobalTransactionMgr(); + masterTransMgr.setEditLog(masterCatalog.getEditLog()); + + slaveTransMgr = slaveCatalog.getGlobalTransactionMgr(); + slaveTransMgr.setEditLog(slaveCatalog.getEditLog()); + analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + clause = new AddRollupClause(CatalogTestUtil.testRollupIndex2, Lists.newArrayList("k1", "v"), null, + CatalogTestUtil.testIndex1, null); + clause.analyze(analyzer); + + FeConstants.runningUnitTest = true; + } + + @Test + public void testAddSchemaChange() throws UserException { + FakeCatalog.setCatalog(masterCatalog); + RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(clause); + Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); + OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); + rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); + Map alterJobsV2 = rollupHandler.getAlterJobsV2(); + Assert.assertEquals(1, alterJobsV2.size()); + Assert.assertEquals(OlapTableState.ROLLUP, olapTable.getState()); + } + + // start a schema change, then finished + @Test + public void testSchemaChange1() throws Exception { + FakeCatalog.setCatalog(masterCatalog); + RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); + + // add a rollup job + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(clause); + Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); + OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); + Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); + rollupHandler.process(alterClauses, db.getClusterName(), db, olapTable); + Map alterJobsV2 = rollupHandler.getAlterJobsV2(); + Assert.assertEquals(1, alterJobsV2.size()); + RollupJobV2 rollupJob = (RollupJobV2) alterJobsV2.values().stream().findAny().get(); + + // runPendingJob + rollupHandler.runOneCycle(); + Assert.assertEquals(JobState.WAITING_TXN, rollupJob.getJobState()); + Assert.assertEquals(2, testPartition.getMaterializedIndices(IndexExtState.ALL).size()); + Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.SHADOW).size()); + + // runWaitingTxnJob + rollupHandler.runOneCycle(); + Assert.assertEquals(JobState.RUNNING, rollupJob.getJobState()); + + // runWaitingTxnJob, task not finished + rollupHandler.runOneCycle(); + Assert.assertEquals(JobState.RUNNING, rollupJob.getJobState()); + + // finish all tasks + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(3, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + MaterializedIndex shadowIndex = testPartition.getMaterializedIndices(IndexExtState.SHADOW).get(0); + for (Tablet shadowTablet : shadowIndex.getTablets()) { + for (Replica shadowReplica : shadowTablet.getReplicas()) { + shadowReplica.updateVersionInfo(testPartition.getVisibleVersion(), + testPartition.getVisibleVersionHash(), shadowReplica.getDataSize(), + shadowReplica.getRowCount()); + } + } + + rollupHandler.runOneCycle(); + Assert.assertEquals(JobState.FINISHED, rollupJob.getJobState()); + + /* + Assert.assertEquals(CatalogTestUtil.testIndexId1, rollupJob.getBaseIndexId()); + Assert.assertEquals(CatalogTestUtil.testRollupIndex2, rollupJob.getRollupIndexName()); + MaterializedIndex rollupIndex = rollupJob.getRollupIndex(CatalogTestUtil.testPartitionId1); + MaterializedIndex baseIndex = testPartition.getBaseIndex(); + assertEquals(IndexState.ROLLUP, rollupIndex.getState()); + assertEquals(IndexState.NORMAL, baseIndex.getState()); + assertEquals(OlapTableState.ROLLUP, olapTable.getState()); + assertEquals(PartitionState.ROLLUP, testPartition.getState()); + Tablet rollupTablet = rollupIndex.getTablets().get(0); + List replicas = rollupTablet.getReplicas(); + Replica rollupReplica1 = replicas.get(0); + Replica rollupReplica2 = replicas.get(1); + Replica rollupReplica3 = replicas.get(2); + + assertEquals(-1, rollupReplica1.getVersion()); + assertEquals(-1, rollupReplica2.getVersion()); + assertEquals(-1, rollupReplica3.getVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastFailedVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica2.getLastFailedVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica3.getLastFailedVersion()); + assertEquals(-1, rollupReplica1.getLastSuccessVersion()); + assertEquals(-1, rollupReplica2.getLastSuccessVersion()); + assertEquals(-1, rollupReplica3.getLastSuccessVersion()); + + // rollup handler run one cycle, agent task is generated and send tasks + rollupHandler.runOneCycle(); + AgentTask task1 = AgentTaskQueue.getTask(rollupReplica1.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); + AgentTask task2 = AgentTaskQueue.getTask(rollupReplica2.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); + AgentTask task3 = AgentTaskQueue.getTask(rollupReplica3.getBackendId(), TTaskType.ROLLUP, rollupTablet.getId()); + + // be report finishe rollup success + TTabletInfo tTabletInfo = new TTabletInfo(rollupTablet.getId(), CatalogTestUtil.testSchemaHash1, + CatalogTestUtil.testStartVersion, CatalogTestUtil.testStartVersionHash, 0, 0); + rollupHandler.handleFinishedReplica(task1, tTabletInfo, -1); + rollupHandler.handleFinishedReplica(task2, tTabletInfo, -1); + rollupHandler.handleFinishedReplica(task3, tTabletInfo, -1); + + // rollup hander run one cycle again, the rollup job is finishing + rollupHandler.runOneCycle(); + Assert.assertEquals(JobState.FINISHING, rollupJob.getState()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica2.getVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica3.getVersion()); + assertEquals(-1, rollupReplica1.getLastFailedVersion()); + assertEquals(-1, rollupReplica2.getLastFailedVersion()); + assertEquals(-1, rollupReplica3.getLastFailedVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastSuccessVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastSuccessVersion()); + assertEquals(CatalogTestUtil.testStartVersion, rollupReplica1.getLastSuccessVersion()); + */ + } + +} diff --git a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobTest.java b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobTest.java deleted file mode 100644 index f37704569b8447..00000000000000 --- a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobTest.java +++ /dev/null @@ -1,280 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.alter; - -import static org.junit.Assert.assertEquals; - -import org.apache.doris.alter.AlterJob.JobState; -import org.apache.doris.analysis.AccessTestUtil; -import org.apache.doris.analysis.AddColumnClause; -import org.apache.doris.analysis.AlterClause; -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ColumnDef; -import org.apache.doris.analysis.ColumnDef.DefaultValue; -import org.apache.doris.analysis.ColumnPosition; -import org.apache.doris.analysis.TypeDef; -import org.apache.doris.catalog.AggregateType; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.CatalogTestUtil; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.FakeCatalog; -import org.apache.doris.catalog.FakeEditLog; -import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.OlapTable.OlapTableState; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.Partition.PartitionState; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.Replica; -import org.apache.doris.catalog.Replica.ReplicaState; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Tablet; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.meta.MetaContext; -import org.apache.doris.task.AgentTask; -import org.apache.doris.task.AgentTaskQueue; -import org.apache.doris.thrift.TTabletInfo; -import org.apache.doris.thrift.TTaskType; -import org.apache.doris.transaction.FakeTransactionIDGenerator; -import org.apache.doris.transaction.GlobalTransactionMgr; -import org.apache.doris.transaction.TabletCommitInfo; -import org.apache.doris.transaction.TransactionState; -import org.apache.doris.transaction.TransactionState.LoadJobSourceType; -import org.apache.doris.transaction.TransactionStatus; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -public class SchemaChangeJobTest { - - private static FakeEditLog fakeEditLog; - private static FakeCatalog fakeCatalog; - private static FakeTransactionIDGenerator fakeTransactionIDGenerator; - private static GlobalTransactionMgr masterTransMgr; - private static GlobalTransactionMgr slaveTransMgr; - private static Catalog masterCatalog; - private static Catalog slaveCatalog; - - private String transactionSource = "localfe"; - private static Analyzer analyzer; - private static ColumnDef newCol = new ColumnDef("add_v", new TypeDef(ScalarType.createType(PrimitiveType.INT)), false, AggregateType.MAX, - false, new DefaultValue(true, "1"), ""); - private static AddColumnClause addColumnClause = new AddColumnClause(newCol, new ColumnPosition("v"), null, null); - - @Before - public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException, AnalysisException { - fakeEditLog = new FakeEditLog(); - fakeCatalog = new FakeCatalog(); - fakeTransactionIDGenerator = new FakeTransactionIDGenerator(); - masterCatalog = CatalogTestUtil.createTestCatalog(); - slaveCatalog = CatalogTestUtil.createTestCatalog(); - MetaContext metaContext = new MetaContext(); - metaContext.setMetaVersion(FeMetaVersion.VERSION_40); - metaContext.setThreadLocalInfo(); - - // masterCatalog.setJournalVersion(FeMetaVersion.VERSION_40); - // slaveCatalog.setJournalVersion(FeMetaVersion.VERSION_40); - masterTransMgr = masterCatalog.getGlobalTransactionMgr(); - masterTransMgr.setEditLog(masterCatalog.getEditLog()); - slaveTransMgr = slaveCatalog.getGlobalTransactionMgr(); - slaveTransMgr.setEditLog(slaveCatalog.getEditLog()); - analyzer = AccessTestUtil.fetchAdminAnalyzer(false); - addColumnClause.analyze(analyzer); - } - - @Test - public void testAddSchemaChange() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler(); - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(addColumnClause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - schemaChangeHandler.process(alterClauses, "default", db, olapTable); - SchemaChangeJob schemaChangeJob = (SchemaChangeJob) schemaChangeHandler - .getAlterJob(CatalogTestUtil.testTableId1); - Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); - } - - // start a schema change, then finished - @Test - public void testSchemaChange1() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler(); - - // add a schema change job - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(addColumnClause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - schemaChangeHandler.process(alterClauses, "default", db, olapTable); - SchemaChangeJob schemaChangeJob = (SchemaChangeJob) schemaChangeHandler - .getAlterJob(CatalogTestUtil.testTableId1); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); - assertEquals(IndexState.SCHEMA_CHANGE, baseIndex.getState()); - assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); - assertEquals(PartitionState.SCHEMA_CHANGE, testPartition.getState()); - Tablet baseTablet = baseIndex.getTablets().get(0); - List replicas = baseTablet.getReplicas(); - Replica replica1 = replicas.get(0); - Replica replica2 = replicas.get(1); - Replica replica3 = replicas.get(2); - - assertEquals(CatalogTestUtil.testStartVersion, replica1.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica2.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica3.getVersion()); - assertEquals(-1, replica1.getLastFailedVersion()); - assertEquals(-1, replica2.getLastFailedVersion()); - assertEquals(-1, replica3.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica2.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica3.getLastSuccessVersion()); - - // schemachange handler run one cycle, agent task is generated and send tasks - schemaChangeHandler.runOneCycle(); - AgentTask task1 = AgentTaskQueue.getTask(replica1.getBackendId(), TTaskType.SCHEMA_CHANGE, baseTablet.getId()); - AgentTask task2 = AgentTaskQueue.getTask(replica2.getBackendId(), TTaskType.SCHEMA_CHANGE, baseTablet.getId()); - AgentTask task3 = AgentTaskQueue.getTask(replica3.getBackendId(), TTaskType.SCHEMA_CHANGE, baseTablet.getId()); - - // be report finishe schema change success, report the new schema hash - TTabletInfo tTabletInfo = new TTabletInfo(baseTablet.getId(), - schemaChangeJob.getSchemaHashByIndexId(CatalogTestUtil.testIndexId1), CatalogTestUtil.testStartVersion, - CatalogTestUtil.testStartVersionHash, 0, 0); - schemaChangeHandler.handleFinishedReplica(task1, tTabletInfo, -1); - schemaChangeHandler.handleFinishedReplica(task2, tTabletInfo, -1); - schemaChangeHandler.handleFinishedReplica(task3, tTabletInfo, -1); - - // schema change hander run one cycle again, the rollup job is finishing - schemaChangeHandler.runOneCycle(); - Assert.assertEquals(JobState.FINISHING, schemaChangeJob.getState()); - assertEquals(CatalogTestUtil.testStartVersion, replica1.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica2.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica3.getVersion()); - assertEquals(-1, replica1.getLastFailedVersion()); - assertEquals(-1, replica2.getLastFailedVersion()); - assertEquals(-1, replica3.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica2.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion, replica3.getLastSuccessVersion()); - } - - // load some data and one replica has errors - // start a schema change and then load data - // load finished and schema change finished - @Test - public void testSchemaChange2() throws Exception { - FakeCatalog.setCatalog(masterCatalog); - SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler(); - // load one transaction with backend 2 has errors - long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, - CatalogTestUtil.testTxnLable1, - transactionSource, - LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second); - // commit a transaction, backend 2 has errors - TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId1); - TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId2); - TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(CatalogTestUtil.testTabletId1, - CatalogTestUtil.testBackendId3); - List transTablets = Lists.newArrayList(); - transTablets.add(tabletCommitInfo1); - transTablets.add(tabletCommitInfo2); - transTablets.add(tabletCommitInfo3); - masterTransMgr.commitTransaction(CatalogTestUtil.testDbId1, transactionId, transTablets); - TransactionState transactionState = fakeEditLog.getTransaction(transactionId); - assertEquals(TransactionStatus.COMMITTED, transactionState.getTransactionStatus()); - Set errorReplicaIds = Sets.newHashSet(); - // errorReplicaIds.add(CatalogTestUtil.testReplicaId2); - masterTransMgr.finishTransaction(transactionId, errorReplicaIds); - transactionState = fakeEditLog.getTransaction(transactionId); - assertEquals(TransactionStatus.VISIBLE, transactionState.getTransactionStatus()); - - // start a schema change - ArrayList alterClauses = new ArrayList<>(); - alterClauses.add(addColumnClause); - Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); - OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); - Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); - schemaChangeHandler.process(alterClauses, "default", db, olapTable); - SchemaChangeJob schemaChangeJob = (SchemaChangeJob) schemaChangeHandler - .getAlterJob(CatalogTestUtil.testTableId1); - MaterializedIndex baseIndex = testPartition.getBaseIndex(); - assertEquals(IndexState.SCHEMA_CHANGE, baseIndex.getState()); - assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); - assertEquals(PartitionState.SCHEMA_CHANGE, testPartition.getState()); - Tablet baseTablet = baseIndex.getTablets().get(0); - List replicas = baseTablet.getReplicas(); - Replica replica1 = replicas.get(0); - Replica replica2 = replicas.get(1); - Replica replica3 = replicas.get(2); - assertEquals(3, baseTablet.getReplicas().size()); - - assertEquals(ReplicaState.SCHEMA_CHANGE, replica1.getState()); - assertEquals(ReplicaState.SCHEMA_CHANGE, replica2.getState()); - assertEquals(ReplicaState.SCHEMA_CHANGE, replica3.getState()); - - assertEquals(CatalogTestUtil.testStartVersion + 1, replica1.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica2.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica3.getVersion()); - assertEquals(-1, replica1.getLastFailedVersion()); - assertEquals(-1, replica2.getLastFailedVersion()); - assertEquals(-1, replica3.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica2.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica3.getLastSuccessVersion()); - - // schemachange handler run one cycle, agent task is generated and send tasks - schemaChangeHandler.runOneCycle(); - AgentTask task1 = AgentTaskQueue.getTask(replica1.getBackendId(), TTaskType.SCHEMA_CHANGE, baseTablet.getId()); - AgentTask task2 = AgentTaskQueue.getTask(replica2.getBackendId(), TTaskType.SCHEMA_CHANGE, baseTablet.getId()); - AgentTask task3 = AgentTaskQueue.getTask(replica3.getBackendId(), TTaskType.SCHEMA_CHANGE, baseTablet.getId()); - - // be report finish schema change success, report the new schema hash - TTabletInfo tTabletInfo = new TTabletInfo(baseTablet.getId(), - schemaChangeJob.getSchemaHashByIndexId(CatalogTestUtil.testIndexId1), CatalogTestUtil.testStartVersion, - CatalogTestUtil.testStartVersionHash, 0, 0); - schemaChangeHandler.handleFinishedReplica(task1, tTabletInfo, -1); - schemaChangeHandler.handleFinishedReplica(task2, tTabletInfo, -1); - schemaChangeHandler.handleFinishedReplica(task3, tTabletInfo, -1); - - // rollup hander run one cycle again, the rollup job is finishing - schemaChangeHandler.runOneCycle(); - Assert.assertEquals(JobState.FINISHING, schemaChangeJob.getState()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica1.getVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica3.getVersion()); - assertEquals(-1, replica1.getLastFailedVersion()); - assertEquals(-1, replica3.getLastFailedVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica1.getLastSuccessVersion()); - assertEquals(CatalogTestUtil.testStartVersion + 1, replica3.getLastSuccessVersion()); - } -} diff --git a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java new file mode 100644 index 00000000000000..085d91bcb5feda --- /dev/null +++ b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import static org.junit.Assert.assertEquals; + +import org.apache.doris.alter.AlterJobV2.JobState; +import org.apache.doris.analysis.AccessTestUtil; +import org.apache.doris.analysis.AddColumnClause; +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.ColumnDef; +import org.apache.doris.analysis.ColumnDef.DefaultValue; +import org.apache.doris.analysis.ColumnPosition; +import org.apache.doris.analysis.TypeDef; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.FakeCatalog; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Partition.PartitionState; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.UserException; +import org.apache.doris.meta.MetaContext; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.transaction.FakeTransactionIDGenerator; +import org.apache.doris.transaction.GlobalTransactionMgr; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/* + * Author: Chenmingyu + * Date: Aug 21, 2019 + */ + +public class SchemaChangeJobV2Test { + + private static FakeEditLog fakeEditLog; + private static FakeCatalog fakeCatalog; + private static FakeTransactionIDGenerator fakeTransactionIDGenerator; + private static GlobalTransactionMgr masterTransMgr; + private static GlobalTransactionMgr slaveTransMgr; + private static Catalog masterCatalog; + private static Catalog slaveCatalog; + + private static Analyzer analyzer; + private static ColumnDef newCol = new ColumnDef("add_v", new TypeDef(ScalarType.createType(PrimitiveType.INT)), + false, AggregateType.MAX, false, new DefaultValue(true, "1"), ""); + private static AddColumnClause addColumnClause = new AddColumnClause(newCol, new ColumnPosition("v"), null, null); + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, AnalysisException { + fakeEditLog = new FakeEditLog(); + fakeCatalog = new FakeCatalog(); + fakeTransactionIDGenerator = new FakeTransactionIDGenerator(); + masterCatalog = CatalogTestUtil.createTestCatalog(); + slaveCatalog = CatalogTestUtil.createTestCatalog(); + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_60); + metaContext.setThreadLocalInfo(); + + masterTransMgr = masterCatalog.getGlobalTransactionMgr(); + masterTransMgr.setEditLog(masterCatalog.getEditLog()); + slaveTransMgr = slaveCatalog.getGlobalTransactionMgr(); + slaveTransMgr.setEditLog(slaveCatalog.getEditLog()); + analyzer = AccessTestUtil.fetchAdminAnalyzer(false); + addColumnClause.analyze(analyzer); + + FeConstants.runningUnitTest = true; + } + + @Test + public void testAddSchemaChange() throws UserException { + FakeCatalog.setCatalog(masterCatalog); + SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(addColumnClause); + Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); + OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); + schemaChangeHandler.process(alterClauses, "default_cluster", db, olapTable); + Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, alterJobsV2.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + } + + // start a schema change, then finished + @Test + public void testSchemaChange1() throws Exception { + FakeCatalog.setCatalog(masterCatalog); + SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler(); + + // add a schema change job + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(addColumnClause); + Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1); + OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1); + Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1); + schemaChangeHandler.process(alterClauses, "default_cluster", db, olapTable); + Map alterJobsV2 = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, alterJobsV2.size()); + SchemaChangeJobV2 schemaChangeJob = (SchemaChangeJobV2) alterJobsV2.values().stream().findAny().get(); + + MaterializedIndex baseIndex = testPartition.getBaseIndex(); + assertEquals(IndexState.NORMAL, baseIndex.getState()); + assertEquals(PartitionState.NORMAL, testPartition.getState()); + assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + Tablet baseTablet = baseIndex.getTablets().get(0); + List replicas = baseTablet.getReplicas(); + Replica replica1 = replicas.get(0); + Replica replica2 = replicas.get(1); + Replica replica3 = replicas.get(2); + + assertEquals(CatalogTestUtil.testStartVersion, replica1.getVersion()); + assertEquals(CatalogTestUtil.testStartVersion, replica2.getVersion()); + assertEquals(CatalogTestUtil.testStartVersion, replica3.getVersion()); + assertEquals(-1, replica1.getLastFailedVersion()); + assertEquals(-1, replica2.getLastFailedVersion()); + assertEquals(-1, replica3.getLastFailedVersion()); + assertEquals(CatalogTestUtil.testStartVersion, replica1.getLastSuccessVersion()); + assertEquals(CatalogTestUtil.testStartVersion, replica2.getLastSuccessVersion()); + assertEquals(CatalogTestUtil.testStartVersion, replica3.getLastSuccessVersion()); + + // runPendingJob + schemaChangeHandler.runOneCycle(); + Assert.assertEquals(JobState.WAITING_TXN, schemaChangeJob.getJobState()); + Assert.assertEquals(2, testPartition.getMaterializedIndices(IndexExtState.ALL).size()); + Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.VISIBLE).size()); + Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.SHADOW).size()); + + // runWaitingTxnJob + schemaChangeHandler.runOneCycle(); + Assert.assertEquals(JobState.RUNNING, schemaChangeJob.getJobState()); + + // runWaitingTxnJob, task not finished + schemaChangeHandler.runOneCycle(); + Assert.assertEquals(JobState.RUNNING, schemaChangeJob.getJobState()); + + // runRunningJob + schemaChangeHandler.runOneCycle(); + // task not finished, still running + Assert.assertEquals(JobState.RUNNING, schemaChangeJob.getJobState()); + + // finish alter tasks + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(3, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + MaterializedIndex shadowIndex = testPartition.getMaterializedIndices(IndexExtState.SHADOW).get(0); + for (Tablet shadowTablet : shadowIndex.getTablets()) { + for (Replica shadowReplica : shadowTablet.getReplicas()) { + shadowReplica.updateVersionInfo(testPartition.getVisibleVersion(), testPartition.getVisibleVersionHash(), shadowReplica.getDataSize(), shadowReplica.getRowCount()); + } + } + + schemaChangeHandler.runOneCycle(); + Assert.assertEquals(JobState.FINISHED, schemaChangeJob.getJobState()); + } + +} From b82afc26f9c91458314c7349717c37f8f48bcdbd Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 Aug 2019 22:34:17 +0800 Subject: [PATCH 56/79] modify log level --- .../org/apache/doris/transaction/GlobalTransactionMgr.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index 49c3ae38da4707..c83aa89a74468c 100644 --- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -758,8 +758,8 @@ public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId) continue; } if (entry.getKey() <= endTransactionId) { - LOG.debug("find a running txn with txn_id={}, less than schema change txn_id {}", - entry.getKey(), endTransactionId); + LOG.info("find a running txn with txn_id={} on db: {}, less than watermark txn_id {}", + entry.getKey(), dbId, endTransactionId); return false; } } From 29f69dd3e5c76bd88fc91979ef05edcca8935469 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 23 Aug 2019 19:15:58 +0800 Subject: [PATCH 57/79] fix ut --- fe/src/main/java/org/apache/doris/catalog/Catalog.java | 6 +----- .../org/apache/doris/alter/SchemaChangeJobV2Test.java | 1 + .../java/org/apache/doris/http/DorisHttpTestCase.java | 1 - .../apache/doris/load/loadv2/BrokerLoadJobTest.java | 7 +++---- .../java/org/apache/doris/load/loadv2/LoadJobTest.java | 10 ++++++++-- .../apache/doris/planner/StreamLoadScanNodeTest.java | 5 +++++ 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index a493f8d88803e4..074a2273091c5b 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -173,7 +173,6 @@ import org.apache.doris.system.HeartbeatMgr; import org.apache.doris.system.SystemInfoService; import org.apache.doris.task.AgentBatchTask; -import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.CreateReplicaTask; @@ -3301,10 +3300,7 @@ private Partition createPartitionWithIndices(String clusterName, long dbId, long if (!ok || !countDownLatch.getStatus().ok()) { errMsg = "Failed to create partition[" + partitionName + "]. Timeout."; // clear tasks - List tasks = batchTask.getAllTasks(); - for (AgentTask task : tasks) { - AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.CREATE, task.getSignature()); - } + AgentTaskQueue.removeBatchTask(batchTask, TTaskType.CREATE); if (!countDownLatch.getStatus().ok()) { errMsg += " Error: " + countDownLatch.getStatus().getErrorMsg(); diff --git a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java index 085d91bcb5feda..efc0319f95fa3e 100644 --- a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java +++ b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java @@ -105,6 +105,7 @@ public void setUp() throws InstantiationException, IllegalAccessException, Illeg addColumnClause.analyze(analyzer); FeConstants.runningUnitTest = true; + AgentTaskQueue.clearAllTasks(); } @Test diff --git a/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java index e6757ad4322333..ee02064ff52975 100644 --- a/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java +++ b/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java @@ -186,7 +186,6 @@ private static EsTable newEsTable(String name) { return table; } - private static Catalog newDelegateCatalog() { try { Catalog catalog = EasyMock.createMock(Catalog.class); diff --git a/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java b/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java index c308e2f72d8ffc..dec2c6b0331953 100644 --- a/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java +++ b/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java @@ -98,7 +98,7 @@ public void testFromLoadStmt(@Injectable LoadStmt loadStmt, } @Test - public void testFromLoadStmt(@Injectable LoadStmt loadStmt, + public void testFromLoadStmt2(@Injectable LoadStmt loadStmt, @Injectable DataDescription dataDescription, @Injectable LabelName labelName, @Injectable Database database, @@ -139,8 +139,8 @@ public void testFromLoadStmt(@Injectable LoadStmt loadStmt, new MockUp() { @Mock public void checkAndCreateSource(Database db, DataDescription dataDescription, - Map>> tableToPartitionSources, - boolean deleteFlag, EtlJobType jobType) { + Map>> tableToPartitionSources, EtlJobType jobType) { + } }; @@ -150,7 +150,6 @@ public void checkAndCreateSource(Database db, DataDescription dataDescription, Assert.assertEquals(label, Deencapsulation.getField(brokerLoadJob, "label")); Assert.assertEquals(JobState.PENDING, Deencapsulation.getField(brokerLoadJob, "state")); Assert.assertEquals(EtlJobType.BROKER, Deencapsulation.getField(brokerLoadJob, "jobType")); - Assert.assertEquals(dataDescriptionList, Deencapsulation.getField(brokerLoadJob, "dataDescriptions")); } catch (DdlException e) { Assert.fail(e.getMessage()); } diff --git a/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java b/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java index 3f5a72289c0b8b..0f32c130ffee08 100644 --- a/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java +++ b/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java @@ -169,9 +169,15 @@ public void testUpdateStateToLoading() { @Test public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo, - @Mocked LongCounterMetric longCounterMetric) { - metricRepo.COUNTER_LOAD_FINISHED = longCounterMetric; + @Mocked LongCounterMetric longCounterMetric) { + + MetricRepo.COUNTER_LOAD_FINISHED = longCounterMetric; LoadJob loadJob = new BrokerLoadJob(); + + // TxnStateCallbackFactory factory = Catalog.getCurrentCatalog().getGlobalTransactionMgr().getCallbackFactory(); + Catalog catalog = Catalog.getCurrentCatalog(); + GlobalTransactionMgr mgr = new GlobalTransactionMgr(catalog); + Deencapsulation.setField(catalog, "globalTransactionMgr", mgr); loadJob.updateState(JobState.FINISHED); Assert.assertEquals(JobState.FINISHED, loadJob.getState()); Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "finishTimestamp")); diff --git a/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java b/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java index 0ce5b1850a6cc0..24cbe98c6b4bae 100644 --- a/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java +++ b/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java @@ -153,6 +153,11 @@ public void testNormal() throws UserException { StreamLoadScanNode scanNode = getStreamLoadScanNode(dstDesc, request); new Expectations() {{ dstTable.getBaseSchema(); result = columns; + dstTable.getFullSchema(); result = columns; + dstTable.getColumn("k1"); result = columns.get(0); + dstTable.getColumn("k2"); result = columns.get(1); + dstTable.getColumn("v1"); result = columns.get(2); + dstTable.getColumn("v2"); result = columns.get(3); }}; scanNode.init(analyzer); scanNode.finalize(analyzer); From 7cbed356be87e2a143f3333688f5ef6908042112 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 23 Aug 2019 21:25:39 +0800 Subject: [PATCH 58/79] fix ut2 --- .../test/java/org/apache/doris/http/DorisHttpTestCase.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java index ee02064ff52975..00832c66064a3a 100644 --- a/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java +++ b/fe/src/test/java/org/apache/doris/http/DorisHttpTestCase.java @@ -86,7 +86,6 @@ abstract public class DorisHttpTestCase { public static final String DB_NAME = "testDb"; public static final String TABLE_NAME = "testTbl"; - private static long testBackendId1 = 1000; private static long testBackendId2 = 1001; private static long testBackendId3 = 1002; @@ -95,7 +94,6 @@ abstract public class DorisHttpTestCase { private static long testReplicaId2 = 2001; private static long testReplicaId3 = 2002; - private static long testDbId = 100L; private static long testTableId = 200L; private static long testPartitionId = 201L; @@ -112,10 +110,8 @@ abstract public class DorisHttpTestCase { protected static final String URI = "http://localhost:" + HTTP_PORT + "/api/" + DB_NAME + "/" + TABLE_NAME; - protected String rootAuth = Credentials.basic("root", ""); - public static OlapTable newTable(String name) { Catalog.getCurrentInvertedIndex().clear(); Column k1 = new Column("k1", PrimitiveType.BIGINT); @@ -158,7 +154,7 @@ public static OlapTable newTable(String name) { distributionInfo); table.addPartition(partition); table.setIndexSchemaInfo(testIndexId, "testIndex", columns, 0, testSchemaHash, (short) 1); - + table.setBaseIndexId(testIndexId); return table; } @@ -226,7 +222,6 @@ private static Catalog newDelegateCatalog() { Startup.initializeIfPossible(); } - private static void assignBackends() { Backend backend1 = new Backend(testBackendId1, "node-1", 9308); backend1.setBePort(9300); From d836723a893749249a0b3dc5a166f716550f5e31 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 27 Aug 2019 22:49:24 +0800 Subject: [PATCH 59/79] rebase master --- .../main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 2 +- .../test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index e6b9a4236344ca..2f492f89e76d54 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -508,7 +508,7 @@ public void write(DataOutput out) throws IOException { public void readFields(DataInput in) throws IOException { super.readFields(in); brokerDesc = BrokerDesc.read(in); - if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_60) { + if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_61) { dataSourceInfo.readFields(in); } diff --git a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java index efc0319f95fa3e..897507bc4e9421 100644 --- a/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java +++ b/fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java @@ -94,7 +94,7 @@ public void setUp() throws InstantiationException, IllegalAccessException, Illeg masterCatalog = CatalogTestUtil.createTestCatalog(); slaveCatalog = CatalogTestUtil.createTestCatalog(); MetaContext metaContext = new MetaContext(); - metaContext.setMetaVersion(FeMetaVersion.VERSION_60); + metaContext.setMetaVersion(FeMetaVersion.VERSION_61); metaContext.setThreadLocalInfo(); masterTransMgr = masterCatalog.getGlobalTransactionMgr(); From 8509701769c1d9d69128902de90fc2deecf72670 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 28 Aug 2019 16:12:08 +0800 Subject: [PATCH 60/79] fix by miaoling review --- .../sql-statements/Data Manipulation/BROKER LOAD.md | 2 +- .../org/apache/doris/load/loadv2/BrokerLoadJob.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md index 4063c28c537c0e..aca925de42e4d1 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Manipulation/BROKER LOAD.md @@ -133,7 +133,7 @@ max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 exec_mem_limit: 设置导入使用的内存上限。默认为2G，单位字节。这里是指单个 BE 节点的内存上限。 一个导入可能分布于多个BE。我们假设 1GB 数据在单个节点处理需要最大5GB内存。那么假设1GB文件分布在2个节点处理，那么理论上，每个节点需要内存为2.5GB。则该参数可以设置为 2684354560，即2.5GB - strict mode： 是否对数据进行严格限制。默认为true。 + strict mode： 是否对数据进行严格限制。默认为true。 timezone: 指定某些受时区影响的函数的时区，如 strftime/alignment_timestamp/from_unixtime 等等，具体请查阅 [时区] 文档。如果不指定，则使用 "Asia/Shanghai" 时区。 5. 导入数据格式样例 diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 2f492f89e76d54..5a1fac0afc6ffa 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -115,14 +115,14 @@ public static BrokerLoadJob fromLoadStmt(LoadStmt stmt, String originStmt) throw BrokerLoadJob brokerLoadJob = new BrokerLoadJob(db.getId(), stmt.getLabel().getLabelName(), stmt.getBrokerDesc(), originStmt); brokerLoadJob.setJobProperties(stmt.getProperties()); - brokerLoadJob.checkAndDataSourceInfo(db, stmt.getDataDescriptions()); + brokerLoadJob.checkAndSetDataSourceInfo(db, stmt.getDataDescriptions()); return brokerLoadJob; } catch (MetaNotFoundException e) { throw new DdlException(e.getMessage()); } } - private void checkAndDataSourceInfo(Database db, List dataDescriptions) throws DdlException { + private void checkAndSetDataSourceInfo(Database db, List dataDescriptions) throws DdlException { // check data source info db.readLock(); try { @@ -278,7 +278,7 @@ public void analyze() { if (db == null) { throw new DdlException("Database[" + dbId + "] does not exist"); } - checkAndDataSourceInfo(db, stmt.getDataDescriptions()); + checkAndSetDataSourceInfo(db, stmt.getDataDescriptions()); } catch (Exception e) { LOG.info(new LogBuilder(LogKey.LOAD_JOB, id) .add("origin_stmt", originStmt) @@ -340,6 +340,7 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme // divide job into broker loading task by table db.readLock(); try { + List newLoadingTasks = Lists.newArrayList(); for (Map.Entry> entry : dataSourceInfo.getIdToFileGroups().entrySet()) { long tableId = entry.getKey(); @@ -362,6 +363,9 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme TUniqueId loadId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); task.init(loadId, attachment.getFileStatusByTable(tableId), attachment.getFileNumByTable(tableId)); idToTasks.put(task.getSignature(), task); + // idToTasks contains previous LoadPendingTasks, so idToTasks is just used to save all tasks. + // use newLoadingTasks to save new created loading tasks and submit them later. + newLoadingTasks.add(task); loadStatistic.numLoadedRowsMap.put(loadId, new AtomicLong(0)); // save all related tables and rollups in transaction state @@ -372,7 +376,7 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme txnState.addTableIndexes(table); } // submit all tasks together - for (LoadTask loadTask : idToTasks.values()) { + for (LoadTask loadTask : newLoadingTasks) { Catalog.getCurrentCatalog().getLoadTaskScheduler().submit(loadTask); } } finally { From d8d46e3500b6c8ae23bd85cb55699d12e2755661 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 28 Aug 2019 21:33:00 +0800 Subject: [PATCH 61/79] fix compile bug --- fe/src/main/cup/sql_parser.cup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/cup/sql_parser.cup b/fe/src/main/cup/sql_parser.cup index c8620b07a20249..9b720b0fb065d5 100644 --- a/fe/src/main/cup/sql_parser.cup +++ b/fe/src/main/cup/sql_parser.cup @@ -1868,7 +1868,7 @@ show_param ::= :} | KW_TABLET INTEGER_LITERAL:tabletId {: - RESULT = new ShowTabletStmt(null, tabletId, null); + RESULT = new ShowTabletStmt(null, tabletId); :} | KW_TABLET KW_FROM table_name:dbTblName opt_partitions:partitionNames opt_wild_where order_by_clause:orderByClause limit_clause:limitClause {: From bace253d8f72d92b1e31bfe17cb4f4a430b4b9d8 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 29 Aug 2019 08:48:58 +0800 Subject: [PATCH 62/79] fix ALTER status bug --- fe/src/main/java/org/apache/doris/analysis/LoadStmt.java | 2 +- fe/src/main/java/org/apache/doris/catalog/Tablet.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java index 1cdd20b720025b..957e0dfff87661 100644 --- a/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -199,7 +199,7 @@ public static void checkProperties(Map properties) throws DdlExc } // time zone - final String timezone = properties.get(STRICT_MODE); + final String timezone = properties.get(TIMEZONE); if (timezone != null) { TimeUtils.checkTimeZoneValid(timezone); } diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/src/main/java/org/apache/doris/catalog/Tablet.java index 485053ae69e8fa..d0dff0fcaa9351 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java @@ -196,7 +196,7 @@ public Multimap getNormalReplicaBackendPathMap() { ReplicaState state = replica.getState(); if (infoService.checkBackendAlive(replica.getBackendId()) - && (state == ReplicaState.NORMAL || state == ReplicaState.SCHEMA_CHANGE)) { + && (state == ReplicaState.NORMAL || state == ReplicaState.ALTER)) { map.put(replica.getBackendId(), replica.getPathHash()); } } From cb5656d58a6161a8a0585efc25753c45b64f079a Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 29 Aug 2019 21:47:06 +0800 Subject: [PATCH 63/79] fix by review yiguolei --- be/src/olap/schema_change.cpp | 8 +++++--- .../java/org/apache/doris/alter/RollupJobV2.java | 2 +- .../apache/doris/task/HadoopLoadPendingTask.java | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 21d312eae4e90b..c8d6f203b1a145 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1337,9 +1337,11 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe } } while(0); - // _validate_alter_result should be outside the above while loop. - // to avoid requiring the header lock twice. - res = _validate_alter_result(new_tablet, request); + if (res == OLAP_SUCCESS) { + // _validate_alter_result should be outside the above while loop. + // to avoid requiring the header lock twice. + res = _validate_alter_result(new_tablet, request); + } // if failed convert history data, then just remove the new tablet if (res != OLAP_SUCCESS) { diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 1b837534af90d6..ce4899b8e05717 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -147,7 +147,7 @@ protected void runPendingJob() { LOG.info("begin to send create rollup replica tasks. job: {}", jobId); Database db = Catalog.getCurrentCatalog().getDb(dbId); if (db == null) { - cancelImpl("Databasee " + dbId + " does not exist"); + cancelImpl("Database " + dbId + " does not exist"); return; } diff --git a/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java b/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java index fb11b3c407ce62..8c22d9d51d8778 100644 --- a/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java +++ b/fe/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.HashDistributionInfo; @@ -41,6 +42,7 @@ import org.apache.doris.load.Source; import org.apache.doris.load.TableLoadInfo; import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.transaction.TransactionState; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -67,7 +69,6 @@ public HadoopLoadPendingTask(LoadJob job) { @Override protected void createEtlRequest() throws Exception { - // yiguolei: add a db read lock here? because the schema maybe changed during create etl task db.readLock(); try { EtlTaskConf taskConf = new EtlTaskConf(); @@ -88,6 +89,19 @@ protected void createEtlRequest() throws Exception { etlTaskConf = taskConf.toDppTaskConf(); Preconditions.checkNotNull(etlTaskConf); + + // add table indexes to transaction state + TransactionState txnState = Catalog.getCurrentGlobalTransactionMgr().getTransactionState(job.getTransactionId()); + if (txnState == null) { + throw new LoadException("txn does not exist: " + job.getTransactionId()); + } + for (long tableId : job.getIdToTableLoadInfo().keySet()) { + OlapTable table = (OlapTable) db.getTable(tableId); + if (table == null) { + throw new LoadException("table does not exist. id: " + tableId); + } + txnState.addTableIndexes(table); + } } finally { db.readUnlock(); } From 18a847cc0b9081518fc57dfe57884b0a787f8863 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 30 Aug 2019 11:06:38 +0800 Subject: [PATCH 64/79] change FeConstants metaversion to 61 --- .../en/administrator-guide/time-zone_EN.md | 72 +++++++++++++++++++ .../org/apache/doris/common/FeConstants.java | 2 +- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 docs/documentation/en/administrator-guide/time-zone_EN.md diff --git a/docs/documentation/en/administrator-guide/time-zone_EN.md b/docs/documentation/en/administrator-guide/time-zone_EN.md new file mode 100644 index 00000000000000..1ba66682d5105a --- /dev/null +++ b/docs/documentation/en/administrator-guide/time-zone_EN.md @@ -0,0 +1,72 @@ +# Time zone + +Doris supports multiple time zone settings + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. + +## Basic concepts + +There are multiple time zone related parameters in Doris + +* `system_time_zone`: + +When the server starts, it will be set automatically according to the time zone set by the machine, which can not be modified after setting. + +* `time_zone`: + +Server current time zone, set it at session level or global level. + +## Specific operations + +1. `SHOW VARIABLES LIKE '% time_zone%'` + + View the current time zone related configuration + +2. `SET time_zone = 'Asia/Shanghai'` + + This command can set the session level time zone, which will fail after disconnection. + +3. `SET global time_zone = 'Asia/Shanghai'` + + This command can set time zone parameters at the global level. The FE will persist the parameters and will not fail when the connection is disconnected. + +### Impact of time zone + +Time zone setting affects the display and storage of time zone sensitive values. + +It includes the values displayed by time functions such as `NOW()` or `CURTIME()`, as well as the time values in `SHOW LOAD` and `SHOW BACKENDS` statements. + +However, it does not affect the `LESS THAN VALUE` of the time-type partition column in the `CREATE TABLE` statement, nor does it affect the display of values stored as `DATE/DATETIME` type. + +Functions affected by time zone: + +* `FROM_UNIXTIME`: Given a UTC timestamp, return the date and time of the specified time zone, such as `FROM_UNIXTIME(0)`, return the CST time zone: `1970-01-08:00`. + +* `UNIX_TIMESTAMP`: Given a specified time zone date and time, return UTC timestamp, such as CST time zone `UNIX_TIMESTAMP('1970-01 08:00:00')`, return `0`. + +* `CURTIME`: Returns the datetime of specified time zone. + +* `NOW`: Returns the specified date and time of specified time zone. + +* `CONVERT_TZ`: Converts a date and time from one specified time zone to another. + +## Restrictions + +Time zone values can be given in several formats, case-insensitive: + +* A string representing UTC offset, such as '+10:00' or '-6:00'. + +* Standard time zone formats, such as "Asia/Shanghai", "America/Los_Angeles" + +* Abbreviated time zone formats such as MET and CTT are not supported. Because the abbreviated time zone is ambiguous in different scenarios, it is not recommended to use it. + +* In order to be compatible with Doris and support CST abbreviated time zone, CST will be internally transferred to "Asia/Shanghai", which is Chinese standard time zone. + +## Time zone format list + +[List of TZ database time zones] (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) + +[Edit on GitHub](https://github.com/apache/incubator-doris/blob/master/docs/documentation/en/administrator-guide/time-zone_EN.md) \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/common/FeConstants.java b/fe/src/main/java/org/apache/doris/common/FeConstants.java index 7fc7cfc13a0f2b..79e2add21d541c 100644 --- a/fe/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/src/main/java/org/apache/doris/common/FeConstants.java @@ -38,5 +38,5 @@ public class FeConstants { // general model // Current meta data version. Use this version to write journals and image - public static int meta_version = FeMetaVersion.VERSION_60; + public static int meta_version = FeMetaVersion.VERSION_61; } From 02e9b8d4475d3bd782231c76d0c92f07321a64d4 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 3 Sep 2019 10:48:18 +0800 Subject: [PATCH 65/79] fix init column bug --- .../doris/analysis/DataDescription.java | 51 +-- .../apache/doris/load/BrokerFileGroup.java | 87 ++-- .../main/java/org/apache/doris/load/Load.java | 397 ++++++++++++++++-- .../doris/load/loadv2/BrokerLoadJob.java | 3 +- .../org/apache/doris/load/loadv2/LoadJob.java | 13 - .../apache/doris/planner/BrokerScanNode.java | 259 +----------- .../doris/planner/StreamLoadScanNode.java | 91 +--- .../org/apache/doris/task/StreamLoadTask.java | 2 +- .../doris/analysis/DataDescriptionTest.java | 2 +- 9 files changed, 456 insertions(+), 449 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index d1d346f8f9ce04..1968ee5fc0dc39 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -74,18 +74,21 @@ public class DataDescription { "default_value", "md5sum", "replace_value", - "now", "hll_hash", + "now", + "hll_hash", "substitute"); private final String tableName; private final List partitionNames; private final List filePaths; - // the column name list of data desc - private List columns; private final ColumnSeparator columnSeparator; private final String fileFormat; - private final List columnsFromPath; private final boolean isNegative; + + // column names of source files + private List fileFieldNames; + // column names in the path + private final List columnsFromPath; // save column mapping in SET(xxx = xxx) clause private final List columnMappingList; @@ -93,10 +96,8 @@ public class DataDescription { private TNetworkAddress beAddr; private String lineDelimiter; - /* - * Merged from columns and columnMappingList - * ImportColumnDesc: column name to expr or null - */ + // Merged from fileFieldNames, columnsFromPath and columnMappingList + // ImportColumnDesc: column name to (expr or null) private List parsedColumnExprList = Lists.newArrayList(); /* * This param only include the hadoop function which need to be checked in the future. @@ -130,7 +131,7 @@ public DataDescription(String tableName, this.tableName = tableName; this.partitionNames = partitionNames; this.filePaths = filePaths; - this.columns = columns; + this.fileFieldNames = columns; this.columnSeparator = columnSeparator; this.fileFormat = fileFormat; this.columnsFromPath = columnsFromPath; @@ -150,12 +151,11 @@ public List getFilePaths() { return filePaths; } - // only return the column names of SlotRef in columns - public List getColumnNames() { - if (columns == null || columns.isEmpty()) { + public List getFileFieldNames() { + if (fileFieldNames == null || fileFieldNames.isEmpty()) { return null; } - return columns; + return fileFieldNames; } public String getFileFormat() { @@ -193,6 +193,7 @@ public void setLineDelimiter(String lineDelimiter) { this.lineDelimiter = lineDelimiter; } + @Deprecated public void addColumnMapping(String functionName, Pair> pair) { if (Strings.isNullOrEmpty(functionName) || pair == null) { return; @@ -216,20 +217,20 @@ public boolean isHadoopLoad() { return isHadoopLoad; } - /** + /* * Analyze parsedExprMap and columnToHadoopFunction from columns, columns from path and columnMappingList * Example: * columns (col1, tmp_col2, tmp_col3) * columns from path as (col4, col5) * set (col2=tmp_col2+1, col3=strftime("%Y-%m-%d %H:%M:%S", tmp_col3)) - * Result: - * + * + * Result: * parsedExprMap = {"col1": null, "tmp_col2": null, "tmp_col3": null, "col4": null, "col5": null, * "col2": "tmp_col2+1", "col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} * columnToHadoopFunction = {"col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"} */ private void analyzeColumns() throws AnalysisException { - if ((columns == null || columns.isEmpty()) && (columnsFromPath != null && !columnsFromPath.isEmpty())) { + if ((fileFieldNames == null || fileFieldNames.isEmpty()) && (columnsFromPath != null && !columnsFromPath.isEmpty())) { throw new AnalysisException("Can not specify columns_from_path without column_list"); } @@ -238,8 +239,8 @@ private void analyzeColumns() throws AnalysisException { // merge columns exprs from columns, columns from path and columnMappingList // 1. analyze columns - if (columns != null && !columns.isEmpty()) { - for (String columnName : columns) { + if (fileFieldNames != null && !fileFieldNames.isEmpty()) { + for (String columnName : fileFieldNames) { if (!columnNames.add(columnName)) { throw new AnalysisException("Duplicate column: " + columnName); } @@ -563,11 +564,11 @@ public void analyzeWithoutCheckPriv() throws AnalysisException { * */ public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlException { - if (columns != null && !columns.isEmpty()) { + if (fileFieldNames != null && !fileFieldNames.isEmpty()) { return; } - columns = Lists.newArrayList(); + fileFieldNames = Lists.newArrayList(); Set mappingColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (ImportColumnDesc importColumnDesc : parsedColumnExprList) { @@ -578,10 +579,10 @@ public void fillColumnInfoIfNotSpecified(List baseSchema) throws DdlExce if (!mappingColNames.contains(column.getName())) { parsedColumnExprList.add(new ImportColumnDesc(column.getName(), null)); } - columns.add(column.getName()); + fileFieldNames.add(column.getName()); } - LOG.debug("after fill column info. columns: {}, parsed column exprs: {}", columns, parsedColumnExprList); + LOG.debug("after fill column info. columns: {}, parsed column exprs: {}", fileFieldNames, parsedColumnExprList); } public String toSql() { @@ -608,9 +609,9 @@ public String apply(String s) { sb.append(" COLUMNS FROM PATH AS ("); Joiner.on(", ").appendTo(sb, columnsFromPath).append(")"); } - if (columns != null && !columns.isEmpty()) { + if (fileFieldNames != null && !fileFieldNames.isEmpty()) { sb.append(" ("); - Joiner.on(", ").appendTo(sb, columns).append(")"); + Joiner.on(", ").appendTo(sb, fileFieldNames).append(")"); } if (columnMappingList != null && !columnMappingList.isEmpty()) { sb.append(" SET ("); diff --git a/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java b/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java index b0ce4e21ca36b5..bc1b62386c8f3e 100644 --- a/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java +++ b/fe/src/main/java/org/apache/doris/load/BrokerFileGroup.java @@ -21,15 +21,20 @@ import org.apache.doris.analysis.DataDescription; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.BrokerTable; import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; @@ -53,25 +58,22 @@ public class BrokerFileGroup implements Writable { private static final Logger LOG = LogManager.getLogger(BrokerFileGroup.class); - // input - private DataDescription dataDescription; - private long tableId; private String valueSeparator; private String lineDelimiter; // fileFormat may be null, which means format will be decided by file's suffix private String fileFormat; - private List columnsFromPath; private boolean isNegative; private List partitionIds; - private List fileFieldNames; private List filePaths; - // this is a compatible param which only happens before the function of broker has been supported. - @Deprecated - private Map exprColumnMap; + private List fileFieldNames; + private List columnsFromPath; + // columnExprList includes all fileFieldNames, columnsFromPath and column mappings // this param will be recreated by data desc when the log replay private List columnExprList; + // this is only for hadoop function check + private Map>> columnToHadoopFunction; // Used for recovery from edit log private BrokerFileGroup() { @@ -88,24 +90,23 @@ public BrokerFileGroup(BrokerTable table) throws AnalysisException { } public BrokerFileGroup(DataDescription dataDescription) { - this.dataDescription = dataDescription; + this.fileFieldNames = dataDescription.getFileFieldNames(); this.columnsFromPath = dataDescription.getColumnsFromPath(); - this.exprColumnMap = null; - this.fileFieldNames = dataDescription.getColumnNames(); this.columnExprList = dataDescription.getParsedColumnExprList(); + this.columnToHadoopFunction = dataDescription.getColumnToHadoopFunction(); } // NOTE: DBLock will be held // This will parse the input DataDescription to list for BrokerFileInfo - public void parse(Database db) throws DdlException { + public void parse(Database db, DataDescription dataDescription) throws DdlException { // tableId Table table = db.getTable(dataDescription.getTableName()); if (table == null) { - throw new DdlException("Unknown table(" + dataDescription.getTableName() - + ") in database(" + db.getFullName() + ")"); + throw new DdlException("Unknown table " + dataDescription.getTableName() + + " in database " + db.getFullName()); } if (!(table instanceof OlapTable)) { - throw new DdlException("Table(" + table.getName() + ") is not OlapTable"); + throw new DdlException("Table " + table.getName() + " is not OlapTable"); } OlapTable olapTable = (OlapTable) table; tableId = table.getId(); @@ -119,13 +120,29 @@ public void parse(Database db) throws DdlException { for (String pName : dataDescription.getPartitionNames()) { Partition partition = olapTable.getPartition(pName); if (partition == null) { - throw new DdlException("Unknown partition(" + pName + ") in table(" - + table.getName() + ")"); + throw new DdlException("Unknown partition" + pName + " in table" + table.getName()); } partitionIds.add(partition.getId()); } } + if (olapTable.getState() == OlapTableState.RESTORE) { + throw new DdlException("Table [" + table.getName() + "] is under restore"); + } + + if (olapTable.getKeysType() != KeysType.AGG_KEYS && dataDescription.isNegative()) { + throw new DdlException("Load for AGG_KEYS table should not specify NEGATIVE"); + } + + // check negative for sum aggregate type + if (dataDescription.isNegative()) { + for (Column column : table.getBaseSchema()) { + if (!column.isKey() && column.getAggregationType() != AggregateType.SUM) { + throw new DdlException("Column is not SUM AggreateType. column:" + column.getName()); + } + } + } + // column valueSeparator = dataDescription.getColumnSeparator(); if (valueSeparator == null) { @@ -139,7 +156,7 @@ public void parse(Database db) throws DdlException { fileFormat = dataDescription.getFileFormat(); if (fileFormat != null) { if (!fileFormat.toLowerCase().equals("parquet") && !fileFormat.toLowerCase().equals("csv")) { - throw new DdlException("File Format Type("+fileFormat+") Is Invalid. Only support 'csv' or 'parquet'"); + throw new DdlException("File Format Type "+fileFormat+" is invalid. Only support 'csv' or 'parquet'"); } } isNegative = dataDescription.isNegative(); @@ -164,10 +181,6 @@ public String getFileFormat() { return fileFormat; } - public List getColumnsFromPath() { - return columnsFromPath; - } - public boolean isNegative() { return isNegative; } @@ -180,14 +193,18 @@ public List getFilePaths() { return filePaths; } - public List getFileFieldNames() { - return fileFieldNames; + public List getColumnsFromPath() { + return columnsFromPath; } public List getColumnExprList() { return columnExprList; } + public Map>> getColumnToHadoopFunction() { + return columnToHadoopFunction; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -278,16 +295,8 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, path); } // expr column map will be null after broker load supports function - if (exprColumnMap == null) { - out.writeInt(0); - } else { - int size = exprColumnMap.size(); - out.writeInt(size); - for (Map.Entry entry : exprColumnMap.entrySet()) { - Text.writeString(out, entry.getKey()); - Expr.writeTo(entry.getValue(), out); - } - } + out.writeInt(0); + // fileFormat if (fileFormat == null) { out.writeBoolean(false); @@ -332,14 +341,12 @@ public void readFields(DataInput in) throws IOException { } } // expr column map + Map exprColumnMap = Maps.newHashMap(); { int size = in.readInt(); - if (size > 0) { - exprColumnMap = Maps.newHashMap(); - for (int i = 0; i < size; ++i) { - final String name = Text.readString(in); - exprColumnMap.put(name, Expr.readIn(in)); - } + for (int i = 0; i < size; ++i) { + final String name = Text.readString(in); + exprColumnMap.put(name, Expr.readIn(in)); } } // file format diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index dfe57c592450ad..73a0bcb240248a 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -18,19 +18,28 @@ package org.apache.doris.load; import org.apache.doris.alter.SchemaChangeHandler; +import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CancelLoadStmt; import org.apache.doris.analysis.ColumnSeparator; import org.apache.doris.analysis.DataDescription; import org.apache.doris.analysis.DeleteStmt; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprSubstitutionMap; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionName; +import org.apache.doris.analysis.FunctionParams; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.LabelName; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.LoadStmt; +import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.Predicate; +import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.backup.BlobStorage; import org.apache.doris.backup.Status; import org.apache.doris.catalog.AggregateType; @@ -47,6 +56,7 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Tablet; @@ -65,6 +75,7 @@ import org.apache.doris.common.LoadException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.AsyncDeleteJob.DeleteState; @@ -79,6 +90,7 @@ import org.apache.doris.task.AgentClient; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.PushTask; +import org.apache.doris.thrift.TBrokerScanRangeParams; import org.apache.doris.thrift.TEtlState; import org.apache.doris.thrift.TMiniLoadRequest; import org.apache.doris.thrift.TNetworkAddress; @@ -484,11 +496,11 @@ private LoadJob createLoadJob(LoadStmt stmt, EtlJobType etlJobType, PullLoadSourceInfo sourceInfo = new PullLoadSourceInfo(); for (DataDescription dataDescription : dataDescriptions) { BrokerFileGroup fileGroup = new BrokerFileGroup(dataDescription); - fileGroup.parse(db); + fileGroup.parse(db, dataDescription); sourceInfo.addFileGroup(fileGroup); } job.setPullLoadSourceInfo(sourceInfo); - LOG.info("Source info is {}", sourceInfo); + LOG.info("source info is {}", sourceInfo); } if (etlJobType == EtlJobType.MINI) { @@ -592,7 +604,7 @@ private LoadJob createLoadJob(LoadStmt stmt, EtlJobType etlJobType, } /* - * This is used for both hadoop load and broker load v2 + * This is only used for hadoop load */ public static void checkAndCreateSource(Database db, DataDescription dataDescription, Map>> tableToPartitionSources, EtlJobType jobType) throws DdlException { @@ -642,8 +654,8 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // source columns List columnNames = Lists.newArrayList(); List assignColumnNames = Lists.newArrayList(); - if (dataDescription.getColumnNames() != null) { - assignColumnNames.addAll(dataDescription.getColumnNames()); + if (dataDescription.getFileFieldNames() != null) { + assignColumnNames.addAll(dataDescription.getFileFieldNames()); if (dataDescription.getColumnsFromPath() != null) { assignColumnNames.addAll(dataDescription.getColumnsFromPath()); } @@ -719,36 +731,40 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = substitute(B)); for (Column column : table.getFullSchema()) { if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { - /* - * There is a case that if user does not specify the related origin column, eg: - * COLUMNS (A, C), and B is not specified, but B is being modified so there is a shadow column '__doris_shadow_B'. - * We can not just add a mapping function "__doris_shadow_B = substitute(B)", because Doris can not find column B. - * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping - */ String originCol = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); - Expr mappingExpr = parsedColumnExprMap.get(originCol); - if (mappingExpr != null) { - /* - * eg: - * (A, C) SET (B = func(xx)) - * -> - * (A, C) SET (B = func(xx), __doris_shadow_B = func(xxx)) - */ - if (columnToHadoopFunction.containsKey(originCol)) { - columnToHadoopFunction.put(column.getName(), columnToHadoopFunction.get(originCol)); + if (parsedColumnExprMap.containsKey(originCol)) { + Expr mappingExpr = parsedColumnExprMap.get(originCol); + if (mappingExpr != null) { + /* + * eg: + * (A, C) SET (B = func(xx)) + * -> + * (A, C) SET (B = func(xx), __doris_shadow_B = func(xxx)) + */ + if (columnToHadoopFunction.containsKey(originCol)) { + columnToHadoopFunction.put(column.getName(), columnToHadoopFunction.get(originCol)); + } + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), mappingExpr); + parsedColumnExprList.add(importColumnDesc); + } else { + /* + * eg: + * (A, B, C) + * -> + * (A, B, C) SET (__doris_shadow_B = substitute(B)) + */ + columnToHadoopFunction.put(column.getName(), Pair.create("substitute", Lists.newArrayList(originCol))); + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, originCol)); + parsedColumnExprList.add(importColumnDesc); } - ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), mappingExpr); - parsedColumnExprList.add(importColumnDesc); } else { /* - * eg: - * (A, B, C) - * -> - * (A, B, C) SET (__doris_shadow_B = substitute(B)) + * There is a case that if user does not specify the related origin column, eg: + * COLUMNS (A, C), and B is not specified, but B is being modified so there is a shadow column '__doris_shadow_B'. + * We can not just add a mapping function "__doris_shadow_B = substitute(B)", because Doris can not find column B. + * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping */ - columnToHadoopFunction.put(column.getName(), Pair.create("substitute", Lists.newArrayList(originCol))); - ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, originCol)); - parsedColumnExprList.add(importColumnDesc); + // do nothing } } @@ -844,6 +860,327 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip } } + /* + * This function will do followings: + * 1. fill the column exprs if user does not specify any column or column mapping. + * 2. For not specified columns, check if they have default value. + * 3. Add any shadow columns if have. + * 4. validate hadoop functions + * 5. init slot descs and expr map for load plan + * + * This function should be used for broker load v2 and stream load. + * And it must be called in same db lock when planing. + */ + public static void initColumns(Table tbl, List columnExprs, + Map>> columnToHadoopFunction, + Map exprsByName, Analyzer analyzer, TupleDescriptor srcTupleDesc, + Map slotDescByName, TBrokerScanRangeParams params) throws UserException { + // If user does not specify the column expr descs, generate it by using base schema of table. + // So that the following process can be unified + if (columnExprs == null || columnExprs.isEmpty()) { + List columns = tbl.getBaseSchema(); + for (Column column : columns) { + ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName()); + LOG.debug("add base column {} to stream load task", column.getName()); + columnExprs.add(columnDesc); + } + } + // generate a map for checking easily + Map columnExprMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + for (ImportColumnDesc importColumnDesc : columnExprs) { + columnExprMap.put(importColumnDesc.getColumnName(), importColumnDesc.getExpr()); + } + + // check default value + for (Column column : tbl.getBaseSchema()) { + String columnName = column.getName(); + if (columnExprMap.containsKey(columnName)) { + continue; + } + if (column.getDefaultValue() != null || column.isAllowNull()) { + continue; + } + throw new DdlException("Column has no default value. column: " + columnName); + } + + // When doing schema change, there may have some 'shadow' columns, with prefix '__doris_shadow_' in + // their names. These columns are invisible to user, but we need to generate data for these columns. + // So we add column mappings for these column. + // eg1: + // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' + // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = substitute(B)); + for (Column column : tbl.getFullSchema()) { + if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + String originCol = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); + if (columnExprMap.containsKey(originCol)) { + Expr mappingExpr = columnExprMap.get(originCol); + if (mappingExpr != null) { + /* + * eg: + * (A, C) SET (B = func(xx)) + * -> + * (A, C) SET (B = func(xx), __doris_shadow_B = func(xxx)) + */ + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), mappingExpr); + columnExprs.add(importColumnDesc); + } else { + /* + * eg: + * (A, B, C) + * -> + * (A, B, C) SET (__doris_shadow_B = B) + */ + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), + new SlotRef(null, originCol)); + columnExprs.add(importColumnDesc); + } + } else { + /* + * There is a case that if user does not specify the related origin column, eg: + * COLUMNS (A, C), and B is not specified, but B is being modified so there is a shadow column '__doris_shadow_B'. + * We can not just add a mapping function "__doris_shadow_B = substitute(B)", because Doris can not find column B. + * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping + */ + // do nothing + } + } + } + + // validate hadoop functions + if (columnToHadoopFunction != null) { + Map columnNameMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + for (ImportColumnDesc importColumnDesc : columnExprs) { + if (importColumnDesc.isColumn()) { + columnNameMap.put(importColumnDesc.getColumnName(), importColumnDesc.getColumnName()); + } + } + for (Entry>> entry : columnToHadoopFunction.entrySet()) { + String mappingColumnName = entry.getKey(); + Column mappingColumn = tbl.getColumn(mappingColumnName); + if (mappingColumn == null) { + throw new DdlException("Mapping column is not in table. column: " + mappingColumnName); + } + + Pair> function = entry.getValue(); + try { + DataDescription.validateMappingFunction(function.first, function.second, columnNameMap, + mappingColumn, false); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + } + } + + // init slot desc add expr map, also transform hadoop functions + for (ImportColumnDesc importColumnDesc : columnExprs) { + // make column name case match with real column name + String columnName = importColumnDesc.getColumnName(); + String realColName = tbl.getColumn(columnName) == null ? columnName + : tbl.getColumn(columnName).getName(); + if (importColumnDesc.getExpr() != null) { + Expr expr = transformHadoopFunctionExpr(tbl, realColName, importColumnDesc.getExpr()); + exprsByName.put(realColName, expr); + } else { + SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); + slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); + slotDesc.setIsMaterialized(true); + // ISSUE A: src slot should be nullable even if the column is not nullable. + // because src slot is what we read from file, not represent to real column value. + // If column is not nullable, error will be thrown when filling the dest slot, + // which is not nullable. + slotDesc.setIsNullable(true); + params.addToSrc_slot_ids(slotDesc.getId().asInt()); + slotDescByName.put(realColName, slotDesc); + } + } + LOG.debug("slotDescByName: {}, exprsByName: {}", slotDescByName, exprsByName); + + // analyze all exprs + for (Map.Entry entry : exprsByName.entrySet()) { + ExprSubstitutionMap smap = new ExprSubstitutionMap(); + List slots = Lists.newArrayList(); + entry.getValue().collect(SlotRef.class, slots); + for (SlotRef slot : slots) { + SlotDescriptor slotDesc = slotDescByName.get(slot.getColumnName()); + if (slotDesc == null) { + throw new UserException("unknown reference column, column=" + entry.getKey() + + ", reference=" + slot.getColumnName()); + } + smap.getLhs().add(slot); + smap.getRhs().add(new SlotRef(slotDesc)); + } + Expr expr = entry.getValue().clone(smap); + expr.analyze(analyzer); + + // check if contain aggregation + List funcs = Lists.newArrayList(); + expr.collect(FunctionCallExpr.class, funcs); + for (FunctionCallExpr fn : funcs) { + if (fn.isAggregateFunction()) { + throw new AnalysisException("Don't support aggregation function in load expression"); + } + } + exprsByName.put(entry.getKey(), expr); + } + LOG.debug("after init column, exprMap: {}", exprsByName); + } + + /** + * This method is used to transform hadoop function. + * The hadoop function includes: replace_value, strftime, time_format, alignment_timestamp, default_value, now. + * It rewrites those function with real function name and param. + * For the other function, the expr only go through this function and the origin expr is returned. + * + * @param columnName + * @param originExpr + * @return + * @throws UserException + */ + private static Expr transformHadoopFunctionExpr(Table tbl, String columnName, Expr originExpr) + throws UserException { + Column column = tbl.getColumn(columnName); + if (column == null) { + // the unknown column will be checked later. + return originExpr; + } + + // To compatible with older load version + if (originExpr instanceof FunctionCallExpr) { + FunctionCallExpr funcExpr = (FunctionCallExpr) originExpr; + String funcName = funcExpr.getFnName().getFunction(); + + if (funcName.equalsIgnoreCase("replace_value")) { + List exprs = Lists.newArrayList(); + SlotRef slotRef = new SlotRef(null, columnName); + // We will convert this to IF(`col` != child0, `col`, child1), + // because we need the if return type equal to `col`, we use NE + + /* + * We will convert this based on different cases: + * case 1: k1 = replace_value(null, anyval); + * to: k1 = if (k1 is not null, k1, anyval); + * + * case 2: k1 = replace_value(anyval1, anyval2); + * to: k1 = if (k1 is not null, if(k1 != anyval1, k1, anyval2), null); + */ + if (funcExpr.getChild(0) instanceof NullLiteral) { + // case 1 + exprs.add(new IsNullPredicate(slotRef, true)); + exprs.add(slotRef); + if (funcExpr.hasChild(1)) { + exprs.add(funcExpr.getChild(1)); + } else { + if (column.getDefaultValue() != null) { + exprs.add(new StringLiteral(column.getDefaultValue())); + } else { + if (column.isAllowNull()) { + exprs.add(NullLiteral.create(Type.VARCHAR)); + } else { + throw new UserException("Column(" + columnName + ") has no default value."); + } + } + } + } else { + // case 2 + exprs.add(new IsNullPredicate(slotRef, true)); + List innerIfExprs = Lists.newArrayList(); + innerIfExprs.add(new BinaryPredicate(BinaryPredicate.Operator.NE, slotRef, funcExpr.getChild(0))); + innerIfExprs.add(slotRef); + if (funcExpr.hasChild(1)) { + innerIfExprs.add(funcExpr.getChild(1)); + } else { + if (column.getDefaultValue() != null) { + innerIfExprs.add(new StringLiteral(column.getDefaultValue())); + } else { + if (column.isAllowNull()) { + innerIfExprs.add(NullLiteral.create(Type.VARCHAR)); + } else { + throw new UserException("Column(" + columnName + ") has no default value."); + } + } + } + FunctionCallExpr innerIfFn = new FunctionCallExpr("if", innerIfExprs); + exprs.add(innerIfFn); + exprs.add(NullLiteral.create(Type.VARCHAR)); + } + + LOG.debug("replace_value expr: {}", exprs); + FunctionCallExpr newFn = new FunctionCallExpr("if", exprs); + return newFn; + } else if (funcName.equalsIgnoreCase("strftime")) { + // FROM_UNIXTIME(val) + FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); + List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); + FunctionCallExpr fromUnixFunc = new FunctionCallExpr( + fromUnixName, new FunctionParams(false, fromUnixArgs)); + + return fromUnixFunc; + } else if (funcName.equalsIgnoreCase("time_format")) { + // DATE_FORMAT(STR_TO_DATE(dt_str, dt_fmt)) + FunctionName strToDateName = new FunctionName("STR_TO_DATE"); + List strToDateExprs = Lists.newArrayList(funcExpr.getChild(2), funcExpr.getChild(1)); + FunctionCallExpr strToDateFuncExpr = new FunctionCallExpr( + strToDateName, new FunctionParams(false, strToDateExprs)); + + FunctionName dateFormatName = new FunctionName("DATE_FORMAT"); + List dateFormatArgs = Lists.newArrayList(strToDateFuncExpr, funcExpr.getChild(0)); + FunctionCallExpr dateFormatFunc = new FunctionCallExpr( + dateFormatName, new FunctionParams(false, dateFormatArgs)); + + return dateFormatFunc; + } else if (funcName.equalsIgnoreCase("alignment_timestamp")) { + /* + * change to: + * UNIX_TIMESTAMP(DATE_FORMAT(FROM_UNIXTIME(ts), "%Y-01-01 00:00:00")); + * + */ + + // FROM_UNIXTIME + FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); + List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); + FunctionCallExpr fromUnixFunc = new FunctionCallExpr( + fromUnixName, new FunctionParams(false, fromUnixArgs)); + + // DATE_FORMAT + StringLiteral precision = (StringLiteral) funcExpr.getChild(0); + StringLiteral format; + if (precision.getStringValue().equalsIgnoreCase("year")) { + format = new StringLiteral("%Y-01-01 00:00:00"); + } else if (precision.getStringValue().equalsIgnoreCase("month")) { + format = new StringLiteral("%Y-%m-01 00:00:00"); + } else if (precision.getStringValue().equalsIgnoreCase("day")) { + format = new StringLiteral("%Y-%m-%d 00:00:00"); + } else if (precision.getStringValue().equalsIgnoreCase("hour")) { + format = new StringLiteral("%Y-%m-%d %H:00:00"); + } else { + throw new UserException("Unknown precision(" + precision.getStringValue() + ")"); + } + FunctionName dateFormatName = new FunctionName("DATE_FORMAT"); + List dateFormatArgs = Lists.newArrayList(fromUnixFunc, format); + FunctionCallExpr dateFormatFunc = new FunctionCallExpr( + dateFormatName, new FunctionParams(false, dateFormatArgs)); + + // UNIX_TIMESTAMP + FunctionName unixTimeName = new FunctionName("UNIX_TIMESTAMP"); + List unixTimeArgs = Lists.newArrayList(); + unixTimeArgs.add(dateFormatFunc); + FunctionCallExpr unixTimeFunc = new FunctionCallExpr( + unixTimeName, new FunctionParams(false, unixTimeArgs)); + + return unixTimeFunc; + } else if (funcName.equalsIgnoreCase("default_value")) { + return funcExpr.getChild(0); + } else if (funcName.equalsIgnoreCase("now")) { + FunctionName nowFunctionName = new FunctionName("NOW"); + FunctionCallExpr newFunc = new FunctionCallExpr(nowFunctionName, new FunctionParams(null)); + return newFunc; + } else if (funcName.equalsIgnoreCase("substitute")) { + return funcExpr.getChild(0); + } + } + return originExpr; + } + public void unprotectAddLoadJob(LoadJob job, boolean isReplay) throws DdlException { long jobId = job.getId(); long dbId = job.getDbId(); diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 5a1fac0afc6ffa..7a5f6829b26086 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -126,10 +126,9 @@ private void checkAndSetDataSourceInfo(Database db, List dataDe // check data source info db.readLock(); try { - LoadJob.checkDataSourceInfo(db, dataDescriptions, EtlJobType.BROKER); for (DataDescription dataDescription : dataDescriptions) { BrokerFileGroup fileGroup = new BrokerFileGroup(dataDescription); - fileGroup.parse(db); + fileGroup.parse(db, dataDescription); dataSourceInfo.addFileGroup(fileGroup); } } finally { diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index aff28c52d0a6f8..d769f8b4233392 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -17,7 +17,6 @@ package org.apache.doris.load.loadv2; -import org.apache.doris.analysis.DataDescription; import org.apache.doris.analysis.LoadStmt; import org.apache.doris.catalog.AuthorizationInfo; import org.apache.doris.catalog.Catalog; @@ -40,7 +39,6 @@ import org.apache.doris.load.EtlStatus; import org.apache.doris.load.FailMsg; import org.apache.doris.load.Load; -import org.apache.doris.load.Source; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.privilege.PaloPrivilege; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -311,17 +309,6 @@ protected void setJobProperties(Map properties) throws DdlExcept } } - protected static void checkDataSourceInfo(Database db, List dataDescriptions, - EtlJobType jobType) throws DdlException { - for (DataDescription dataDescription : dataDescriptions) { - // loadInfo is a temporary param for the method of checkAndCreateSource. - // >> - Map>> loadInfo = Maps.newHashMap(); - // only support broker load now - Load.checkAndCreateSource(db, dataDescription, loadInfo, jobType); - } - } - public void isJobTypeRead(boolean jobTypeRead) { isJobTypeRead = jobTypeRead; } diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 988491a98a45ee..27639dfc75b036 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -17,19 +17,12 @@ package org.apache.doris.planner; -import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ArithmeticExpr; -import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.FunctionName; -import org.apache.doris.analysis.FunctionParams; -import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; @@ -41,7 +34,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FsBroker; import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -49,6 +41,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.load.BrokerFileGroup; +import org.apache.doris.load.Load; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TBrokerRangeDesc; @@ -125,7 +118,6 @@ public int compare(TBrokerFileStatus o1, TBrokerFileStatus o2) { private int nextBe = 0; private Analyzer analyzer; - private List partitionExprs; private static class ParamCreateContext { public BrokerFileGroup fileGroup; @@ -166,16 +158,18 @@ public void init(Analyzer analyzer) throws UserException { getFileStatusAndCalcInstance(); paramCreateContexts = Lists.newArrayList(); + int i = 0; for (BrokerFileGroup fileGroup : fileGroups) { ParamCreateContext context = new ParamCreateContext(); context.fileGroup = fileGroup; context.timezone = analyzer.getTimezone(); try { - initParams(context); + initParams(context, fileStatusesList.get(i)); } catch (AnalysisException e) { throw new UserException(e.getMessage()); } paramCreateContexts.add(context); + ++i; } } @@ -207,7 +201,8 @@ public void setLoadInfo(long loadJobId, } // Called from init, construct source tuple information - private void initParams(ParamCreateContext context) throws AnalysisException, UserException { + private void initParams(ParamCreateContext context, List fileStatus) + throws AnalysisException, UserException { TBrokerScanRangeParams params = new TBrokerScanRangeParams(); context.params = params; @@ -229,247 +224,12 @@ private void initParams(ParamCreateContext context) throws AnalysisException, Us * @throws UserException */ private void initColumns(ParamCreateContext context) throws UserException { - List sourceFileColumns = context.fileGroup.getFileFieldNames(); - List pathColumns = context.fileGroup.getColumnsFromPath(); - List originColumnNameToExprList = context.fileGroup.getColumnExprList(); - // originColumnNameToExprList must has elements. because it is always filled by user or by system - Preconditions.checkState(originColumnNameToExprList != null && !originColumnNameToExprList.isEmpty()); - - // 1. create source slots, from sourceFileColumns and pathColumns context.tupleDescriptor = analyzer.getDescTbl().createTupleDescriptor(); context.slotDescByName = Maps.newHashMap(); - List allColumns = Lists.newArrayList(sourceFileColumns); - if (pathColumns != null) { - allColumns.addAll(pathColumns); - } - for (String sourceColName : allColumns) { - SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(context.tupleDescriptor); - slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); - slotDesc.setIsMaterialized(true); - // src slot should be nullable even if the column is not nullable. - // because src slot is what we read from file, not represent to real column value. - // If column is not nullable, error will be thrown when filling the dest slot, - // which is not nullable - slotDesc.setIsNullable(true); - context.params.addToSrc_slot_ids(slotDesc.getId().asInt()); - String realColName = targetTable.getColumn(sourceColName) == null ? sourceColName - : targetTable.getColumn(sourceColName).getName(); - context.slotDescByName.put(realColName, slotDesc); - } - context.params.setSrc_tuple_id(context.tupleDescriptor.getId().asInt()); - - // 2. handle column mapping exprs context.exprMap = Maps.newHashMap(); - for (ImportColumnDesc originColumnNameToExpr : originColumnNameToExprList) { - String columnName = originColumnNameToExpr.getColumnName(); - Expr columnExpr = originColumnNameToExpr.getExpr(); - Column col = targetTable.getColumn(columnName); - if (col == null) { - // maybe 1) shadow column, 2) unknown column - if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { - /* - * The shadow column mapping expr is added when creating load job. - * But the load job is being actually scheduled, the schema change may already finished. - * So the shadow column may not be found here. - * We can just ignore this shadow column's mapping expr, like it does not exist. - */ - continue; - } else if (columnExpr == null) { - // this is a unknown column, but the column expr is null, so we just consider it as - // a placeholder column, ignore it - continue; - } - // unknown column but has column expr, which is not allowed. - throw new UserException("Unknown column(" + columnName + ")"); - } - Preconditions.checkNotNull(col, columnName); - String realColName = col.getName(); - if (columnExpr != null) { - columnExpr = transformHadoopFunctionExpr(columnName, columnExpr, context.timezone); - context.exprMap.put(realColName, columnExpr); - } - } - // analyze all column mapping exprs - for (Map.Entry entry : context.exprMap.entrySet()) { - ExprSubstitutionMap smap = new ExprSubstitutionMap(); - List slots = Lists.newArrayList(); - entry.getValue().collect(SlotRef.class, slots); - for (SlotRef slot : slots) { - SlotDescriptor slotDesc = context.slotDescByName.get(slot.getColumnName()); - if (slotDesc == null) { - throw new UserException("unknown reference column, column=" + entry.getKey() - + ", reference=" + slot.getColumnName()); - } - smap.getLhs().add(slot); - smap.getRhs().add(new SlotRef(slotDesc)); - } - Expr expr = entry.getValue().clone(smap); - expr.analyze(analyzer); - - // check if contain aggregation - List funcs = Lists.newArrayList(); - expr.collect(FunctionCallExpr.class, funcs); - for (FunctionCallExpr fn : funcs) { - if (fn.isAggregateFunction()) { - throw new AnalysisException("Don't support aggregation function in load expression"); - } - } - context.exprMap.put(entry.getKey(), expr); - } - LOG.debug("after init column, exprMap: {}", context.exprMap); - } - - /** - * This method is used to transform hadoop function. - * The hadoop function includes: replace_value, strftime, time_format, alignment_timestamp, default_value, now. - * It rewrites those function with real function name and param. - * For the other function, the expr only go through this function and the origin expr is returned. - * @param columnName - * @param originExpr - * @return - * @throws UserException - */ - private Expr transformHadoopFunctionExpr(String columnName, Expr originExpr, String timezone) throws UserException { - Column column = targetTable.getColumn(columnName); - if (column == null) { - throw new UserException("Unknown column(" + columnName + ")"); - } - - // To compatible with older load version - if (originExpr instanceof FunctionCallExpr) { - FunctionCallExpr funcExpr = (FunctionCallExpr) originExpr; - String funcName = funcExpr.getFnName().getFunction(); - - if (funcName.equalsIgnoreCase("replace_value")) { - List exprs = Lists.newArrayList(); - SlotRef slotRef = new SlotRef(null, columnName); - // We will convert this to IF(`col` != child0, `col`, child1), - // because we need the if return type equal to `col`, we use NE - - /* - * We will convert this based on different cases: - * case 1: k1 = replace_value(null, anyval); - * to: k1 = if (k1 is not null, k1, anyval); - * - * case 2: k1 = replace_value(anyval1, anyval2); - * to: k1 = if (k1 is not null, if(k1 != anyval1, k1, anyval2), null); - */ - if (funcExpr.getChild(0) instanceof NullLiteral) { - // case 1 - exprs.add(new IsNullPredicate(slotRef, true)); - exprs.add(slotRef); - if (funcExpr.hasChild(1)) { - exprs.add(funcExpr.getChild(1)); - } else { - if (column.getDefaultValue() != null) { - exprs.add(new StringLiteral(column.getDefaultValue())); - } else { - if (column.isAllowNull()) { - exprs.add(NullLiteral.create(Type.VARCHAR)); - } else { - throw new UserException("Column(" + columnName + ") has no default value."); - } - } - } - } else { - // case 2 - exprs.add(new IsNullPredicate(slotRef, true)); - List innerIfExprs = Lists.newArrayList(); - innerIfExprs.add(new BinaryPredicate(BinaryPredicate.Operator.NE, slotRef, funcExpr.getChild(0))); - innerIfExprs.add(slotRef); - if (funcExpr.hasChild(1)) { - innerIfExprs.add(funcExpr.getChild(1)); - } else { - if (column.getDefaultValue() != null) { - innerIfExprs.add(new StringLiteral(column.getDefaultValue())); - } else { - if (column.isAllowNull()) { - innerIfExprs.add(NullLiteral.create(Type.VARCHAR)); - } else { - throw new UserException("Column(" + columnName + ") has no default value."); - } - } - } - FunctionCallExpr innerIfFn = new FunctionCallExpr("if", innerIfExprs); - exprs.add(innerIfFn); - exprs.add(NullLiteral.create(Type.VARCHAR)); - } - - LOG.debug("replace_value expr: {}", exprs); - FunctionCallExpr newFn = new FunctionCallExpr("if", exprs); - return newFn; - } else if (funcName.equalsIgnoreCase("strftime")) { - // FROM_UNIXTIME(val) - FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); - List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); - FunctionCallExpr fromUnixFunc = new FunctionCallExpr( - fromUnixName, new FunctionParams(false, fromUnixArgs)); - - return fromUnixFunc; - } else if (funcName.equalsIgnoreCase("time_format")) { - // DATE_FORMAT(STR_TO_DATE(dt_str, dt_fmt)) - FunctionName strToDateName = new FunctionName("STR_TO_DATE"); - List strToDateExprs = Lists.newArrayList(funcExpr.getChild(2), funcExpr.getChild(1)); - FunctionCallExpr strToDateFuncExpr = new FunctionCallExpr( - strToDateName, new FunctionParams(false, strToDateExprs)); - - FunctionName dateFormatName = new FunctionName("DATE_FORMAT"); - List dateFormatArgs = Lists.newArrayList(strToDateFuncExpr, funcExpr.getChild(0)); - FunctionCallExpr dateFormatFunc = new FunctionCallExpr( - dateFormatName, new FunctionParams(false, dateFormatArgs)); - - return dateFormatFunc; - } else if (funcName.equalsIgnoreCase("alignment_timestamp")) { - /* - * change to: - * UNIX_TIMESTAMP(DATE_FORMAT(FROM_UNIXTIME(ts), "%Y-01-01 00:00:00")); - * - */ - - // FROM_UNIXTIME - FunctionName fromUnixName = new FunctionName("FROM_UNIXTIME"); - List fromUnixArgs = Lists.newArrayList(funcExpr.getChild(1)); - FunctionCallExpr fromUnixFunc = new FunctionCallExpr( - fromUnixName, new FunctionParams(false, fromUnixArgs)); - - // DATE_FORMAT - StringLiteral precision = (StringLiteral) funcExpr.getChild(0); - StringLiteral format; - if (precision.getStringValue().equalsIgnoreCase("year")) { - format = new StringLiteral("%Y-01-01 00:00:00"); - } else if (precision.getStringValue().equalsIgnoreCase("month")) { - format = new StringLiteral("%Y-%m-01 00:00:00"); - } else if (precision.getStringValue().equalsIgnoreCase("day")) { - format = new StringLiteral("%Y-%m-%d 00:00:00"); - } else if (precision.getStringValue().equalsIgnoreCase("hour")) { - format = new StringLiteral("%Y-%m-%d %H:00:00"); - } else { - throw new UserException("Unknown precision(" + precision.getStringValue() + ")"); - } - FunctionName dateFormatName = new FunctionName("DATE_FORMAT"); - List dateFormatArgs = Lists.newArrayList(fromUnixFunc, format); - FunctionCallExpr dateFormatFunc = new FunctionCallExpr( - dateFormatName, new FunctionParams(false, dateFormatArgs)); - - // UNIX_TIMESTAMP - FunctionName unixTimeName = new FunctionName("UNIX_TIMESTAMP"); - List unixTimeArgs = Lists.newArrayList(); - unixTimeArgs.add(dateFormatFunc); - FunctionCallExpr unixTimeFunc = new FunctionCallExpr( - unixTimeName, new FunctionParams(false, unixTimeArgs)); - - return unixTimeFunc; - } else if (funcName.equalsIgnoreCase("default_value")) { - return funcExpr.getChild(0); - } else if (funcName.equalsIgnoreCase("now")) { - FunctionName nowFunctionName = new FunctionName("NOW"); - FunctionCallExpr newFunc = new FunctionCallExpr(nowFunctionName, new FunctionParams(null)); - return newFunc; - } else if (funcName.equalsIgnoreCase("substitute")) { - return funcExpr.getChild(0); - } - } - return originExpr; + Load.initColumns(targetTable, context.fileGroup.getColumnExprList(), + context.fileGroup.getColumnToHadoopFunction(), + context.exprMap, analyzer, context.tupleDescriptor, context.slotDescByName, context.params); } private void finalizeParams(ParamCreateContext context) throws UserException, AnalysisException { @@ -535,6 +295,7 @@ private void finalizeParams(ParamCreateContext context) throws UserException, An context.params.putToExpr_of_dest_slot(destSlotDesc.getId().asInt(), expr.treeToThrift()); } context.params.setDest_sid_to_src_sid_without_trans(destSidToSrcSidWithoutTrans); + context.params.setSrc_tuple_id(context.tupleDescriptor.getId().asInt()); context.params.setDest_tuple_id(desc.getId().asInt()); context.params.setStrict_mode(strictMode); // Need re compute memory layout after set some slot descriptor to nullable diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 6eeec812e74b28..3a4a6a54128440 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -17,13 +17,11 @@ package org.apache.doris.planner; -import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotDescriptor; @@ -33,11 +31,11 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.load.Load; import org.apache.doris.task.StreamLoadTask; import org.apache.doris.thrift.TBrokerRangeDesc; import org.apache.doris.thrift.TBrokerScanNode; @@ -50,7 +48,6 @@ import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TUniqueId; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -122,90 +119,8 @@ public void init(Analyzer analyzer) throws UserException { TBrokerScanRangeParams params = new TBrokerScanRangeParams(); params.setStrict_mode(streamLoadTask.isStrictMode()); - // parse columns header. this contain map from input column to column of destination table - // columns: k1, k2, v1, v2=k1 + k2 - // this means that there are three columns(k1, k2, v1) in source file, - // and v2 is derived from (k1 + k2) - - // If user does not specify the column expr descs, generate it by using base schema of table. - // So that the following process can be unified - if (streamLoadTask.getColumnExprDescs() == null || streamLoadTask.getColumnExprDescs().isEmpty()) { - List columns = dstTable.getBaseSchema(); - for (Column column : columns) { - ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName()); - LOG.debug("add base column {} to stream load task", column.getName()); - streamLoadTask.addColumnExprDesc(columnDesc); - } - } - - // When doing schema change, there may have some 'shadow' columns, with prefix '__doris_shadow_' in - // their names. These columns are visible to user, but we need to generate data for these columns. - // So we add column mappings for these column. - // eg: - // base schema is (A, B, C), and B is under schema change, so there will be a shadow column: '__doris_shadow_B' - // So the final column mapping should looks like: (A, B, C, __doris_shadow_B = B); - List fullSchema = dstTable.getFullSchema(); - for (Column column : fullSchema) { - if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { - String baseColName = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); - ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName(), new SlotRef(null, baseColName)); - LOG.debug("add shadow column {} to stream load task, base name: {}", column.getName(), baseColName); - streamLoadTask.addColumnExprDesc(columnDesc); - } - } - Preconditions.checkState(streamLoadTask.getColumnExprDescs() != null); - Preconditions.checkState(!streamLoadTask.getColumnExprDescs().isEmpty()); - - for (ImportColumnDesc importColumnDesc : streamLoadTask.getColumnExprDescs()) { - // make column name case match with real column name - String columnName = importColumnDesc.getColumnName(); - String realColName = dstTable.getColumn(columnName) == null ? columnName - : dstTable.getColumn(columnName).getName(); - if (importColumnDesc.getExpr() != null) { - exprsByName.put(realColName, importColumnDesc.getExpr()); - } else { - SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(srcTupleDesc); - slotDesc.setType(ScalarType.createType(PrimitiveType.VARCHAR)); - slotDesc.setIsMaterialized(true); - // ISSUE A: src slot should be nullable even if the column is not nullable. - // because src slot is what we read from file, not represent to real column value. - // If column is not nullable, error will be thrown when filling the dest slot, - // which is not nullable. - slotDesc.setIsNullable(true); - params.addToSrc_slot_ids(slotDesc.getId().asInt()); - slotDescByName.put(realColName, slotDesc); - } - } - - LOG.debug("slotDescByName: {}, exprsByName: {}", slotDescByName, exprsByName); - - // analyze all exprs - for (Map.Entry entry : exprsByName.entrySet()) { - ExprSubstitutionMap smap = new ExprSubstitutionMap(); - List slots = Lists.newArrayList(); - entry.getValue().collect(SlotRef.class, slots); - for (SlotRef slot : slots) { - SlotDescriptor slotDesc = slotDescByName.get(slot.getColumnName()); - if (slotDesc == null) { - throw new UserException("unknown reference column, column=" + entry.getKey() - + ", reference=" + slot.getColumnName()); - } - smap.getLhs().add(slot); - smap.getRhs().add(new SlotRef(slotDesc)); - } - Expr expr = entry.getValue().clone(smap); - expr.analyze(analyzer); - - // check if contain aggregation - List funcs = Lists.newArrayList(); - expr.collect(FunctionCallExpr.class, funcs); - for (FunctionCallExpr fn : funcs) { - if (fn.isAggregateFunction()) { - throw new AnalysisException("Don't support aggregation function in load expression"); - } - } - exprsByName.put(entry.getKey(), expr); - } + Load.initColumns(dstTable, streamLoadTask.getColumnExprDescs(), null /* no hadoop function */, + exprsByName, analyzer, srcTupleDesc, slotDescByName, params); // analyze where statement if (streamLoadTask.getWhereExpr() != null) { diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index 999cddb4afbf49..fbd80fea18318d 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -52,7 +52,7 @@ public class StreamLoadTask { private TFileFormatType formatType; // optional - private List columnExprDescs; + private List columnExprDescs = Lists.newArrayList(); private Expr whereExpr; private ColumnSeparator columnSeparator; private String partitions; diff --git a/fe/src/test/java/org/apache/doris/analysis/DataDescriptionTest.java b/fe/src/test/java/org/apache/doris/analysis/DataDescriptionTest.java index a01cac0c26f2b0..6a1a2808b7236a 100644 --- a/fe/src/test/java/org/apache/doris/analysis/DataDescriptionTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/DataDescriptionTest.java @@ -76,7 +76,7 @@ public void testNormal() throws AnalysisException { desc.analyze("testDb"); Assert.assertEquals("DATA INFILE ('abc.txt') NEGATIVE INTO TABLE testTable (col1, col2)", desc.toString()); Assert.assertEquals("testTable", desc.getTableName()); - Assert.assertEquals("[col1, col2]", desc.getColumnNames().toString()); + Assert.assertEquals("[col1, col2]", desc.getFileFieldNames().toString()); Assert.assertEquals("[abc.txt]", desc.getFilePaths().toString()); Assert.assertTrue(desc.isNegative()); Assert.assertNull(desc.getColumnSeparator()); From ac6f411285442d65254c3592f926d9e3c0c56d19 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 3 Sep 2019 16:21:51 +0800 Subject: [PATCH 66/79] fix not drop old tablet bug --- .../java/org/apache/doris/alter/SchemaChangeJobV2.java | 8 +++++++- .../java/org/apache/doris/analysis/SlotDescriptor.java | 6 ++++++ .../main/java/org/apache/doris/clone/TabletScheduler.java | 2 +- .../main/java/org/apache/doris/task/StreamLoadTask.java | 3 +-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index ba87b502b915dd..dee1889c3d54e2 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -470,7 +470,13 @@ private void onFinished(OlapTable tbl) { // in catalog. MaterializedIndex shadowIdx = partition.getIndex(shadowIdxId); Preconditions.checkNotNull(shadowIdx, shadowIdxId); - partition.deleteRollupIndex(originIdxId); + MaterializedIndex droppedIdx = partition.deleteRollupIndex(originIdxId); + + // delete origin replicas + for (Tablet originTablet : droppedIdx.getTablets()) { + Catalog.getCurrentInvertedIndex().deleteTablet(originTablet.getId()); + } + // set replica state for (Tablet tablet : shadowIdx.getTablets()) { for (Replica replica : tablet.getReplicas()) { diff --git a/fe/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 96fd0de5c158e7..1d2feb5e9374ec 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.ColumnStats; import org.apache.doris.catalog.Type; import org.apache.doris.thrift.TSlotDescriptor; + import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -263,4 +264,9 @@ public String debugString() { .add("nullIndicatorBit", nullIndicatorBit) .add("slotIdx", slotIdx).toString(); } + + @Override + public String toString() { + return debugString(); + } } diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java index 680a22e142f5d7..feebc40aa06247 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -442,7 +442,7 @@ private synchronized TabletSchedCtx takeRunningTablets(long tabletId) { * Try to schedule a single tablet. */ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException { - LOG.debug("schedule tablet: {}", tabletCtx.getTabletId()); + LOG.debug("schedule tablet: {}, type: {}, status: {}", tabletCtx.getTabletId(), tabletCtx.getType(), tabletCtx.getTabletStatus()); long currentTime = System.currentTimeMillis(); tabletCtx.setLastSchedTime(currentTime); tabletCtx.setLastVisitedTime(currentTime); diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index fbd80fea18318d..c30af73edb2178 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -174,8 +174,6 @@ private void setOptionalFromRoutineLoadJob(RoutineLoadJob routineLoadJob) { // so we keep the columnExprDescs in routine load job as origin. if (routineLoadJob.getColumnDescs() != null) { columnExprDescs = Lists.newArrayList(routineLoadJob.getColumnDescs()); - } else { - columnExprDescs = null; } whereExpr = routineLoadJob.getWhereExpr(); columnSeparator = routineLoadJob.getColumnSeparator(); @@ -183,6 +181,7 @@ private void setOptionalFromRoutineLoadJob(RoutineLoadJob routineLoadJob) { strictMode = routineLoadJob.isStrictMode(); } + // used for stream load private void setColumnToColumnExpr(String columns) throws UserException { String columnsSQL = new String("COLUMNS (" + columns + ")"); SqlParser parser = new SqlParser(new SqlScanner(new StringReader(columnsSQL))); From da2ac3c35916003d4c18f9e69e422d8e799f8434 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 3 Sep 2019 20:16:29 +0800 Subject: [PATCH 67/79] fix unknown column name --- fe/src/main/java/org/apache/doris/load/Load.java | 6 +++--- .../java/org/apache/doris/planner/BrokerScanNode.java | 8 +++++--- .../java/org/apache/doris/planner/StreamLoadScanNode.java | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 73a0bcb240248a..5290b2d39c93a6 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -871,13 +871,13 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip * This function should be used for broker load v2 and stream load. * And it must be called in same db lock when planing. */ - public static void initColumns(Table tbl, List columnExprs, + public static void initColumns(Table tbl, boolean specifyFileFieldNames, List columnExprs, Map>> columnToHadoopFunction, Map exprsByName, Analyzer analyzer, TupleDescriptor srcTupleDesc, Map slotDescByName, TBrokerScanRangeParams params) throws UserException { - // If user does not specify the column expr descs, generate it by using base schema of table. + // If user does not specify the file field names, generate it by using base schema of table. // So that the following process can be unified - if (columnExprs == null || columnExprs.isEmpty()) { + if (!specifyFileFieldNames) { List columns = tbl.getBaseSchema(); for (Column column : columns) { ImportColumnDesc columnDesc = new ImportColumnDesc(column.getName()); diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 27639dfc75b036..9679b44a8f4422 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -227,9 +227,11 @@ private void initColumns(ParamCreateContext context) throws UserException { context.tupleDescriptor = analyzer.getDescTbl().createTupleDescriptor(); context.slotDescByName = Maps.newHashMap(); context.exprMap = Maps.newHashMap(); - Load.initColumns(targetTable, context.fileGroup.getColumnExprList(), - context.fileGroup.getColumnToHadoopFunction(), - context.exprMap, analyzer, context.tupleDescriptor, context.slotDescByName, context.params); + + boolean specifyFileFieldNames = context.fileGroup.getColumnExprList().stream().anyMatch(p -> p.isColumn()); + Load.initColumns(targetTable, specifyFileFieldNames, context.fileGroup.getColumnExprList(), + context.fileGroup.getColumnToHadoopFunction(), context.exprMap, analyzer, context.tupleDescriptor, + context.slotDescByName, context.params); } private void finalizeParams(ParamCreateContext context) throws UserException, AnalysisException { diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 3a4a6a54128440..01e47f021ca8fd 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -119,8 +119,9 @@ public void init(Analyzer analyzer) throws UserException { TBrokerScanRangeParams params = new TBrokerScanRangeParams(); params.setStrict_mode(streamLoadTask.isStrictMode()); - Load.initColumns(dstTable, streamLoadTask.getColumnExprDescs(), null /* no hadoop function */, - exprsByName, analyzer, srcTupleDesc, slotDescByName, params); + boolean specifyFileFieldNames = streamLoadTask.getColumnExprDescs().stream().anyMatch(p -> p.isColumn()); + Load.initColumns(dstTable, specifyFileFieldNames, streamLoadTask.getColumnExprDescs(), + null /* no hadoop function */, exprsByName, analyzer, srcTupleDesc, slotDescByName, params); // analyze where statement if (streamLoadTask.getWhereExpr() != null) { From 0a2f632a5e2ad4bfd34d9b926d511bc72353c8d2 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 3 Sep 2019 20:46:10 +0800 Subject: [PATCH 68/79] fix drop origin index NPE --- be/src/exec/tablet_sink.h | 2 +- .../apache/doris/alter/SchemaChangeJobV2.java | 16 +++++++++++----- .../java/org/apache/doris/catalog/OlapTable.java | 8 +------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/be/src/exec/tablet_sink.h b/be/src/exec/tablet_sink.h index 6ceeac63d5db11..00776046c03319 100644 --- a/be/src/exec/tablet_sink.h +++ b/be/src/exec/tablet_sink.h @@ -219,7 +219,7 @@ class OlapTableSink : public DataSink { int64_t _db_id = -1; int64_t _table_id = -1; int _num_repicas = -1; - bool _need_gen_rollup = true; + bool _need_gen_rollup = false; std::string _db_name; std::string _table_name; int _tuple_desc_id = -1; diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index dee1889c3d54e2..7229b8129c718f 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -470,12 +470,13 @@ private void onFinished(OlapTable tbl) { // in catalog. MaterializedIndex shadowIdx = partition.getIndex(shadowIdxId); Preconditions.checkNotNull(shadowIdx, shadowIdxId); - MaterializedIndex droppedIdx = partition.deleteRollupIndex(originIdxId); - - // delete origin replicas - for (Tablet originTablet : droppedIdx.getTablets()) { - Catalog.getCurrentInvertedIndex().deleteTablet(originTablet.getId()); + MaterializedIndex droppedIdx = null; + if (originIdxId == partition.getBaseIndex().getId()) { + droppedIdx = partition.getBaseIndex(); + } else { + droppedIdx = partition.deleteRollupIndex(originIdxId); } + Preconditions.checkNotNull(droppedIdx, originIdxId + " vs. " + shadowIdxId); // set replica state for (Tablet tablet : shadowIdx.getTablets()) { @@ -485,6 +486,11 @@ private void onFinished(OlapTable tbl) { } partition.visualiseShadowIndex(shadowIdxId, originIdxId == partition.getBaseIndex().getId()); + + // delete origin replicas + for (Tablet originTablet : droppedIdx.getTablets()) { + Catalog.getCurrentInvertedIndex().deleteTablet(originTablet.getId()); + } } } diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index accf4b540e1dac..9ae4dbaf0bb646 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -17,8 +17,6 @@ package org.apache.doris.catalog; -import org.apache.doris.alter.AlterJob.JobState; -import org.apache.doris.alter.RollupJob; import org.apache.doris.analysis.AddPartitionClause; import org.apache.doris.analysis.AddRollupClause; import org.apache.doris.analysis.AlterClause; @@ -600,11 +598,7 @@ public void setColocateGroup(String colocateGroup) { // it is used for stream load // the caller should get db lock when call this method public boolean shouldLoadToNewRollup() { - RollupJob rollupJob = (RollupJob) Catalog.getInstance().getRollupHandler().getAlterJob(id); - if (rollupJob != null && rollupJob.getState() == JobState.FINISHING) { - return false; - } - return true; + return false; } public TTableDescriptor toThrift() { From 657492b31bc97d0f8844371b6ea567f82b71264d Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 3 Sep 2019 21:43:50 +0800 Subject: [PATCH 69/79] fix show job proc error --- .../org/apache/doris/alter/AlterHandler.java | 4 ++ .../apache/doris/common/proc/JobsProcDir.java | 58 +++++-------------- .../java/org/apache/doris/load/ExportMgr.java | 2 +- .../main/java/org/apache/doris/load/Load.java | 4 +- 4 files changed, 23 insertions(+), 45 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 23baa70a7f8a2d..93978fa6c21c95 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -160,6 +160,10 @@ public int getAlterJobNum(JobState state, long dbId) { return jobNum; } + public Long getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState state, long dbId) { + return alterJobsV2.values().stream().filter(e -> e.getJobState() == state && e.getDbId() == dbId).count(); + } + @Deprecated public Map unprotectedGetAlterJobs() { return this.alterJobs; diff --git a/fe/src/main/java/org/apache/doris/common/proc/JobsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/JobsProcDir.java index 17fd645c8d092e..437ba82dcbd915 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/JobsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/JobsProcDir.java @@ -94,22 +94,22 @@ public ProcResult fetchResult() throws AnalysisException { // load Load load = Catalog.getInstance().getLoadInstance(); LoadManager loadManager = Catalog.getCurrentCatalog().getLoadManager(); - Integer pendingNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.PENDING, dbId) + Long pendingNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.PENDING, dbId) + loadManager.getLoadJobNum(org.apache.doris.load.loadv2.JobState.PENDING, dbId); - Integer runningNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.ETL, dbId) + Long runningNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.ETL, dbId) + load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.LOADING, dbId) + loadManager.getLoadJobNum(org.apache.doris.load.loadv2.JobState.LOADING, dbId); - Integer finishedNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.QUORUM_FINISHED, dbId) + Long finishedNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.QUORUM_FINISHED, dbId) + load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.FINISHED, dbId) + loadManager.getLoadJobNum(org.apache.doris.load.loadv2.JobState.FINISHED, dbId); - Integer cancelledNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.CANCELLED, dbId) + Long cancelledNum = load.getLoadJobNum(org.apache.doris.load.LoadJob.JobState.CANCELLED, dbId) + loadManager.getLoadJobNum(org.apache.doris.load.loadv2.JobState.CANCELLED, dbId); - Integer totalNum = pendingNum + runningNum + finishedNum + cancelledNum; + Long totalNum = pendingNum + runningNum + finishedNum + cancelledNum; result.addRow(Lists.newArrayList(LOAD, pendingNum.toString(), runningNum.toString(), finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); // delete - pendingNum = 0; + pendingNum = 0L; runningNum = load.getDeleteJobNumByState(dbId, org.apache.doris.load.LoadJob.JobState.LOADING); finishedNum = load.getDeleteJobNumByState(dbId, org.apache.doris.load.LoadJob.JobState.FINISHED); cancelledNum = load.getDeleteJobNumByState(dbId, org.apache.doris.load.LoadJob.JobState.CANCELLED); @@ -119,52 +119,26 @@ public ProcResult fetchResult() throws AnalysisException { // rollup RollupHandler rollupHandler = Catalog.getInstance().getRollupHandler(); - pendingNum = rollupHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.PENDING, dbId); - runningNum = rollupHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.RUNNING, dbId) - + rollupHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.FINISHING, dbId); - finishedNum = rollupHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.FINISHED, dbId); - cancelledNum = rollupHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.CANCELLED, dbId); + pendingNum = rollupHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.PENDING, dbId); + runningNum = rollupHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.WAITING_TXN, dbId) + + rollupHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.RUNNING, dbId); + finishedNum = rollupHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.FINISHED, dbId); + cancelledNum = rollupHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.CANCELLED, dbId); totalNum = pendingNum + runningNum + finishedNum + cancelledNum; result.addRow(Lists.newArrayList(ROLLUP, pendingNum.toString(), runningNum.toString(), finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); // schema change SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler(); - pendingNum = schemaChangeHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.PENDING, dbId); - runningNum = schemaChangeHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.RUNNING, dbId) - + schemaChangeHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.FINISHING, dbId); - finishedNum = schemaChangeHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.FINISHED, dbId); - cancelledNum = schemaChangeHandler.getAlterJobNum(org.apache.doris.alter.AlterJob.JobState.CANCELLED, dbId); + pendingNum = schemaChangeHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.PENDING, dbId); + runningNum = schemaChangeHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.WAITING_TXN, dbId) + + schemaChangeHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.RUNNING, dbId); + finishedNum = schemaChangeHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.FINISHED, dbId); + cancelledNum = schemaChangeHandler.getAlterJobV2Num(org.apache.doris.alter.AlterJobV2.JobState.CANCELLED, dbId); totalNum = pendingNum + runningNum + finishedNum + cancelledNum; result.addRow(Lists.newArrayList(SCHEMA_CHANGE, pendingNum.toString(), runningNum.toString(), finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); - /* - // backup - BackupHandler backupHandler = Catalog.getInstance().getBackupHandler(); - pendingNum = backupHandler.getBackupJobNum(BackupJobState.PENDING, dbId); - runningNum = backupHandler.getBackupJobNum(BackupJobState.SNAPSHOT, dbId) - + backupHandler.getBackupJobNum(BackupJobState.UPLOAD, dbId) - + backupHandler.getBackupJobNum(BackupJobState.UPLOADING, dbId) - + backupHandler.getBackupJobNum(BackupJobState.FINISHING, dbId); - finishedNum = backupHandler.getBackupJobNum(BackupJobState.FINISHED, dbId); - cancelledNum = backupHandler.getBackupJobNum(BackupJobState.CANCELLED, dbId); - totalNum = pendingNum + runningNum + finishedNum + cancelledNum; - result.addRow(Lists.newArrayList(BACKUP, pendingNum.toString(), runningNum.toString(), - finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); - - // restore - pendingNum = backupHandler.getRestoreJobNum(RestoreJobState.PENDING, dbId); - runningNum = backupHandler.getRestoreJobNum(RestoreJobState.RESTORE_META, dbId) - + backupHandler.getRestoreJobNum(RestoreJobState.DOWNLOAD, dbId) - + backupHandler.getRestoreJobNum(RestoreJobState.DOWNLOADING, dbId); - finishedNum = backupHandler.getRestoreJobNum(RestoreJobState.FINISHED, dbId); - cancelledNum = backupHandler.getRestoreJobNum(RestoreJobState.CANCELLED, dbId); - totalNum = pendingNum + runningNum + finishedNum + cancelledNum; - result.addRow(Lists.newArrayList(RESTORE, pendingNum.toString(), runningNum.toString(), - finishedNum.toString(), cancelledNum.toString(), totalNum.toString())); - */ - // export ExportMgr exportMgr = Catalog.getInstance().getExportMgr(); pendingNum = exportMgr.getJobNum(ExportJob.JobState.PENDING, dbId); diff --git a/fe/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/src/main/java/org/apache/doris/load/ExportMgr.java index 0de207c53369b0..08133298c8d047 100644 --- a/fe/src/main/java/org/apache/doris/load/ExportMgr.java +++ b/fe/src/main/java/org/apache/doris/load/ExportMgr.java @@ -277,7 +277,7 @@ public void replayUpdateJobState(long jobId, ExportJob.JobState newState) { } } - public Integer getJobNum(ExportJob.JobState state, long dbId) { + public long getJobNum(ExportJob.JobState state, long dbId) { int size = 0; readLock(); try { diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 5290b2d39c93a6..2b70a7339f43cf 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -1723,7 +1723,7 @@ public List getLoadJobs(JobState jobState) { return jobs; } - public int getLoadJobNum(JobState jobState, long dbId) { + public long getLoadJobNum(JobState jobState, long dbId) { readLock(); try { List loadJobs = this.dbToLoadJobs.get(dbId); @@ -3399,7 +3399,7 @@ public List> getAsyncDeleteJobInfo(long jobId) { return infos; } - public int getDeleteJobNumByState(long dbId, JobState state) { + public long getDeleteJobNumByState(long dbId, JobState state) { readLock(); try { List deleteJobs = dbToDeleteJobs.get(dbId); From f6daaa0be3828c2ae182aa0744c25baf097f443b Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 4 Sep 2019 17:34:08 +0800 Subject: [PATCH 70/79] add more detail log --- be/src/agent/task_worker_pool.cpp | 2 +- be/src/olap/data_dir.cpp | 4 ++-- be/src/olap/schema_change.cpp | 2 +- be/src/olap/task/engine_alter_tablet_task.cpp | 2 +- be/src/olap/txn_manager.cpp | 3 ++- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 793fe008525531..a4aaa9ddb78a94 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -554,7 +554,7 @@ void* TaskWorkerPool::_alter_tablet_worker_thread_callback(void* arg_this) { int64_t time_elapsed = time(nullptr) - agent_task_req.recv_time; if (time_elapsed > config::report_task_interval_seconds * 20) { LOG(INFO) << "task elapsed " << time_elapsed - << " since it is inserted to queue, it is timeout"; + << " seconds since it is inserted to queue, it is timeout"; is_task_timeout = true; } } diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index a900bbf75594f6..cf66348184120a 100755 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -1026,10 +1026,10 @@ void DataDir::perform_path_scan() { void DataDir::_process_garbage_path(const std::string& path) { if (check_dir_existed(path)) { - LOG(INFO) << "collect garbage dir path:" << path; + LOG(INFO) << "collect garbage dir path: " << path; OLAPStatus status = remove_all_dir(path); if (status != OLAP_SUCCESS) { - LOG(WARNING) << "remove garbage dir path:" << path << " failed"; + LOG(WARNING) << "remove garbage dir path: " << path << " failed"; } } } diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index c8d6f203b1a145..e42cf94a2e5861 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1181,7 +1181,7 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe request.new_tablet_id, request.new_schema_hash); if (new_tablet == nullptr) { LOG(WARNING) << "fail to find new tablet." - << ", new_tablet=" << request.new_tablet_id + << " new_tablet=" << request.new_tablet_id << ", new_schema_hash=" << request.new_schema_hash; return OLAP_ERR_TABLE_NOT_FOUND; } diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp b/be/src/olap/task/engine_alter_tablet_task.cpp index 5a127a14a4062d..057975c1d2e4a9 100644 --- a/be/src/olap/task/engine_alter_tablet_task.cpp +++ b/be/src/olap/task/engine_alter_tablet_task.cpp @@ -39,7 +39,7 @@ OLAPStatus EngineAlterTabletTask::execute() { OLAPStatus res = handler.process_alter_tablet_v2(_alter_tablet_req); if (res != OLAP_SUCCESS) { - LOG(WARNING) << "failed to do rollup. res=" << res + LOG(WARNING) << "failed to do alter task. res=" << res << " base_tablet_id=" << _alter_tablet_req.base_tablet_id << ", base_schema_hash=" << _alter_tablet_req.base_schema_hash << ", new_tablet_id=" << _alter_tablet_req.new_tablet_id diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index d7eb37adf46020..d949ac909e2dc5 100755 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -201,7 +201,8 @@ OLAPStatus TxnManager::commit_txn( << " partition_id: " << key.first << ", transaction_id: " << key.second << ", tablet: " << tablet_info.to_string() - << ", rowsetid: " << rowset_ptr->rowset_id(); + << ", rowsetid: " << rowset_ptr->rowset_id() + << ", version: " << rowset_ptr->version().first; } return OLAP_SUCCESS; } From cf1f6b44f8e81f5c19e7a2a4d4a5e3f99da97a0f Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 7 Sep 2019 17:29:35 +0800 Subject: [PATCH 71/79] fix meta version --- fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java | 2 +- fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index d769f8b4233392..04d023562d74a3 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -913,7 +913,7 @@ public void readFields(DataInput in) throws IOException { authorizationInfo.readFields(in); } } - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_61) { timezone = Text.readString(in); } } diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java index ea2c00e64f3149..a8dabe71fa804d 100644 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -84,7 +84,7 @@ public void setUp() throws InstantiationException, IllegalAccessException, Illeg masterCatalog = CatalogTestUtil.createTestCatalog(); slaveCatalog = CatalogTestUtil.createTestCatalog(); MetaContext metaContext = new MetaContext(); - metaContext.setMetaVersion(FeMetaVersion.VERSION_60); + metaContext.setMetaVersion(FeMetaVersion.VERSION_61); metaContext.setThreadLocalInfo(); // masterCatalog.setJournalVersion(FeMetaVersion.VERSION_40); // slaveCatalog.setJournalVersion(FeMetaVersion.VERSION_40); From cda990a892987469851836cf512ae737a0a75fd1 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 7 Sep 2019 20:19:36 +0800 Subject: [PATCH 72/79] fix empty rollup rowset bug --- be/src/olap/schema_change.cpp | 2 +- fe/src/main/java/org/apache/doris/alter/RollupHandler.java | 2 +- .../java/org/apache/doris/catalog/MaterializedIndex.java | 2 ++ fe/src/main/java/org/apache/doris/catalog/Table.java | 5 +++-- fe/src/main/java/org/apache/doris/clone/TabletChecker.java | 7 ++----- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index e42cf94a2e5861..9bf5bb01d9be81 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -981,6 +981,7 @@ bool SchemaChangeWithSorting::process( row_block_arr.push_back(new_row_block); } else { + LOG(INFO) << "new block num rows is: " << new_row_block->row_block_info().row_num; _row_block_allocator->release(new_row_block); new_row_block = nullptr; } @@ -1348,7 +1349,6 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe LOG(WARNING) << "failed to alter tablet. base_tablet=" << base_tablet->full_name() << ", drop new_tablet=" << new_tablet->full_name(); // do not drop the new tablet and its data. GC thread will - // StorageEngine::instance()->tablet_manager()->drop_tablet(new_tablet->tablet_id(), new_tablet->schema_hash()); } return res; diff --git a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java index 03f04d86d663f9..bd796ac874740d 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupHandler.java @@ -304,7 +304,7 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl /* * create all rollup indexes. and set state. - * After setting, Tables' state will be RO + * After setting, Tables' state will be ROLLUP */ for (Partition partition : olapTable.getPartitions()) { long partitionId = partition.getId(); diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 3324006a90b34b..ebcbe4a72392d1 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -37,7 +37,9 @@ public class MaterializedIndex extends MetaObject implements Writable { public enum IndexState { NORMAL, + @Deprecated ROLLUP, + @Deprecated SCHEMA_CHANGE, SHADOW; // index in SHADOW state is visible to load process, but invisible to query diff --git a/fe/src/main/java/org/apache/doris/catalog/Table.java b/fe/src/main/java/org/apache/doris/catalog/Table.java index 69e1ba2c176bc8..6992c3d64a1e92 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/src/main/java/org/apache/doris/catalog/Table.java @@ -273,9 +273,10 @@ public String toString() { /* * 1. Only schedule OLAP table. * 2. If table is colocate with other table, not schedule it. - * 3. if table's state is ROLLUP or SCHEMA_CHANGE, but alter job's state is FINISHING, we should also + * 3. (deprecated). if table's state is ROLLUP or SCHEMA_CHANGE, but alter job's state is FINISHING, we should also * schedule the tablet to repair it(only for VERSION_IMCOMPLETE case, this will be checked in * TabletScheduler). + * 4. Even if table's state is ROLLUP or SCHEMA_CHANGE, check it. Because we can repair the tablet of base index. */ public boolean needSchedule() { if (type != TableType.OLAP) { @@ -283,7 +284,7 @@ public boolean needSchedule() { } OlapTable olapTable = (OlapTable) this; - + if (Catalog.getCurrentColocateIndex().isColocateTable(olapTable.getId())) { LOG.debug("table {} is a colocate table, skip tablet checker.", name); return false; diff --git a/fe/src/main/java/org/apache/doris/clone/TabletChecker.java b/fe/src/main/java/org/apache/doris/clone/TabletChecker.java index 9dbedda6758b9f..003e16a638dca0 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletChecker.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletChecker.java @@ -209,12 +209,9 @@ private void checkTablets() { boolean isInPrios = isInPrios(dbId, table.getId(), partition.getId()); boolean prioPartIsHealthy = true; /* - * Here we get all ALL indexes, including SHADOW indexes. - * SHADOW index should be treated as a special NORMAL index. - * It can be repaired, but CAN NOT be balanced, added or removed. - * The above restrictions will be checked in tablet scheduler. + * Tablet in SHADOW index can not be repaired of balanced */ - for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.ALL)) { + for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Tablet tablet : idx.getTablets()) { totalTabletNum++; From 948683c443da59900e9b4c584f3b63ab4a94dc69 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 8 Sep 2019 21:44:12 +0800 Subject: [PATCH 73/79] cancel the alter job if table is dropped --- .../org/apache/doris/alter/RollupJobV2.java | 26 ++++++++++++++----- .../apache/doris/alter/SchemaChangeJobV2.java | 26 ++++++++++++++----- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index ce4899b8e05717..51694a81a73423 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -354,6 +354,26 @@ protected void runWaitingTxnJob() { @Override protected void runRunningJob() { Preconditions.checkState(jobState == JobState.RUNNING, jobState); + // must check if db or table still exist first. + // or if table is dropped, the tasks will never be finished, + // and the job will be in RUNNING state forever. + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancelImpl("Databasee " + dbId + " does not exist"); + return; + } + + db.readLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancelImpl("Table " + tableId + " does not exist"); + return; + } + } finally { + db.readUnlock(); + } + if (!rollupBatchTask.isFinished()) { LOG.info("rollup tasks not finished. job: {}", jobId); return; @@ -363,12 +383,6 @@ protected void runRunningJob() { * all tasks are finished. check the integrity. * we just check whether all rollup replicas are healthy. */ - Database db = Catalog.getCurrentCatalog().getDb(dbId); - if (db == null) { - cancelImpl("Databasee " + dbId + " does not exist"); - return; - } - db.writeLock(); try { OlapTable tbl = (OlapTable) db.getTable(tableId); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 7229b8129c718f..a7eb698ec42748 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -388,6 +388,26 @@ protected void runWaitingTxnJob() { @Override protected void runRunningJob() { Preconditions.checkState(jobState == JobState.RUNNING, jobState); + // must check if db or table still exist first. + // or if table is dropped, the tasks will never be finished, + // and the job will be in RUNNING state forever. + Database db = Catalog.getCurrentCatalog().getDb(dbId); + if (db == null) { + cancelImpl("Databasee " + dbId + " does not exist"); + return; + } + + db.readLock(); + try { + OlapTable tbl = (OlapTable) db.getTable(tableId); + if (tbl == null) { + cancelImpl("Table " + tableId + " does not exist"); + return; + } + } finally { + db.readUnlock(); + } + if (!schemaChangeBatchTask.isFinished()) { LOG.info("schema change tasks not finished. job: {}", jobId); return; @@ -397,12 +417,6 @@ protected void runRunningJob() { * all tasks are finished. check the integrity. * we just check whether all new replicas are healthy. */ - Database db = Catalog.getCurrentCatalog().getDb(dbId); - if (db == null) { - cancelImpl("Databasee " + dbId + " does not exist"); - return; - } - db.writeLock(); try { OlapTable tbl = (OlapTable) db.getTable(tableId); From 8f6374508ff05c6d9e4de528fa00d2e0edb03197 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 9 Sep 2019 11:24:36 +0800 Subject: [PATCH 74/79] fix meta version of partition to 61 --- be/src/olap/tablet.cpp | 6 ++++-- fe/src/main/java/org/apache/doris/catalog/Partition.java | 2 +- .../main/java/org/apache/doris/master/ReportHandler.java | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index f4fc08ddd58bf9..b3d6694598aa5d 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -282,7 +282,8 @@ OLAPStatus Tablet::modify_rowsets(const vector& to_add, const RowsetSharedPtr Tablet::get_rowset_by_version(const Version& version) const { auto iter = _rs_version_map.find(version); if (iter == _rs_version_map.end()) { - LOG(INFO) << "no rowset for version:" << version.first << "-" << version.second; + LOG(INFO) << "no rowset for version:" << version.first << "-" << version.second + << ", tablet: " << full_name(); return nullptr; } RowsetSharedPtr rowset = iter->second; @@ -294,7 +295,8 @@ size_t Tablet::get_rowset_size_by_version(const Version& version) { << "invalid version:" << version.first << "-" << version.second; auto iter = _rs_version_map.find(version); if (iter == _rs_version_map.end()) { - LOG(WARNING) << "no rowset for version:" << version.first << "-" << version.second; + LOG(WARNING) << "no rowset for version:" << version.first << "-" << version.second + << ", tablet: " << full_name(); return -1; } RowsetSharedPtr rowset = iter->second; diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index ff4647276b5363..286f7fdc28c77c 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -345,7 +345,7 @@ public void readFields(DataInput in) throws IOException { idToVisibleRollupIndex.put(rollupTable.getId(), rollupTable); } - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_61) { int shadowIndexCount = in.readInt(); for (int i = 0; i < shadowIndexCount; i++) { MaterializedIndex shadowIndex = MaterializedIndex.read(in); diff --git a/fe/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/src/main/java/org/apache/doris/master/ReportHandler.java index 5225f54df5f749..83123823097e2c 100644 --- a/fe/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/src/main/java/org/apache/doris/master/ReportHandler.java @@ -135,23 +135,23 @@ public TMasterResult handleReport(TReportRequest request) throws TException { String reportType = ""; if (request.isSetTasks()) { tasks = request.getTasks(); - reportType += " task"; + reportType += "task"; } if (request.isSetDisks()) { disks = request.getDisks(); - reportType += " disk"; + reportType += "disk"; } if (request.isSetTablets()) { tablets = request.getTablets(); reportVersion = request.getReport_version(); - reportType += " tablet"; + reportType += "tablet"; } else if (request.isSetTablet_list()) { // the 'tablets' member will be deprecated in future. tablets = buildTabletMap(request.getTablet_list()); reportVersion = request.getReport_version(); - reportType += " tablet"; + reportType += "tablet"; } if (request.isSetForce_recovery()) { From 38c138e740178e5bd621132645d521540a8f4000 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 10 Sep 2019 10:45:10 +0800 Subject: [PATCH 75/79] fix meta version to 61 --- fe/src/main/java/org/apache/doris/catalog/Catalog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 074a2273091c5b..49ea7535907405 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1616,7 +1616,7 @@ public long loadAlterJob(DataInputStream dis, long checksum, JobType type) throw } // alter job v2 - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_61) { size = dis.readInt(); newChecksum ^= size; for (int i = 0; i < size; i++) { From 1047adfab660b344438ebf6e28bbe38c7bd7fa88 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 11 Sep 2019 09:58:27 +0800 Subject: [PATCH 76/79] fix replay waiting txn bug --- fe/src/main/java/org/apache/doris/alter/AlterHandler.java | 6 +++--- fe/src/main/java/org/apache/doris/alter/RollupJobV2.java | 3 ++- .../main/java/org/apache/doris/alter/SchemaChangeJobV2.java | 3 ++- fe/src/main/java/org/apache/doris/master/MasterImpl.java | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 93978fa6c21c95..f1fff2d57bf1b8 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -394,8 +394,8 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce throw new MetaNotFoundException("replica " + task.getNewReplicaId() + " does not exist"); } - LOG.info("before handle alter task replica: {}, task version: {}-{}", - replica, task.getVersion(), task.getVersionHash()); + LOG.info("before handle alter task tablet {}, replica: {}, task version: {}-{}", + task.getSignature(), replica, task.getVersion(), task.getVersionHash()); boolean versionChanged = false; if (replica.getVersion() > task.getVersion()) { // Case 2.2, do nothing @@ -422,7 +422,7 @@ public void handleFinishAlterTask(AlterReplicaTask task) throws MetaNotFoundExce Catalog.getInstance().getEditLog().logUpdateReplica(info); } - LOG.info("after handle alter task replica: {}", replica); + LOG.info("after handle alter task tablet: {}, replica: {}", task.getSignature(), replica); } finally { db.writeUnlock(); } diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 51694a81a73423..838cc4845fa220 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -654,7 +654,8 @@ private void replayWaitingTxn(RollupJobV2 replayedJob) { db.writeUnlock(); } - this.jobState = JobState.RUNNING; + // should still be in WAITING_TXN state, so that the alter tasks will be resend again + this.jobState = JobState.WAITING_TXN; this.watershedTxnId = replayedJob.watershedTxnId; LOG.info("replay waiting txn rollup job: {}", jobId); diff --git a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index a7eb698ec42748..afabb99fe04e19 100644 --- a/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -675,7 +675,8 @@ private void replayWaitingTxn(SchemaChangeJobV2 replayedJob) { db.writeUnlock(); } - jobState = JobState.RUNNING; + // should still be in WAITING_TXN state, so that the alter tasks will be resend again + this.jobState = JobState.WAITING_TXN; this.watershedTxnId = replayedJob.watershedTxnId; LOG.info("replay waiting txn schema change job: {}", jobId); } diff --git a/fe/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/src/main/java/org/apache/doris/master/MasterImpl.java index 7943f99f68d8fd..1393fd6300cb98 100644 --- a/fe/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/src/main/java/org/apache/doris/master/MasterImpl.java @@ -787,7 +787,7 @@ private void finishAlterTask(AgentTask task) { } alterTask.setFinished(true); } catch (MetaNotFoundException e) { - LOG.warn("failed to handle finish alter task: {}", e.getMessage()); + LOG.warn("failed to handle finish alter task: {}, {}", task.getSignature(), e.getMessage()); } AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); } From 644d159939e6db4961991475aee360b3d2c1dba6 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 11 Sep 2019 10:31:45 +0800 Subject: [PATCH 77/79] allow drop rollup if table is unstable --- .../java/org/apache/doris/alter/Alter.java | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/Alter.java b/fe/src/main/java/org/apache/doris/alter/Alter.java index 136afc199da9d3..9c2ad2e0e7d97c 100644 --- a/fe/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/src/main/java/org/apache/doris/alter/Alter.java @@ -89,8 +89,9 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { // schema change ops can appear several in one alter stmt without other alter ops entry boolean hasSchemaChange = false; - // rollup ops, if has, should appear one and only one entry - boolean hasRollup = false; + // rollup ops, if has, should appear one and only one add or drop rollup entry + boolean hasAddRollup = false; + boolean hasDropRollup = false; // partition ops, if has, should appear one and only one entry boolean hasPartition = false; // rename ops, if has, should appear one and only one entry @@ -100,18 +101,19 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { // check conflict alter ops first List alterClauses = stmt.getOps(); - // check conflict alter ops first - // if all alter clauses are DropPartitionClause, no need to call checkQuota. - boolean allDropPartitionClause = true; - + // check conflict alter ops first + + // if all alter clauses are DropPartitionClause or DropRollupClause, no need to check quota. + boolean allIsDropOps = true; for (AlterClause alterClause : alterClauses) { - if (!(alterClause instanceof DropPartitionClause)) { - allDropPartitionClause = false; + if (!(alterClause instanceof DropPartitionClause) + && !(alterClause instanceof DropRollupClause)) { + allIsDropOps = false; break; } } - if (!allDropPartitionClause) { + if (!allIsDropOps) { // check db quota db.checkQuota(); } @@ -123,30 +125,30 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { || alterClause instanceof ModifyColumnClause || alterClause instanceof ReorderColumnsClause || alterClause instanceof ModifyTablePropertiesClause) - && !hasRollup && !hasPartition && !hasRename) { + && !hasAddRollup && !hasDropRollup && !hasPartition && !hasRename) { hasSchemaChange = true; - } else if (alterClause instanceof AddRollupClause && !hasSchemaChange && !hasRollup && !hasPartition - && !hasRename && !hasModifyProp) { - hasRollup = true; - } else if (alterClause instanceof DropRollupClause && !hasSchemaChange && !hasRollup && !hasPartition - && !hasRename && !hasModifyProp) { - hasRollup = true; - } else if (alterClause instanceof AddPartitionClause && !hasSchemaChange && !hasRollup && !hasPartition - && !hasRename && !hasModifyProp) { - hasPartition = true; - } else if (alterClause instanceof DropPartitionClause && !hasSchemaChange && !hasRollup && !hasPartition - && !hasRename && !hasModifyProp) { + } else if (alterClause instanceof AddRollupClause && !hasSchemaChange && !hasAddRollup && !hasDropRollup + && !hasPartition && !hasRename && !hasModifyProp) { + hasAddRollup = true; + } else if (alterClause instanceof DropRollupClause && !hasSchemaChange && !hasAddRollup && !hasDropRollup + && !hasPartition && !hasRename && !hasModifyProp) { + hasDropRollup = true; + } else if (alterClause instanceof AddPartitionClause && !hasSchemaChange && !hasAddRollup && !hasDropRollup + && !hasPartition && !hasRename && !hasModifyProp) { hasPartition = true; - } else if (alterClause instanceof ModifyPartitionClause && !hasSchemaChange && !hasRollup + } else if (alterClause instanceof DropPartitionClause && !hasSchemaChange && !hasAddRollup && !hasDropRollup && !hasPartition && !hasRename && !hasModifyProp) { hasPartition = true; + } else if (alterClause instanceof ModifyPartitionClause && !hasSchemaChange && !hasAddRollup + && !hasDropRollup && !hasPartition && !hasRename && !hasModifyProp) { + hasPartition = true; } else if ((alterClause instanceof TableRenameClause || alterClause instanceof RollupRenameClause || alterClause instanceof PartitionRenameClause || alterClause instanceof ColumnRenameClause) - && !hasSchemaChange && !hasRollup && !hasPartition && !hasRename && !hasModifyProp) { + && !hasSchemaChange && !hasAddRollup && !hasDropRollup && !hasPartition && !hasRename + && !hasModifyProp) { hasRename = true; - } else if (alterClause instanceof ModifyTablePropertiesClause && !hasSchemaChange && !hasRollup - && !hasPartition - && !hasRename && !hasModifyProp) { + } else if (alterClause instanceof ModifyTablePropertiesClause && !hasSchemaChange && !hasAddRollup + && !hasDropRollup && !hasPartition && !hasRename && !hasModifyProp) { hasModifyProp = true; } else { throw new DdlException("Conflicting alter clauses. see help for more information"); @@ -176,7 +178,7 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { throw new DdlException("Table[" + table.getName() + "]'s state is not NORMAL. Do not allow doing ALTER ops"); } - if (hasSchemaChange || hasModifyProp || hasRollup) { + if (hasSchemaChange || hasModifyProp || hasAddRollup) { // check if all tablets are healthy, and no tablet is in tablet scheduler boolean isStable = olapTable.isStable(Catalog.getCurrentSystemInfo(), Catalog.getCurrentCatalog().getTabletScheduler(), @@ -191,7 +193,7 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { if (hasSchemaChange || hasModifyProp) { schemaChangeHandler.process(alterClauses, clusterName, db, olapTable); - } else if (hasRollup) { + } else if (hasAddRollup || hasDropRollup) { rollupHandler.process(alterClauses, clusterName, db, olapTable); } else if (hasPartition) { Preconditions.checkState(alterClauses.size() == 1); From 3497b1cc9cbbc8ce8911bba8309946b598aec1ef Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 12 Sep 2019 10:38:25 +0800 Subject: [PATCH 78/79] fix review by zhaochun --- .../java/org/apache/doris/analysis/DataDescription.java | 2 +- fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java | 3 +++ fe/src/main/java/org/apache/doris/task/StreamLoadTask.java | 7 ------- .../org/apache/doris/transaction/TransactionState.java | 5 +++++ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index 1968ee5fc0dc39..ff8617706017c8 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -104,7 +104,7 @@ public class DataDescription { * For hadoop load, this param is also used to persistence. * The function in this param is copied from 'parsedColumnExprList' */ - private Map>> columnToHadoopFunction = Maps.newHashMap(); + private Map>> columnToHadoopFunction = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); private boolean isHadoopLoad = false; diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index 04d023562d74a3..42191067ca56a8 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -305,6 +305,9 @@ protected void setJobProperties(Map properties) throws DdlExcept if (properties.containsKey(LoadStmt.TIMEZONE)) { timezone = properties.get(LoadStmt.TIMEZONE); + } else if (ConnectContext.get() != null) { + // get timezone for session variable + timezone = ConnectContext.get().getSessionVariable().getTimeZone(); } } } diff --git a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java index c30af73edb2178..3ac9722c444ccc 100644 --- a/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -88,13 +88,6 @@ public List getColumnExprDescs() { return columnExprDescs; } - public void addColumnExprDesc(ImportColumnDesc columnExprDesc) { - if (columnExprDescs == null) { - columnExprDescs = Lists.newArrayList(); - } - this.columnExprDescs.add(columnExprDesc); - } - public Expr getWhereExpr() { return whereExpr; } diff --git a/fe/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/src/main/java/org/apache/doris/transaction/TransactionState.java index c1f2f1d768f934..82ae27574a3072 100644 --- a/fe/src/main/java/org/apache/doris/transaction/TransactionState.java +++ b/fe/src/main/java/org/apache/doris/transaction/TransactionState.java @@ -414,6 +414,11 @@ public void setTxnCommitAttachment(TxnCommitAttachment txnCommitAttachment) { this.txnCommitAttachment = txnCommitAttachment; } + /* + * Add related table indexes to the transaction. + * If function should always be called before adding this transaction state to transaction manager, + * No other thread will access this state. So no need to lock + */ public void addTableIndexes(OlapTable table) { Set indexIds = loadedTblIndexes.get(table.getId()); if (indexIds == null) { From 40d78c0cc8eda7329978aeab54488ce1e639cf07 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 12 Sep 2019 14:36:10 +0800 Subject: [PATCH 79/79] fix review by zc2 --- .../org/apache/doris/alter/AlterJobV2.java | 5 -- .../main/java/org/apache/doris/load/Load.java | 63 ++++++++++--------- .../apache/doris/planner/BrokerScanNode.java | 7 +-- .../doris/planner/StreamLoadScanNode.java | 5 +- 4 files changed, 38 insertions(+), 42 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java index 155a6d670d53bb..8233f38c0540c5 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -30,11 +30,6 @@ import java.io.IOException; import java.util.List; -/* - * Author: Chenmingyu - * Date: Jul 8, 2019 - */ - /* * Version 2 of AlterJob, for replacing the old version of AlterJob. * This base class of RollupJob and SchemaChangeJob diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 2b70a7339f43cf..f7f238920ba585 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -871,12 +871,13 @@ public static void checkAndCreateSource(Database db, DataDescription dataDescrip * This function should be used for broker load v2 and stream load. * And it must be called in same db lock when planing. */ - public static void initColumns(Table tbl, boolean specifyFileFieldNames, List columnExprs, + public static void initColumns(Table tbl, List columnExprs, Map>> columnToHadoopFunction, Map exprsByName, Analyzer analyzer, TupleDescriptor srcTupleDesc, Map slotDescByName, TBrokerScanRangeParams params) throws UserException { // If user does not specify the file field names, generate it by using base schema of table. // So that the following process can be unified + boolean specifyFileFieldNames = columnExprs.stream().anyMatch(p -> p.isColumn()); if (!specifyFileFieldNames) { List columns = tbl.getBaseSchema(); for (Column column : columns) { @@ -910,39 +911,41 @@ public static void initColumns(Table tbl, boolean specifyFileFieldNames, List - * (A, C) SET (B = func(xx), __doris_shadow_B = func(xxx)) - */ - ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), mappingExpr); - columnExprs.add(importColumnDesc); - } else { - /* - * eg: - * (A, B, C) - * -> - * (A, B, C) SET (__doris_shadow_B = B) - */ - ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), - new SlotRef(null, originCol)); - columnExprs.add(importColumnDesc); - } + if (!column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) { + continue; + } + + String originCol = column.getNameWithoutPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX); + if (columnExprMap.containsKey(originCol)) { + Expr mappingExpr = columnExprMap.get(originCol); + if (mappingExpr != null) { + /* + * eg: + * (A, C) SET (B = func(xx)) + * -> + * (A, C) SET (B = func(xx), __doris_shadow_B = func(xxx)) + */ + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), mappingExpr); + columnExprs.add(importColumnDesc); } else { /* - * There is a case that if user does not specify the related origin column, eg: - * COLUMNS (A, C), and B is not specified, but B is being modified so there is a shadow column '__doris_shadow_B'. - * We can not just add a mapping function "__doris_shadow_B = substitute(B)", because Doris can not find column B. - * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping + * eg: + * (A, B, C) + * -> + * (A, B, C) SET (__doris_shadow_B = B) */ - // do nothing + ImportColumnDesc importColumnDesc = new ImportColumnDesc(column.getName(), + new SlotRef(null, originCol)); + columnExprs.add(importColumnDesc); } + } else { + /* + * There is a case that if user does not specify the related origin column, eg: + * COLUMNS (A, C), and B is not specified, but B is being modified so there is a shadow column '__doris_shadow_B'. + * We can not just add a mapping function "__doris_shadow_B = substitute(B)", because Doris can not find column B. + * In this case, __doris_shadow_B can use its default value, so no need to add it to column mapping + */ + // do nothing } } diff --git a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java index 9679b44a8f4422..e5817d8dadc96a 100644 --- a/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -228,10 +228,9 @@ private void initColumns(ParamCreateContext context) throws UserException { context.slotDescByName = Maps.newHashMap(); context.exprMap = Maps.newHashMap(); - boolean specifyFileFieldNames = context.fileGroup.getColumnExprList().stream().anyMatch(p -> p.isColumn()); - Load.initColumns(targetTable, specifyFileFieldNames, context.fileGroup.getColumnExprList(), - context.fileGroup.getColumnToHadoopFunction(), context.exprMap, analyzer, context.tupleDescriptor, - context.slotDescByName, context.params); + Load.initColumns(targetTable, context.fileGroup.getColumnExprList(), + context.fileGroup.getColumnToHadoopFunction(), context.exprMap, analyzer, + context.tupleDescriptor, context.slotDescByName, context.params); } private void finalizeParams(ParamCreateContext context) throws UserException, AnalysisException { diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 01e47f021ca8fd..3a4a6a54128440 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -119,9 +119,8 @@ public void init(Analyzer analyzer) throws UserException { TBrokerScanRangeParams params = new TBrokerScanRangeParams(); params.setStrict_mode(streamLoadTask.isStrictMode()); - boolean specifyFileFieldNames = streamLoadTask.getColumnExprDescs().stream().anyMatch(p -> p.isColumn()); - Load.initColumns(dstTable, specifyFileFieldNames, streamLoadTask.getColumnExprDescs(), - null /* no hadoop function */, exprsByName, analyzer, srcTupleDesc, slotDescByName, params); + Load.initColumns(dstTable, streamLoadTask.getColumnExprDescs(), null /* no hadoop function */, + exprsByName, analyzer, srcTupleDesc, slotDescByName, params); // analyze where statement if (streamLoadTask.getWhereExpr() != null) {