Skip to content

Fix balance with diff storage medium #705

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chaoyli merged 10 commits into from
Mar 11, 2019
Merged

Fix balance with diff storage medium #705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
8fe1b8f
first
morningman Mar 6, 2019
8e47592
Optimize the balance policy for multi storage medium
morningman Mar 7, 2019
6213698
fix bug
morningman Mar 7, 2019
9c748cd
add mix load score
morningman Mar 7, 2019
dc4a967
fix bug
morningman Mar 7, 2019
410d37b
fix ut
morningman Mar 8, 2019
cf3c135
fix bug
morningman Mar 8, 2019
62bbdf3
fix bug
morningman Mar 8, 2019
a968515
add more debug log
morningman Mar 8, 2019
217d0f5
remove balance task if it failed to be scheduled many times
morningman Mar 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion fe/src/main/java/org/apache/doris/alter/RollupHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import org.apache.doris.task.DropReplicaTask;
import org.apache.doris.thrift.TKeysType;
import org.apache.doris.thrift.TResourceInfo;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;

import com.google.common.base.Preconditions;
Expand Down Expand Up @@ -327,13 +328,14 @@ private void processAddRollup(AddRollupClause alterClause, Database db, OlapTabl

for (Partition partition : olapTable.getPartitions()) {
long partitionId = partition.getId();
TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium();
MaterializedIndex rollupIndex = new MaterializedIndex(rollupIndexId, IndexState.ROLLUP);
if (isRestore) {
rollupIndex.setState(IndexState.NORMAL);
}
MaterializedIndex baseIndex = partition.getIndex(baseIndexId);
TabletMeta rollupTabletMeta = new TabletMeta(dbId, tableId, partitionId, rollupIndexId,
rollupSchemaHash);
rollupSchemaHash, medium);
short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId());
for (Tablet baseTablet : baseIndex.getTablets()) {
long baseTabletId = baseTablet.getId();
Expand Down
5 changes: 4 additions & 1 deletion fe/src/main/java/org/apache/doris/alter/RollupJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.apache.doris.task.CreateRollupTask;
import org.apache.doris.thrift.TKeysType;
import org.apache.doris.thrift.TResourceInfo;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TTabletInfo;
import org.apache.doris.thrift.TTaskType;
Expand Down Expand Up @@ -782,11 +783,13 @@ public void replayInitJob(Database db) {
for (Map.Entry<Long, MaterializedIndex> entry : this.partitionIdToRollupIndex.entrySet()) {
Partition partition = olapTable.getPartition(entry.getKey());
partition.setState(PartitionState.ROLLUP);
TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(
partition.getId()).getStorageMedium();

if (!Catalog.isCheckpointThread()) {
MaterializedIndex rollupIndex = entry.getValue();
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, entry.getKey(), rollupIndexId,
rollupSchemaHash);
rollupSchemaHash, medium);
for (Tablet tablet : rollupIndex.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down
14 changes: 6 additions & 8 deletions fe/src/main/java/org/apache/doris/backup/RestoreJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,7 @@ private void checkAndPrepareMeta() {
KeysType keysType = localTbl.getKeysType();
List<Column> columns = localTbl.getSchemaByIndexId(restoredIdx.getId());
TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(),
restoredIdx.getId(), schemaHash);
restoredIdx.getId(), schemaHash, TStorageMedium.HDD);
for (Tablet restoreTablet : restoredIdx.getTablets()) {
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
for (Replica restoreReplica : restoreTablet.getReplicas()) {
Expand All @@ -640,9 +640,7 @@ private void checkAndPrepareMeta() {

// generate create replica task for all restored tables
for (OlapTable restoreTbl : restoredTbls) {
PartitionInfo partInfo = restoreTbl.getPartitionInfo();
for (Partition restorePart : restoreTbl.getPartitions()) {
TStorageMedium storageMedium = partInfo.getDataProperty(restorePart.getId()).getStorageMedium();
Set<String> bfColumns = restoreTbl.getCopiedBfColumns();
double bfFpp = restoreTbl.getBfFpp();
for (MaterializedIndex index : restorePart.getMaterializedIndices()) {
Expand All @@ -651,15 +649,15 @@ private void checkAndPrepareMeta() {
KeysType keysType = restoreTbl.getKeysType();
List<Column> columns = restoreTbl.getSchemaByIndexId(index.getId());
TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), restorePart.getId(),
index.getId(), schemaHash);
index.getId(), schemaHash, TStorageMedium.HDD);
for (Tablet tablet : index.getTablets()) {
Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta);
for (Replica replica : tablet.getReplicas()) {
Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica);
CreateReplicaTask task = new CreateReplicaTask(replica.getBackendId(), dbId,
restoreTbl.getId(), restorePart.getId(), index.getId(), tablet.getId(),
shortKeyColumnCount, schemaHash, replica.getVersion(), replica.getVersionHash(),
keysType, TStorageType.COLUMN, storageMedium, columns,
keysType, TStorageType.COLUMN, TStorageMedium.HDD, columns,
bfColumns, bfFpp, null);
task.setInRestoreMode(true);
batchTask.addTask(task);
Expand Down Expand Up @@ -921,14 +919,14 @@ private void replayCheckAndPrepareMeta() {
Range<PartitionKey> remoteRange = remotePartitionInfo.getRange(remotePartId);
DataProperty remoteDataProperty = remotePartitionInfo.getDataProperty(remotePartId);
localPartitionInfo.addPartition(restorePart.getId(), remoteRange,
remoteDataProperty, (short) restoreReplicationNum);
remoteDataProperty, (short) restoreReplicationNum);
localTbl.addPartition(restorePart);

// modify tablet inverted index
for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices()) {
int schemaHash = localTbl.getSchemaHashByIndexId(restoreIdx.getId());
TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(),
restoreIdx.getId(), schemaHash);
restoreIdx.getId(), schemaHash, TStorageMedium.HDD);
for (Tablet restoreTablet : restoreIdx.getTablets()) {
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
for (Replica restoreReplica : restoreTablet.getReplicas()) {
Expand All @@ -946,7 +944,7 @@ private void replayCheckAndPrepareMeta() {
for (MaterializedIndex restoreIdx : restorePart.getMaterializedIndices()) {
int schemaHash = restoreTbl.getSchemaHashByIndexId(restoreIdx.getId());
TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), restorePart.getId(),
restoreIdx.getId(), schemaHash);
restoreIdx.getId(), schemaHash, TStorageMedium.HDD);
for (Tablet restoreTablet : restoreIdx.getTablets()) {
Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta);
for (Replica restoreReplica : restoreTablet.getReplicas()) {
Expand Down
5 changes: 3 additions & 2 deletions fe/src/main/java/org/apache/doris/backup/RestoreJob_D.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.apache.doris.task.AgentTask;
import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.thrift.TStorageMedium;

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
Expand Down Expand Up @@ -448,7 +449,7 @@ public void finishing(Catalog catalog, boolean isReplay) throws DdlException {
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId,
schemaHash);
schemaHash, TStorageMedium.HDD);
invertedIndex.addTablet(tabletId, tabletMeta);
for (Replica replica : tablet.getReplicas()) {
invertedIndex.addReplica(tabletId, replica);
Expand Down Expand Up @@ -499,7 +500,7 @@ public void finishing(Catalog catalog, boolean isReplay) throws DdlException {
invertedIndex.addReplica(tabletId, replica);
}
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId,
schemaHash);
schemaHash, TStorageMedium.HDD);
invertedIndex.addTablet(tabletId, tabletMeta);
}
}
Expand Down
17 changes: 11 additions & 6 deletions fe/src/main/java/org/apache/doris/catalog/Catalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1332,10 +1332,12 @@ private void recreateTabletInvertIndex() {
long tableId = olapTable.getId();
for (Partition partition : olapTable.getPartitions()) {
long partitionId = partition.getId();
TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(
partitionId).getStorageMedium();
for (MaterializedIndex index : partition.getMaterializedIndices()) {
long indexId = index.getId();
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium);
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down Expand Up @@ -1695,7 +1697,6 @@ public long loadTransactionState(DataInputStream dis, long checksum) throws IOEx
public long loadRecycleBin(DataInputStream dis, long checksum) throws IOException {
if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_10) {
Catalog.getCurrentRecycleBin().readFields(dis);

if (!isCheckpointThread()) {
// add tablet in Recycle bin to TabletInvertedIndex
Catalog.getCurrentRecycleBin().addTabletToInvertedIndex();
Expand Down Expand Up @@ -2999,7 +3000,7 @@ public void replayAddPartition(PartitionPersistInfo info) throws DdlException {
long indexId = index.getId();
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
TabletMeta tabletMeta = new TabletMeta(info.getDbId(), info.getTableId(), partition.getId(),
index.getId(), schemaHash);
index.getId(), schemaHash, info.getDataProperty().getStorageMedium());
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down Expand Up @@ -3220,7 +3221,7 @@ private Partition createPartitionWithIndices(String clusterName, long dbId, long

// create tablets
int schemaHash = indexIdToSchemaHash.get(indexId);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, storageMedium);
createTablets(clusterName, index, ReplicaState.NORMAL, distributionInfo, version, versionHash,
replicationNum, tabletMeta, tabletIdSet);

Expand Down Expand Up @@ -3990,10 +3991,12 @@ public void replayCreateTable(String dbName, Table table) {
long tableId = table.getId();
for (Partition partition : olapTable.getPartitions()) {
long partitionId = partition.getId();
TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(
partitionId).getStorageMedium();
for (MaterializedIndex mIndex : partition.getMaterializedIndices()) {
long indexId = mIndex.getId();
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium);
for (Tablet tablet : mIndex.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down Expand Up @@ -5982,11 +5985,13 @@ public void replayTruncateTable(TruncateTableInfo info) {
TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex();
for (Partition partition : info.getPartitions()) {
long partitionId = partition.getId();
TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(
partitionId).getStorageMedium();
for (MaterializedIndex mIndex : partition.getMaterializedIndices()) {
long indexId = mIndex.getId();
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
TabletMeta tabletMeta = new TabletMeta(db.getId(), olapTable.getId(),
partitionId, indexId, schemaHash);
partitionId, indexId, schemaHash, medium);
for (Tablet tablet : mIndex.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down
10 changes: 7 additions & 3 deletions fe/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.doris.task.AgentBatchTask;
import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.DropReplicaTask;
import org.apache.doris.thrift.TStorageMedium;

import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -573,10 +574,11 @@ public void addTabletToInvertedIndex() {
long tableId = olapTable.getId();
for (Partition partition : olapTable.getPartitions()) {
long partitionId = partition.getId();
TStorageMedium medium = olapTable.getPartitionInfo().getDataProperty(partitionId).getStorageMedium();
for (MaterializedIndex index : partition.getMaterializedIndices()) {
long indexId = index.getId();
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium);
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down Expand Up @@ -622,11 +624,13 @@ public void addTabletToInvertedIndex() {
olapTable = (OlapTable) tableInfo.getTable();
}
Preconditions.checkNotNull(olapTable);

// storage medium should be got from RecyclePartitionInfo, not from olap table. because olap table
// does not have this partition any more
TStorageMedium medium = partitionInfo.getDataProperty().getStorageMedium();
for (MaterializedIndex index : partition.getMaterializedIndices()) {
long indexId = index.getId();
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash);
TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, medium);
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
invertedIndex.addTablet(tabletId, tabletMeta);
Expand Down
2 changes: 2 additions & 0 deletions fe/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.apache.doris.common.util.Util;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TOlapTable;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
Expand Down Expand Up @@ -906,6 +907,7 @@ public OlapTable selectiveCopy(Collection<String> reservedPartNames, boolean res
copied.setState(OlapTableState.NORMAL);
for (Partition partition : copied.getPartitions()) {
partition.setState(PartitionState.NORMAL);
copied.getPartitionInfo().setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD));
for (MaterializedIndex idx : partition.getMaterializedIndices()) {
idx.setState(IndexState.NORMAL);
for (Tablet tablet : idx.getTablets()) {
Expand Down
Loading