From 5232660b209c5b5d1107d22eebfacedfde4bcd67 Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Mon, 4 Sep 2017 19:09:42 +0800 Subject: [PATCH 1/3] remove distinct check with one param in second aggregate pharse --- fe/src/com/baidu/palo/analysis/AggregateInfo.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/src/com/baidu/palo/analysis/AggregateInfo.java b/fe/src/com/baidu/palo/analysis/AggregateInfo.java index e04b24ccdae8f2..c0db99e9241210 100644 --- a/fe/src/com/baidu/palo/analysis/AggregateInfo.java +++ b/fe/src/com/baidu/palo/analysis/AggregateInfo.java @@ -528,8 +528,6 @@ private void createSecondPhaseAggInfo( // SUM(DISTINCT ) -> SUM(); // (MIN(DISTINCT ...) and MAX(DISTINCT ...) have their DISTINCT turned // off during analysis, and AVG() is changed to SUM()/COUNT()) - Preconditions.checkState( - inputExpr.getFnName().getFunction().equalsIgnoreCase("SUM")); Expr aggExprParam = new SlotRef(inputDesc.getSlots().get(origGroupingExprs.size())); aggExpr = new FunctionCallExpr(inputExpr.getFnName(), Lists.newArrayList(aggExprParam)); } From c477e5b4dcc0d049932c61fa86a7808189fc39e8 Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Tue, 5 Sep 2017 16:04:49 +0800 Subject: [PATCH 2/3] select wrong rollup --- fe/src/com/baidu/palo/alter/RollupJob.java | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/fe/src/com/baidu/palo/alter/RollupJob.java b/fe/src/com/baidu/palo/alter/RollupJob.java index 6db03cfacd1683..467f42766bc6c4 100644 --- a/fe/src/com/baidu/palo/alter/RollupJob.java +++ b/fe/src/com/baidu/palo/alter/RollupJob.java @@ -624,6 +624,7 @@ public int tryFinishJob() { MaterializedIndex rollupIndex = this.partitionIdToRollupIndex.get(partitionId); Preconditions.checkNotNull(rollupIndex); + long rollupRowCount = 0L; // 1. record replica info for (Tablet tablet : rollupIndex.getTablets()) { long tabletId = tablet.getId(); @@ -634,8 +635,21 @@ public int tryFinishJob() { replica.getDataSize(), replica.getRowCount()); this.partitionIdToReplicaInfos.put(partitionId, replicaInfo); } + + // calculate rollup index row count + long tabletRowCount = 0L; + for (Replica replica : tablet.getReplicas()) { + long replicaRowCount = replica.getRowCount(); + if (replicaRowCount > tabletRowCount) { + tabletRowCount = replicaRowCount; + } + } + rollupRowCount += tabletRowCount; + } // end for tablets + rollupIndex.setRowCount(rollupRowCount); + // 2. add to partition partition.createRollupIndex(rollupIndex); @@ -717,11 +731,24 @@ public void unprotectedReplayFinish(Database db) { Partition partition = olapTable.getPartition(partitionId); MaterializedIndex rollupIndex = entry.getValue(); + long rollupRowCount = 0L; for (Tablet tablet : rollupIndex.getTablets()) { for (Replica replica : tablet.getReplicas()) { replica.setState(ReplicaState.NORMAL); } + + // calculate rollup index row count + long tabletRowCount = 0L; + for (Replica replica : tablet.getReplicas()) { + long replicaRowCount = replica.getRowCount(); + if (replicaRowCount > tabletRowCount) { + tabletRowCount = replicaRowCount; + } + } + rollupRowCount += tabletRowCount; } + + rollupIndex.setRowCount(rollupRowCount); rollupIndex.setState(IndexState.NORMAL); MaterializedIndex baseIndex = partition.getIndex(baseIndexId); From 1f26563f4718be77cd6f080bec5ba8229033809e Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Tue, 5 Sep 2017 21:26:49 +0800 Subject: [PATCH 3/3] update backend in cluster because of forgeting to remove backend' id when drop backend or decommission in latest version --- fe/src/com/baidu/palo/catalog/Catalog.java | 9 ++++++--- fe/src/com/baidu/palo/common/FeConstants.java | 2 +- fe/src/com/baidu/palo/common/FeMetaVersion.java | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/fe/src/com/baidu/palo/catalog/Catalog.java b/fe/src/com/baidu/palo/catalog/Catalog.java index 2e21e247f8546d..e2cd3acd103f42 100644 --- a/fe/src/com/baidu/palo/catalog/Catalog.java +++ b/fe/src/com/baidu/palo/catalog/Catalog.java @@ -4946,10 +4946,13 @@ public long loadCluster(DataInputStream dis, long checksum) throws IOException, checksum ^= cluster.getId(); // BE is in default_cluster when added , therefore it is possible that the BE - // in default_clsuter are not the latest because cluster cant't be updated when - // loadCluster is after loadBackend. + // in default_cluster are not the latest because cluster cant't be updated when + // loadCluster is after loadBackend. Because of forgeting to remove BE's id in + // cluster when drop BE or decommission in latest versions, need to update cluster's + // BE. List latestBackendIds = systemInfo.getClusterBackendIds(cluster.getName()); - if (cluster.getName().equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + if (cluster.getName().equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER) + || Catalog.getCurrentCatalogJournalVersion() <= FeMetaVersion.VERSION_34) { cluster.setBackendIdList(latestBackendIds); } else { // The cluster has the same number of be as systeminfo recorded diff --git a/fe/src/com/baidu/palo/common/FeConstants.java b/fe/src/com/baidu/palo/common/FeConstants.java index a1bc669776c7c4..1ed0fd520cf0ee 100644 --- a/fe/src/com/baidu/palo/common/FeConstants.java +++ b/fe/src/com/baidu/palo/common/FeConstants.java @@ -38,5 +38,5 @@ public class FeConstants { // general model // Current meta data version. Use this version to write journals and image - public static int meta_version = FeMetaVersion.VERSION_34; + public static int meta_version = FeMetaVersion.VERSION_35; } diff --git a/fe/src/com/baidu/palo/common/FeMetaVersion.java b/fe/src/com/baidu/palo/common/FeMetaVersion.java index 19379133b859a0..4a7a86357edc46 100644 --- a/fe/src/com/baidu/palo/common/FeMetaVersion.java +++ b/fe/src/com/baidu/palo/common/FeMetaVersion.java @@ -68,4 +68,9 @@ public final class FeMetaVersion { // persist LoadJob's execMemLimit public static final int VERSION_34 = 34; + + // update the BE in cluster, because of forgeting + // to remove backend in cluster when drop backend or + // decommission in latest versions. + public static final int VERSION_35= 35; }