From 9ce2c316015c781951ce46919be3917ad276e9e1 Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Fri, 27 Oct 2017 15:40:40 +0800 Subject: [PATCH 1/7] CreateTableStmt and AlterTableStmt cluster name is null Change-Id: Id5869961694fa11808db66ecf121d7cef0898f95 Signed-off-by: chenhao16 --- fe/src/com/baidu/palo/analysis/AlterTableStmt.java | 4 +++- fe/src/com/baidu/palo/analysis/CreateTableStmt.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/src/com/baidu/palo/analysis/AlterTableStmt.java b/fe/src/com/baidu/palo/analysis/AlterTableStmt.java index 89bc5c85789d34..35fb0dc5e9d211 100644 --- a/fe/src/com/baidu/palo/analysis/AlterTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterTableStmt.java @@ -24,6 +24,7 @@ import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.ErrorCode; import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; import com.baidu.palo.common.io.Writable; import com.google.common.collect.Lists; @@ -62,7 +63,8 @@ public void setTableName(String newTableName) { } @Override - public void analyze(Analyzer analyzer) throws AnalysisException { + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); if (tbl == null) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_TABLES_USED); } diff --git a/fe/src/com/baidu/palo/analysis/CreateTableStmt.java b/fe/src/com/baidu/palo/analysis/CreateTableStmt.java index 0694110947c387..d1dee35f925e21 100644 --- a/fe/src/com/baidu/palo/analysis/CreateTableStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateTableStmt.java @@ -187,6 +187,7 @@ public void setTableName(String newTableName) { @Override public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); tableName.analyze(analyzer); FeNameFormat.checkTableName(tableName.getTbl()); From 45bd122524df5f528ca053262ccda561eb5898cf Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Fri, 3 Nov 2017 17:35:06 +0800 Subject: [PATCH 2/7] add des cluster when perform 'allter system add backend' --- .../Contents/Administration/admin_stmt.md | 16 ++++---- .../com/baidu/palo/alter/SystemHandler.java | 10 ++++- .../baidu/palo/analysis/AddBackendClause.java | 37 +++++++++++++---- .../baidu/palo/analysis/AlterClusterStmt.java | 2 +- .../palo/analysis/CreateClusterStmt.java | 5 ++- fe/src/com/baidu/palo/catalog/Catalog.java | 32 ++++++++------- fe/src/com/baidu/palo/cluster/Cluster.java | 3 ++ .../baidu/palo/system/SystemInfoService.java | 41 ++++++++++++++----- gensrc/parser/sql_parser.y | 4 ++ 9 files changed, 108 insertions(+), 42 deletions(-) diff --git a/docs/help/Contents/Administration/admin_stmt.md b/docs/help/Contents/Administration/admin_stmt.md index fa65d1374367ea..9f3dd57769ee1a 100644 --- a/docs/help/Contents/Administration/admin_stmt.md +++ b/docs/help/Contents/Administration/admin_stmt.md @@ -3,19 +3,21 @@ 该语句用于操作一个系统内的节点。（仅管理员使用！） 语法： - 1) 增加节点 + 1) 增加节点(不使用多租户功能则按照此方法添加) ALTER SYSTEM ADD BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 2) 增加空闲节点 + 2) 增加空闲节点(即添加不属于任何cluster的BACKEND) ALTER SYSTEM ADD FREE BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 2) 删除节点 + 3) 增加节点到某个cluster + ALTER SYSTEM ADD BACKEND TO cluster_name "host:heartbeat_port"[,"host:heartbeat_port"...]; + 4) 删除节点 ALTER SYSTEM DROP BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 3) 节点下线 + 5) 节点下线 ALTER SYSTEM DECOMMISSION BACKEND "host:heartbeat_port"[,"host:heartbeat_port"...]; - 4) 增加Broker + 6) 增加Broker ALTER SYSTEM ADD BROKER broker_name "host:port"[,"host:port"...]; - 5) 减少Broker + 7) 减少Broker ALTER SYSTEM DROP BROKER broker_name "host:port"[,"host:port"...]; - 6) 删除所有Broker + 8) 删除所有Broker ALTER SYSTEM DROP ALL BROKER broker_name 说明： diff --git a/fe/src/com/baidu/palo/alter/SystemHandler.java b/fe/src/com/baidu/palo/alter/SystemHandler.java index f249708c06a7df..84cbcf4eadf745 100644 --- a/fe/src/com/baidu/palo/alter/SystemHandler.java +++ b/fe/src/com/baidu/palo/alter/SystemHandler.java @@ -47,6 +47,7 @@ import com.baidu.palo.thrift.TTabletInfo; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.lang.NotImplementedException; @@ -145,7 +146,14 @@ public synchronized void process(List alterClauses, String clusterN if (alterClause instanceof AddBackendClause) { AddBackendClause addBackendClause = (AddBackendClause) alterClause; - Catalog.getCurrentSystemInfo().addBackends(addBackendClause.getHostPortPairs(), addBackendClause.isFree()); + final String destClusterName = addBackendClause.getDestCluster(); + + if (!Strings.isNullOrEmpty(destClusterName) + && Catalog.getInstance().getCluster(destClusterName) == null) { + throw new DdlException("Cluster: " + destClusterName + " does not exist."); + } + Catalog.getCurrentSystemInfo().addBackends(addBackendClause.getHostPortPairs(), + addBackendClause.isFree(), addBackendClause.getDestCluster()); } else if (alterClause instanceof DropBackendClause) { DropBackendClause dropBackendClause = (DropBackendClause) alterClause; if (!dropBackendClause.isForce()) { diff --git a/fe/src/com/baidu/palo/analysis/AddBackendClause.java b/fe/src/com/baidu/palo/analysis/AddBackendClause.java index 0af5daa13c2e8f..827793cde363db 100644 --- a/fe/src/com/baidu/palo/analysis/AddBackendClause.java +++ b/fe/src/com/baidu/palo/analysis/AddBackendClause.java @@ -17,20 +17,43 @@ import java.util.List; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.google.common.base.Strings; + public class AddBackendClause extends BackendClause { // be in free state is not owned by any cluster protected boolean isFree; - + // cluster that backend will be added to + protected String destCluster; + public AddBackendClause(List hostPorts, boolean isFree) { super(hostPorts); this.isFree = isFree; + this.destCluster = ""; + } + + public AddBackendClause(List hostPorts, String destCluster) { + super(hostPorts); + this.isFree = false; + this.destCluster = destCluster; } @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("ADD BACKEND "); + sb.append("ADD "); + if (isFree) { + sb.append("FREE "); + } + sb.append("BACKEND "); + + if (!Strings.isNullOrEmpty(destCluster)) { + sb.append("to").append(destCluster); + } + for (int i = 0; i < hostPorts.size(); i++) { sb.append("\"").append(hostPorts.get(i)).append("\""); if (i != hostPorts.size() - 1) { @@ -40,12 +63,12 @@ public String toSql() { return sb.toString(); } - - public void setFree(boolean isFree) { - this.isFree = isFree; - } - public boolean isFree() { return this.isFree; } + + public String getDestCluster() { + return destCluster; + } + } diff --git a/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java b/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java index 3bb55a5da0d6ab..3822c7e965dc30 100644 --- a/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterClusterStmt.java @@ -56,7 +56,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException, InternalExcepti ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PARAMETER); } - if (instanceNum <= 0) { + if (instanceNum < 0) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_CREATE_ISTANCE_NUM_ERROR); } } diff --git a/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java b/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java index f2873b57f87873..fc0d7acfacd696 100644 --- a/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java +++ b/fe/src/com/baidu/palo/analysis/CreateClusterStmt.java @@ -75,10 +75,11 @@ public void analyze(Analyzer analyzer) throws AnalysisException, InternalExcepti } catch (NumberFormatException e) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_PARAMETER); } - if (instanceNum <= 0) { + + if (instanceNum < 0) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_CREATE_ISTANCE_NUM_ERROR); } - + final String password = passwd; if (!Strings.isNullOrEmpty(password)) { scramblePassword = MysqlPassword.makeScrambledPassword(password); diff --git a/fe/src/com/baidu/palo/catalog/Catalog.java b/fe/src/com/baidu/palo/catalog/Catalog.java index f349049b1a2c98..013a25ef2578bf 100644 --- a/fe/src/com/baidu/palo/catalog/Catalog.java +++ b/fe/src/com/baidu/palo/catalog/Catalog.java @@ -4505,7 +4505,9 @@ public void createCluster(CreateClusterStmt stmt) throws DdlException { ErrorReport.reportDdlException(ErrorCode.ERR_CLUSTER_HAS_EXIST, clusterName); } else { List backendList = systemInfo.createCluster(clusterName, stmt.getInstanceNum()); - if (backendList != null) { + // 1: BE returned is less than requested, throws DdlException. + // 2: BE returned is more than or equal to 0, succeeds. + if (backendList != null || stmt.getInstanceNum() == 0) { final long id = getNextId(); final Cluster cluster = new Cluster(); cluster.setName(clusterName); @@ -4948,8 +4950,13 @@ public void linkDb(LinkDbStmt stmt) throws DdlException { } } - public Cluster getCluster(String cluster) { - return nameToCluster.get(cluster); + public Cluster getCluster(String clusterName) { + readLock(); + try { + return nameToCluster.get(clusterName); + } finally { + readUnlock(); + } } public List getClusterNames() { @@ -5029,19 +5036,16 @@ public long loadCluster(DataInputStream dis, long checksum) throws IOException, cluster.readFields(dis); checksum ^= cluster.getId(); - // BE is in default_cluster when added , therefore it is possible that the BE - // in default_cluster are not the latest because cluster cant't be updated when - // loadCluster is after loadBackend. Because of forgeting to remove BE's id in - // cluster when drop BE or decommission in latest versions, need to update cluster's - // BE. List latestBackendIds = systemInfo.getClusterBackendIds(cluster.getName()); - if (cluster.getName().equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER) - || Catalog.getCurrentCatalogJournalVersion() <= FeMetaVersion.VERSION_34) { - cluster.setBackendIdList(latestBackendIds); - } else { - // The cluster has the same number of be as systeminfo recorded - Preconditions.checkState(latestBackendIds.size() == cluster.getBackendIdList().size()); + if (latestBackendIds.size() != cluster.getBackendIdList().size()) { + LOG.warn("Cluster:" + cluster.getName() + ", backends in Cluster is " + + cluster.getBackendIdList().size() + ", backends in SystemInfoService is " + + cluster.getBackendIdList().size()); } + // The number of BE in cluster is not same as in SystemInfoService, when perform 'ALTER + // SYSTEM ADD BACKEND TO ...' or 'ALTER SYSTEM ADD BACKEND ...', because both of them are + // for adding BE to some Cluster, but loadCluster is after loadBackend. + cluster.setBackendIdList(latestBackendIds); final InfoSchemaDb db = new InfoSchemaDb(cluster.getName()); db.setClusterName(cluster.getName()); diff --git a/fe/src/com/baidu/palo/cluster/Cluster.java b/fe/src/com/baidu/palo/cluster/Cluster.java index eaa64db7c9e7c9..89d798338bf82e 100644 --- a/fe/src/com/baidu/palo/cluster/Cluster.java +++ b/fe/src/com/baidu/palo/cluster/Cluster.java @@ -239,6 +239,9 @@ public List getBackendIdList() { } public void setBackendIdList(List backendIdList) { + if (backendIdList == null) { + return; + } writeLock(); try { this.backendIdSet = Sets.newHashSet(backendIdList); diff --git a/fe/src/com/baidu/palo/system/SystemInfoService.java b/fe/src/com/baidu/palo/system/SystemInfoService.java index 53169979685991..2de5223e2181e8 100644 --- a/fe/src/com/baidu/palo/system/SystemInfoService.java +++ b/fe/src/com/baidu/palo/system/SystemInfoService.java @@ -132,6 +132,18 @@ public void setMaster(String masterHost, int masterPort, int clusterId, String t } public void addBackends(List> hostPortPairs, boolean isFree) throws DdlException { + addBackends(hostPortPairs, isFree, ""); + } + + /** + * + * @param hostPortPairs : backend's host and port + * @param isFree : if true the backend is not owned by any cluster + * @param destCluster : if not null or empty backend will be added to destCluster + * @throws DdlException + */ + public void addBackends(List> hostPortPairs, + boolean isFree, String destCluster) throws DdlException { for (Pair pair : hostPortPairs) { // check is already exist if (getBackendWithHeartbeatPort(pair.first, pair.second) != null) { @@ -140,7 +152,7 @@ public void addBackends(List> hostPortPairs, boolean isFre } for (Pair pair : hostPortPairs) { - addBackend(pair.first, pair.second, isFree); + addBackend(pair.first, pair.second, isFree, destCluster); } } @@ -152,7 +164,15 @@ public void addBackend(Backend backend) { idToBackendRef.set(newIdToBackend); } - private void addBackend(String host, int heartbeatPort, boolean isFree) throws DdlException { + private void setBackendOwner(Backend backend, String clusterName) { + final Cluster cluster = Catalog.getInstance().getCluster(clusterName); + Preconditions.checkState(cluster != null); + cluster.addBackend(backend.getId()); + backend.setOwnerClusterName(clusterName); + backend.setBackendState(BackendState.using); + } + + private void addBackend(String host, int heartbeatPort, boolean isFree, String destCluster) throws DdlException { Backend newBackend = new Backend(Catalog.getInstance().getNextId(), host, heartbeatPort); // update idToBackend Map copiedBackends = Maps.newHashMap(idToBackendRef.get()); @@ -174,13 +194,14 @@ private void addBackend(String host, int heartbeatPort, boolean isFree) throws D ImmutableMap newIdToHeartbeatHandler = ImmutableMap.copyOf(copiedHeartbeatHandlersMap); idToHeartbeatHandlerRef.set(newIdToHeartbeatHandler); - // to add be to DEFAULT_CLUSTER - if (!isFree) { - final Cluster cluster = Catalog.getInstance().getCluster(DEFAULT_CLUSTER); - Preconditions.checkState(cluster != null); - cluster.addBackend(newBackend.getId()); - newBackend.setOwnerClusterName(DEFAULT_CLUSTER); - newBackend.setBackendState(BackendState.using); + if (!Strings.isNullOrEmpty(destCluster)) { + // add backend to destCluster + setBackendOwner(newBackend, destCluster); + } else if (!isFree) { + // add backend to DEFAULT_CLUSTER + setBackendOwner(newBackend, DEFAULT_CLUSTER); + } else { + // backend is free } // log @@ -327,7 +348,7 @@ public List getBackendIds(boolean needAlive) { * * @param clusterName * @param instanceNum - * @return + * @return if BE avaliable is less than requested , return null. */ public List createCluster(String clusterName, int instanceNum) { final List chosenBackendIds = Lists.newArrayList(); diff --git a/gensrc/parser/sql_parser.y b/gensrc/parser/sql_parser.y index 9ded411666c18b..aa43a6d4c5092a 100644 --- a/gensrc/parser/sql_parser.y +++ b/gensrc/parser/sql_parser.y @@ -686,6 +686,10 @@ alter_system_clause ::= {: RESULT = new AddBackendClause(hostPorts, true); :} + | KW_ADD KW_BACKEND KW_TO ident:clusterName string_list:hostPorts + {: + RESULT = new AddBackendClause(hostPorts, clusterName); + :} | KW_DROP KW_BACKEND string_list:hostPorts {: RESULT = new DropBackendClause(hostPorts, false); From 236d5fd073ff6d52236f1d4cc1cb93acc89b14af Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Wed, 22 Nov 2017 11:27:05 +0800 Subject: [PATCH 3/7] find left most node error --- fe/src/com/baidu/palo/qe/Coordinator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/com/baidu/palo/qe/Coordinator.java b/fe/src/com/baidu/palo/qe/Coordinator.java index 8982accfe09ffd..df8d0953e2aef3 100644 --- a/fe/src/com/baidu/palo/qe/Coordinator.java +++ b/fe/src/com/baidu/palo/qe/Coordinator.java @@ -772,7 +772,7 @@ private void computeFragmentHosts() throws Exception { // or INVALID_PLAN_NODE_ID if no such node present. private PlanNode findLeftmostNode(PlanNode plan) { PlanNode newPlan = plan; - while (newPlan.getChildren().size() != 0) { + while (newPlan.getChildren().size() != 0 && !(newPlan instanceof ExchangeNode)) { newPlan = newPlan.getChild(0); } return newPlan; From 46dc0f610e31dde9e1c2e7bd8f1a27206cfc3808 Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Wed, 22 Nov 2017 11:31:23 +0800 Subject: [PATCH 4/7] change white list limit --- .../baidu/palo/analysis/AlterUserStmt.java | 243 +++++++++--------- fe/src/com/baidu/palo/common/Config.java | 3 + 2 files changed, 126 insertions(+), 120 deletions(-) diff --git a/fe/src/com/baidu/palo/analysis/AlterUserStmt.java b/fe/src/com/baidu/palo/analysis/AlterUserStmt.java index 7234467086b96e..b04eafb1659e2d 100644 --- a/fe/src/com/baidu/palo/analysis/AlterUserStmt.java +++ b/fe/src/com/baidu/palo/analysis/AlterUserStmt.java @@ -18,123 +18,126 @@ // specific language governing permissions and limitations // under the License. -package com.baidu.palo.analysis; - -import java.util.List; - -import org.apache.commons.lang.NotImplementedException; - -import com.baidu.palo.catalog.AccessPrivilege; -import com.baidu.palo.cluster.ClusterNamespace; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.ErrorCode; -import com.baidu.palo.common.ErrorReport; -import com.baidu.palo.common.InternalException; -import com.google.common.base.Strings; - -public class AlterUserStmt extends DdlStmt { - private String userName; - private AlterUserClause clause; - - public AlterUserStmt(String userName, AlterClause clause) { - this.userName = userName; - this.clause = (AlterUserClause) clause; - } - - private boolean hasRightToModify(Analyzer analyzer) { - String user = analyzer.getUser(); - String toUser = userName; - - // own can modify own - if (user.equals(toUser)) { - return true; - } - - // admin can modify all - if (analyzer.getCatalog().getUserMgr().isAdmin(user)) { - return true; - } - - // superuse can modify Ordinary user - if (analyzer.getCatalog().getUserMgr().isSuperuser(user) - && !analyzer.getCatalog().getUserMgr().isSuperuser(toUser)) { - return true; - } - return false; - } - - private void checkWhiteListSize(Analyzer analyzer) throws AnalysisException { - if (clause.getAlterUserType() == AlterUserType.ADD_USER_WHITELIST) { - try { - if (analyzer.getCatalog().getUserMgr().getWhiteListSize(userName) > 20) { - throw new AnalysisException("whitelist size excced the max (20)"); - } - } catch (DdlException e) { - throw new AnalysisException(e.getMessage()); - } - } - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { - super.analyze(analyzer); - // check toUser - if (Strings.isNullOrEmpty(userName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "empty user"); - } - userName = ClusterNamespace.getFullName(getClusterName(), userName); - // check destination user if exists - try { - analyzer.getCatalog().getUserMgr().checkUserIfExist(userName); - } catch (DdlException e) { - throw new AnalysisException(e.getMessage()); - } - - // check destination user's whitelist if ecceed max value - checkWhiteListSize(analyzer); - - // only write user can modify - analyzer.checkPrivilege(analyzer.getDefaultDb(), AccessPrivilege.READ_WRITE); - - // check if has the right - if (!hasRightToModify(analyzer)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ALTER CLUSTER"); - } - - // alter clause analysis - clause.analyze(analyzer); - } - - public String getUser() { - return userName; - } - - public List getHosts() { - return clause.getHosts(); - } - - public List getIps() { - return clause.getIps(); - } - - public List getStarIps() { - return clause.getStarIps(); - } - - public AlterUserType getAlterUserType() { - return clause.getAlterUserType(); - } - - @Override - public String toSql() { - throw new NotImplementedException(); - } - - @Override - public String toString() { - throw new NotImplementedException(); - } - - -} +package com.baidu.palo.analysis; + +import java.util.List; + +import org.apache.commons.lang.NotImplementedException; + +import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.cluster.ClusterNamespace; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.ErrorCode; +import com.baidu.palo.common.ErrorReport; +import com.baidu.palo.common.InternalException; +import com.google.common.base.Strings; + +public class AlterUserStmt extends DdlStmt { + private String userName; + private AlterUserClause clause; + + public AlterUserStmt(String userName, AlterClause clause) { + this.userName = userName; + this.clause = (AlterUserClause) clause; + } + + private boolean hasRightToModify(Analyzer analyzer) { + String user = analyzer.getUser(); + String toUser = userName; + + // own can modify own + if (user.equals(toUser)) { + return true; + } + + // admin can modify all + if (analyzer.getCatalog().getUserMgr().isAdmin(user)) { + return true; + } + + // superuse can modify Ordinary user + if (analyzer.getCatalog().getUserMgr().isSuperuser(user) + && !analyzer.getCatalog().getUserMgr().isSuperuser(toUser)) { + return true; + } + return false; + } + + private void checkWhiteListSize(Analyzer analyzer) throws AnalysisException { + if (clause.getAlterUserType() == AlterUserType.ADD_USER_WHITELIST) { + try { + if (analyzer.getCatalog().getUserMgr().getWhiteListSize(userName) + > Config.per_user_white_list_limit) { + throw new AnalysisException("whitelist size excced the max (" + + Config.per_user_white_list_limit + ")"); + } + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + } + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { + super.analyze(analyzer); + // check toUser + if (Strings.isNullOrEmpty(userName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "empty user"); + } + userName = ClusterNamespace.getFullName(getClusterName(), userName); + // check destination user if exists + try { + analyzer.getCatalog().getUserMgr().checkUserIfExist(userName); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + + // check destination user's whitelist if ecceed max value + checkWhiteListSize(analyzer); + + // only write user can modify + analyzer.checkPrivilege(analyzer.getDefaultDb(), AccessPrivilege.READ_WRITE); + + // check if has the right + if (!hasRightToModify(analyzer)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ALTER CLUSTER"); + } + + // alter clause analysis + clause.analyze(analyzer); + } + + public String getUser() { + return userName; + } + + public List getHosts() { + return clause.getHosts(); + } + + public List getIps() { + return clause.getIps(); + } + + public List getStarIps() { + return clause.getStarIps(); + } + + public AlterUserType getAlterUserType() { + return clause.getAlterUserType(); + } + + @Override + public String toSql() { + throw new NotImplementedException(); + } + + @Override + public String toString() { + throw new NotImplementedException(); + } + + +} diff --git a/fe/src/com/baidu/palo/common/Config.java b/fe/src/com/baidu/palo/common/Config.java index 8c0e0407cd29eb..5be2d1c5155c2c 100644 --- a/fe/src/com/baidu/palo/common/Config.java +++ b/fe/src/com/baidu/palo/common/Config.java @@ -461,4 +461,7 @@ public class Config extends ConfigBase { // for forward compatibility, will be removed later. // check token when download image file. @ConfField public static boolean enable_token_check = true; + + // white list limit + @ConfField public static int per_user_white_list_limit = 1024; } From 039f765b92f102965c868d02cc453c0b6b445fe6 Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Tue, 28 Nov 2017 12:05:02 +0800 Subject: [PATCH 5/7] import ExchangeNode in Coordinator --- fe/src/com/baidu/palo/qe/Coordinator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/com/baidu/palo/qe/Coordinator.java b/fe/src/com/baidu/palo/qe/Coordinator.java index df8d0953e2aef3..a7a828a3460860 100644 --- a/fe/src/com/baidu/palo/qe/Coordinator.java +++ b/fe/src/com/baidu/palo/qe/Coordinator.java @@ -27,6 +27,7 @@ import com.baidu.palo.common.util.RuntimeProfile; import com.baidu.palo.planner.DataPartition; import com.baidu.palo.planner.DataSink; +import com.baidu.palo.planner.ExchangeNode; import com.baidu.palo.planner.PlanFragment; import com.baidu.palo.planner.PlanFragmentId; import com.baidu.palo.planner.PlanNode; From fda98cad781a216f1a1e6c909f5b6ca75f405a30 Mon Sep 17 00:00:00 2001 From: chenhao7253886 <510341142@qq.com> Date: Tue, 28 Nov 2017 12:10:08 +0800 Subject: [PATCH 6/7] import ExchangeNode in Coordinator --- fe/src/com/baidu/palo/qe/Coordinator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/com/baidu/palo/qe/Coordinator.java b/fe/src/com/baidu/palo/qe/Coordinator.java index df8d0953e2aef3..a7a828a3460860 100644 --- a/fe/src/com/baidu/palo/qe/Coordinator.java +++ b/fe/src/com/baidu/palo/qe/Coordinator.java @@ -27,6 +27,7 @@ import com.baidu.palo.common.util.RuntimeProfile; import com.baidu.palo.planner.DataPartition; import com.baidu.palo.planner.DataSink; +import com.baidu.palo.planner.ExchangeNode; import com.baidu.palo.planner.PlanFragment; import com.baidu.palo.planner.PlanFragmentId; import com.baidu.palo.planner.PlanNode; From 02e6925d313971f75faaba98d46f6850ac694106 Mon Sep 17 00:00:00 2001 From: chenhao16 Date: Wed, 29 Nov 2017 15:10:10 +0800 Subject: [PATCH 7/7] produce exec param error when plan has multi UnionNode and revert changes about literal by yanlei --- .../com/baidu/palo/analysis/DateLiteral.java | 2 ++ .../baidu/palo/analysis/DecimalLiteral.java | 2 ++ .../com/baidu/palo/analysis/FloatLiteral.java | 1 + .../com/baidu/palo/analysis/IntLiteral.java | 3 +++ .../com/baidu/palo/analysis/LiteralExpr.java | 1 - .../baidu/palo/analysis/StringLiteral.java | 1 + fe/src/com/baidu/palo/qe/Coordinator.java | 27 ++++++++++++++++++- 7 files changed, 35 insertions(+), 2 deletions(-) diff --git a/fe/src/com/baidu/palo/analysis/DateLiteral.java b/fe/src/com/baidu/palo/analysis/DateLiteral.java index ce526a25fa4e45..630a1e66c17eed 100644 --- a/fe/src/com/baidu/palo/analysis/DateLiteral.java +++ b/fe/src/com/baidu/palo/analysis/DateLiteral.java @@ -55,11 +55,13 @@ public DateLiteral(Type type, boolean isMax) throws AnalysisException { } else { date = isMax ? TimeUtils.MAX_DATETIME : TimeUtils.MIN_DATETIME; } + analysisDone(); } public DateLiteral(String s, Type type) throws AnalysisException { super(); init(s, type); + analysisDone(); } protected DateLiteral(DateLiteral other) { diff --git a/fe/src/com/baidu/palo/analysis/DecimalLiteral.java b/fe/src/com/baidu/palo/analysis/DecimalLiteral.java index 978a0675868d14..181e6edc4d745e 100644 --- a/fe/src/com/baidu/palo/analysis/DecimalLiteral.java +++ b/fe/src/com/baidu/palo/analysis/DecimalLiteral.java @@ -50,6 +50,7 @@ public DecimalLiteral() { public DecimalLiteral(BigDecimal value) { init(value); + analysisDone(); } public DecimalLiteral(String value) throws AnalysisException { @@ -60,6 +61,7 @@ public DecimalLiteral(String value) throws AnalysisException { throw new AnalysisException("Invalid floating-point literal: " + value, e); } init(v); + analysisDone(); } protected DecimalLiteral(DecimalLiteral other) { diff --git a/fe/src/com/baidu/palo/analysis/FloatLiteral.java b/fe/src/com/baidu/palo/analysis/FloatLiteral.java index 2594ef13ad7529..f3ce542d35309a 100644 --- a/fe/src/com/baidu/palo/analysis/FloatLiteral.java +++ b/fe/src/com/baidu/palo/analysis/FloatLiteral.java @@ -51,6 +51,7 @@ public FloatLiteral(Double value) { public FloatLiteral(Double value, Type type) { this.value = value.doubleValue(); this.type = type; + analysisDone(); } public FloatLiteral(String value) throws AnalysisException { diff --git a/fe/src/com/baidu/palo/analysis/IntLiteral.java b/fe/src/com/baidu/palo/analysis/IntLiteral.java index 1dae34b3fe57c9..4dd65925f6a0e4 100644 --- a/fe/src/com/baidu/palo/analysis/IntLiteral.java +++ b/fe/src/com/baidu/palo/analysis/IntLiteral.java @@ -64,6 +64,7 @@ private IntLiteral() { public IntLiteral(long value) { super(); init(value); + analysisDone(); } public IntLiteral(long longValue, Type type) throws AnalysisException { @@ -102,6 +103,7 @@ public IntLiteral(long longValue, Type type) throws AnalysisException { this.value = longValue; this.type = type; + analysisDone(); } public IntLiteral(String value, Type type) throws AnalysisException { @@ -147,6 +149,7 @@ public IntLiteral(String value, Type type) throws AnalysisException { this.value = longValue; this.type = type; + analysisDone(); } protected IntLiteral(IntLiteral other) { diff --git a/fe/src/com/baidu/palo/analysis/LiteralExpr.java b/fe/src/com/baidu/palo/analysis/LiteralExpr.java index b8615e7e4eb433..6a3efeb690d3b8 100644 --- a/fe/src/com/baidu/palo/analysis/LiteralExpr.java +++ b/fe/src/com/baidu/palo/analysis/LiteralExpr.java @@ -40,7 +40,6 @@ public abstract class LiteralExpr extends Expr { public LiteralExpr() { numDistinctValues = 1; - isAnalyzed = true; } protected LiteralExpr(LiteralExpr other) { diff --git a/fe/src/com/baidu/palo/analysis/StringLiteral.java b/fe/src/com/baidu/palo/analysis/StringLiteral.java index 64bb1756fc61ac..1478f5ed3834b7 100644 --- a/fe/src/com/baidu/palo/analysis/StringLiteral.java +++ b/fe/src/com/baidu/palo/analysis/StringLiteral.java @@ -50,6 +50,7 @@ public StringLiteral(String value) { super(); this.value = value; type = Type.VARCHAR; + analysisDone(); } protected StringLiteral(StringLiteral other) { diff --git a/fe/src/com/baidu/palo/qe/Coordinator.java b/fe/src/com/baidu/palo/qe/Coordinator.java index a7a828a3460860..bd48fbf1009070 100644 --- a/fe/src/com/baidu/palo/qe/Coordinator.java +++ b/fe/src/com/baidu/palo/qe/Coordinator.java @@ -35,6 +35,7 @@ import com.baidu.palo.planner.Planner; import com.baidu.palo.planner.ResultSink; import com.baidu.palo.planner.ScanNode; +import com.baidu.palo.planner.UnionNode; import com.baidu.palo.service.FrontendOptions; import com.baidu.palo.system.Backend; import com.baidu.palo.task.LoadEtlTask; @@ -713,6 +714,25 @@ private TNetworkAddress toRpcHost(TNetworkAddress host) throws Exception { return dest; } + // estimate if this fragment contains UnionNode + private boolean containsUnionNode(PlanNode node) { + if (node instanceof UnionNode) { + return true; + } + + for (PlanNode child : node.getChildren()) { + if (child instanceof ExchangeNode) { + // Ignore other fragment's node + continue; + } else if (child instanceof UnionNode) { + return true; + } else { + return containsUnionNode(child); + } + } + return false; + } + // For each fragment in fragments, computes hosts on which to run the instances // and stores result in fragmentExecParams.hosts. private void computeFragmentHosts() throws Exception { @@ -737,7 +757,12 @@ private void computeFragmentHosts() throws Exception { } PlanNode leftMostNode = findLeftmostNode(fragment.getPlanRoot()); - if (!(leftMostNode instanceof ScanNode)) { + // When fragment contains UnionNode, because the fragment may has child + // and not all BE will receive the fragment, child fragment's dest must + // be BE that fragment's scannode locates, avoid less data. + // chenhao added + boolean hasUnionNode = containsUnionNode(fragment.getPlanRoot()); + if (!(leftMostNode instanceof ScanNode) && !hasUnionNode) { // there is no leftmost scan; we assign the same hosts as those of our // leftmost input fragment (so that a partitioned aggregation // fragment runs on the hosts that provide the input data)