diff --git a/fe/src/com/baidu/palo/clone/CloneChecker.java b/fe/src/com/baidu/palo/clone/CloneChecker.java index e1468ea3040fa5..716539925b09cc 100644 --- a/fe/src/com/baidu/palo/clone/CloneChecker.java +++ b/fe/src/com/baidu/palo/clone/CloneChecker.java @@ -31,6 +31,7 @@ import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.DataProperty; import com.baidu.palo.catalog.Database; +import com.baidu.palo.catalog.Database.DbState; import com.baidu.palo.catalog.MaterializedIndex; import com.baidu.palo.catalog.MaterializedIndex.IndexState; import com.baidu.palo.catalog.OlapTable; @@ -42,7 +43,6 @@ import com.baidu.palo.catalog.Table; import com.baidu.palo.catalog.Table.TableType; import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.catalog.Database.DbState; import com.baidu.palo.clone.CloneJob.JobPriority; import com.baidu.palo.clone.CloneJob.JobState; import com.baidu.palo.clone.CloneJob.JobType; @@ -238,6 +238,11 @@ private void checkTablets() { List dbNames = catalog.getDbNames(); for (String name : dbNames) { Database db = catalog.getDb(name); + if (db == null) { + LOG.warn("db does not exist. name: {}", name); + continue; + } + final String clusterName = db.getClusterName(); if (Strings.isNullOrEmpty(clusterName)) { @@ -252,16 +257,13 @@ private void checkTablets() { LOG.warn("init backend infos error"); continue; } - final Map> cluserCapacityLevelToBackendIds = initBackendCapacityInfos(clusterBackendInfos); - if (cluserCapacityLevelToBackendIds == null || cluserCapacityLevelToBackendIds.isEmpty()) { + final Map> clusterCapacityLevelToBackendIds = initBackendCapacityInfos( + clusterBackendInfos); + if (clusterCapacityLevelToBackendIds == null || clusterCapacityLevelToBackendIds.isEmpty()) { LOG.warn("init backend capacity infos error"); continue; } - if (db == null) { - LOG.warn("db does not exist. name: {}", db.getName()); - continue; - } long dbId = db.getId(); Set tableNames = db.getTableNamesWithLock(); @@ -395,10 +397,10 @@ private void checkTablets() { && !clusterDistributionLevelToBackendIds.isEmpty()) { // supplement checkSupplement(cloneTabletMap, clusterDistributionLevelToBackendIds, - cluserCapacityLevelToBackendIds, clusterBackendInfos); + clusterCapacityLevelToBackendIds, clusterBackendInfos); // migration checkMigration(backendToTablets, clusterDistributionLevelToBackendIds, - cluserCapacityLevelToBackendIds, clusterBackendInfos); + clusterCapacityLevelToBackendIds, clusterBackendInfos); } else { LOG.warn("init backend distribution infos error"); } diff --git a/fe/src/com/baidu/palo/http/BaseAction.java b/fe/src/com/baidu/palo/http/BaseAction.java index c01b48efb6a3d6..815a3d6059f3f4 100644 --- a/fe/src/com/baidu/palo/http/BaseAction.java +++ b/fe/src/com/baidu/palo/http/BaseAction.java @@ -221,7 +221,7 @@ protected void writeCookies(BaseResponse response, HttpResponse responseObj) { } public static class AuthorizationInfo { - public String user; + public String fullUserName; public String password; public String cluster; } @@ -246,13 +246,14 @@ public boolean parseAuth(BaseRequest request, AuthorizationInfo authInfo) { // Note that password may contain colon, so can not simply use a // colon to split. int index = authString.indexOf(":"); - authInfo.user = authString.substring(0, index); - final String[] elements = authInfo.user.split("@"); + authInfo.fullUserName = authString.substring(0, index); + final String[] elements = authInfo.fullUserName.split("@"); if (elements != null && elements.length < 2) { - authInfo.user = ClusterNamespace.getUserFullName(SystemInfoService.DEFAULT_CLUSTER, authInfo.user); + authInfo.fullUserName = ClusterNamespace.getUserFullName(SystemInfoService.DEFAULT_CLUSTER, + authInfo.fullUserName); authInfo.cluster = SystemInfoService.DEFAULT_CLUSTER; } else if (elements != null && elements.length == 2) { - authInfo.user = ClusterNamespace.getUserFullName(elements[1], elements[0]); + authInfo.fullUserName = ClusterNamespace.getUserFullName(elements[1], elements[0]); authInfo.cluster = elements[1]; } authInfo.password = authString.substring(index + 1); @@ -272,10 +273,10 @@ private AuthorizationInfo checkAndGetUser(BaseRequest request) throws DdlExcepti if (!parseAuth(request, authInfo)) { throw new UnauthorizedException("Need auth information."); } - byte[] hashedPasswd = catalog.getUserMgr().getPassword(authInfo.user); + byte[] hashedPasswd = catalog.getUserMgr().getPassword(authInfo.fullUserName); if (hashedPasswd == null) { // No such user - throw new DdlException("No such user(" + authInfo.user + ")"); + throw new DdlException("No such user(" + authInfo.fullUserName + ")"); } if (!MysqlPassword.checkPlainPass(hashedPasswd, authInfo.password)) { throw new DdlException("Password error"); @@ -285,21 +286,19 @@ private AuthorizationInfo checkAndGetUser(BaseRequest request) throws DdlExcepti protected void checkAdmin(BaseRequest request) throws DdlException { final AuthorizationInfo authInfo = checkAndGetUser(request); - if (!catalog.getUserMgr().isAdmin(authInfo.user)) { + if (!catalog.getUserMgr().isAdmin(authInfo.fullUserName)) { throw new DdlException("Administrator needed"); } } - protected void checkReadPriv(BaseRequest request, String db) throws DdlException { - final AuthorizationInfo authInfo = checkAndGetUser(request); - if (!catalog.getUserMgr().checkAccess(authInfo.user, db, AccessPrivilege.READ_ONLY)) { + protected void checkReadPriv(String fullUserName, String fullDbName) throws DdlException { + if (!catalog.getUserMgr().checkAccess(fullUserName, fullDbName, AccessPrivilege.READ_ONLY)) { throw new DdlException("Read Privilege needed"); } } - protected void checkWritePriv(BaseRequest request, String db) throws DdlException { - final AuthorizationInfo authInfo = checkAndGetUser(request); - if (!catalog.getUserMgr().checkAccess(authInfo.user, db, AccessPrivilege.READ_WRITE)) { + protected void checkWritePriv(String fullUserName, String fullDbName) throws DdlException { + if (!catalog.getUserMgr().checkAccess(fullUserName, fullDbName, AccessPrivilege.READ_WRITE)) { throw new DdlException("Write Privilege needed"); } } diff --git a/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java b/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java index d0aadad75bd172..32298d75153db3 100644 --- a/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java +++ b/fe/src/com/baidu/palo/http/rest/GetLoadInfoAction.java @@ -15,13 +15,13 @@ package com.baidu.palo.http.rest; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.load.Load; - import com.google.common.base.Strings; import io.netty.handler.codec.http.HttpMethod; @@ -35,7 +35,7 @@ public GetLoadInfoAction(ActionController controller) { super(controller); } - public static void registerAction (ActionController controller) + public static void registerAction(ActionController controller) throws IllegalArgException { GetLoadInfoAction action = new GetLoadInfoAction(controller); controller.registerHandler(HttpMethod.GET, "/api/{db}/_load_info", action); @@ -43,16 +43,22 @@ public static void registerAction (ActionController controller) @Override public void execute(BaseRequest request, BaseResponse response) throws DdlException { - Load.JobInfo info = new Load.JobInfo( - request.getSingleParameter(DB_KEY), - request.getSingleParameter(LABEL_KEY)); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + Load.JobInfo info = new Load.JobInfo(request.getSingleParameter(DB_KEY), request.getSingleParameter(LABEL_KEY), + authInfo.cluster); if (Strings.isNullOrEmpty(info.dbName)) { throw new DdlException("No database selected"); } if (Strings.isNullOrEmpty(info.label)) { throw new DdlException("No label selected"); } - checkReadPriv(request, info.dbName); + if (Strings.isNullOrEmpty(info.clusterName)) { + throw new DdlException("No cluster name selected"); + } + + String fullDbName = ClusterNamespace.getDbFullName(info.clusterName, info.dbName); + checkReadPriv(authInfo.fullUserName, fullDbName); + if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/LoadAction.java b/fe/src/com/baidu/palo/http/rest/LoadAction.java index 39881ed92e1110..57d4404f35f137 100644 --- a/fe/src/com/baidu/palo/http/rest/LoadAction.java +++ b/fe/src/com/baidu/palo/http/rest/LoadAction.java @@ -85,14 +85,14 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept throw new DdlException("No table selected."); } - dbName = ClusterNamespace.getDbFullName(authInfo.cluster, dbName); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, dbName); String label = request.getSingleParameter(LABEL_NAME_PARAM); String subLabel = request.getSingleParameter(SUB_LABEL_NAME_PARAM); if (Strings.isNullOrEmpty(label)) { throw new DdlException("No label selected."); } - checkWritePriv(request, dbName); + checkWritePriv(authInfo.fullUserName, fullDbName); // Try to redirect to master if (redirectToMaster(request, response)) { return; diff --git a/fe/src/com/baidu/palo/http/rest/MultiAbort.java b/fe/src/com/baidu/palo/http/rest/MultiAbort.java index dde597bfa4fa02..262db65b8cb586 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiAbort.java +++ b/fe/src/com/baidu/palo/http/rest/MultiAbort.java @@ -15,14 +15,15 @@ package com.baidu.palo.http.rest; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.service.ExecuteEnv; - import com.google.common.base.Strings; + import io.netty.handler.codec.http.HttpMethod; public class MultiAbort extends RestBaseAction { @@ -52,7 +53,10 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept if (Strings.isNullOrEmpty(label)) { throw new DdlException("No label selected"); } - checkWritePriv(request, db); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db); + + checkWritePriv(authInfo.fullUserName, fullDbName); if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/MultiCommit.java b/fe/src/com/baidu/palo/http/rest/MultiCommit.java index 45598807b55f88..97b59f2a09fa90 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiCommit.java +++ b/fe/src/com/baidu/palo/http/rest/MultiCommit.java @@ -15,14 +15,15 @@ package com.baidu.palo.http.rest; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.service.ExecuteEnv; - import com.google.common.base.Strings; + import io.netty.handler.codec.http.HttpMethod; public class MultiCommit extends RestBaseAction { @@ -52,7 +53,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept if (Strings.isNullOrEmpty(label)) { throw new DdlException("No label selected"); } - checkWritePriv(request, db); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db); + + checkWritePriv(authInfo.fullUserName, fullDbName); + if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/MultiDesc.java b/fe/src/com/baidu/palo/http/rest/MultiDesc.java index c3e6ae01032c0a..77b419bd31936e 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiDesc.java +++ b/fe/src/com/baidu/palo/http/rest/MultiDesc.java @@ -15,18 +15,19 @@ package com.baidu.palo.http.rest; +import java.util.List; + +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.service.ExecuteEnv; - import com.google.common.base.Strings; import com.google.common.collect.Lists; -import io.netty.handler.codec.http.HttpMethod; -import java.util.List; +import io.netty.handler.codec.http.HttpMethod; // List all labels of one multi-load public class MultiDesc extends RestBaseAction { @@ -56,7 +57,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept if (Strings.isNullOrEmpty(label)) { throw new DdlException("No label selected"); } - checkReadPriv(request, db); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db); + + checkReadPriv(authInfo.fullUserName, fullDbName); + if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/MultiList.java b/fe/src/com/baidu/palo/http/rest/MultiList.java index bea6c1902cf91c..4c238380b007ac 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiList.java +++ b/fe/src/com/baidu/palo/http/rest/MultiList.java @@ -15,18 +15,19 @@ package com.baidu.palo.http.rest; +import java.util.List; + +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.service.ExecuteEnv; - import com.google.common.base.Strings; import com.google.common.collect.Lists; -import io.netty.handler.codec.http.HttpMethod; -import java.util.List; +import io.netty.handler.codec.http.HttpMethod; // list all multi load before commit public class MultiList extends RestBaseAction { @@ -51,7 +52,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept if (Strings.isNullOrEmpty(db)) { throw new DdlException("No database selected"); } - checkReadPriv(request, db); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db); + + checkReadPriv(authInfo.fullUserName, fullDbName); + if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/http/rest/MultiStart.java b/fe/src/com/baidu/palo/http/rest/MultiStart.java index b206cac4933054..54758f2814ba1a 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiStart.java +++ b/fe/src/com/baidu/palo/http/rest/MultiStart.java @@ -15,19 +15,20 @@ package com.baidu.palo.http.rest; +import java.util.Map; + import com.baidu.palo.analysis.LoadStmt; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.service.ExecuteEnv; - import com.google.common.base.Strings; import com.google.common.collect.Maps; -import io.netty.handler.codec.http.HttpMethod; -import java.util.Map; +import io.netty.handler.codec.http.HttpMethod; // Start multi action public class MultiStart extends RestBaseAction { @@ -57,7 +58,10 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept if (Strings.isNullOrEmpty(label)) { throw new DdlException("No label selected"); } - checkWritePriv(request, db); + AuthorizationInfo authInfo = getAuthorizationInfo(request); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db); + + checkWritePriv(authInfo.fullUserName, fullDbName); if (redirectToMaster(request, response)) { return; diff --git a/fe/src/com/baidu/palo/http/rest/MultiUnload.java b/fe/src/com/baidu/palo/http/rest/MultiUnload.java index c8e0ef22764f52..510f92442ccae2 100644 --- a/fe/src/com/baidu/palo/http/rest/MultiUnload.java +++ b/fe/src/com/baidu/palo/http/rest/MultiUnload.java @@ -15,14 +15,15 @@ package com.baidu.palo.http.rest; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.DdlException; import com.baidu.palo.http.ActionController; import com.baidu.palo.http.BaseRequest; import com.baidu.palo.http.BaseResponse; import com.baidu.palo.http.IllegalArgException; import com.baidu.palo.service.ExecuteEnv; - import com.google.common.base.Strings; + import io.netty.handler.codec.http.HttpMethod; public class MultiUnload extends RestBaseAction { @@ -57,7 +58,11 @@ public void execute(BaseRequest request, BaseResponse response) throws DdlExcept if (Strings.isNullOrEmpty(subLabel)) { throw new DdlException("No sub_label selected"); } - checkWritePriv(request, db); + + AuthorizationInfo authInfo = getAuthorizationInfo(request); + String fullDbName = ClusterNamespace.getDbFullName(authInfo.cluster, db); + checkWritePriv(authInfo.fullUserName, fullDbName); + if (redirectToMaster(request, response)) { return; } diff --git a/fe/src/com/baidu/palo/load/Load.java b/fe/src/com/baidu/palo/load/Load.java index d81f7e08e0067f..21a01f6100817f 100644 --- a/fe/src/com/baidu/palo/load/Load.java +++ b/fe/src/com/baidu/palo/load/Load.java @@ -15,6 +15,25 @@ package com.baidu.palo.load; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.baidu.palo.analysis.BinaryPredicate; import com.baidu.palo.analysis.CancelLoadStmt; import com.baidu.palo.analysis.ColumnSeparator; @@ -43,6 +62,7 @@ import com.baidu.palo.catalog.TabletInvertedIndex; import com.baidu.palo.catalog.TabletMeta; import com.baidu.palo.catalog.Type; +import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; @@ -70,12 +90,11 @@ import com.baidu.palo.task.AgentTaskQueue; import com.baidu.palo.task.CancelDeleteTask; import com.baidu.palo.task.PushTask; -import com.baidu.palo.thrift.TMiniLoadRequest; import com.baidu.palo.thrift.TEtlState; +import com.baidu.palo.thrift.TMiniLoadRequest; import com.baidu.palo.thrift.TNetworkAddress; import com.baidu.palo.thrift.TPriority; import com.baidu.palo.thrift.TPushType; - import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -83,24 +102,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.Gson; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; public class Load { private static final Logger LOG = LogManager.getLogger(Load.class); @@ -1493,20 +1494,23 @@ public void changeLoadErrorHubInfo(LoadErrorHub.Param info) { public static class JobInfo { public String dbName; public String label; + public String clusterName; public JobState state; public String failMsg; public String trackingUrl; - public JobInfo(String dbName, String label) { + public JobInfo(String dbName, String label, String clusterName) { this.dbName = dbName; this.label = label; + this.clusterName = clusterName; } } // Get job state // result saved in info public void getJobInfo(JobInfo info) throws DdlException { - Database db = Catalog.getInstance().getDb(info.dbName); + String fullDbName = ClusterNamespace.getDbFullName(info.clusterName, info.dbName); + Database db = Catalog.getInstance().getDb(fullDbName); if (db == null) { throw new DdlException("Unknown database(" + info.dbName + ")"); } diff --git a/fe/test/com/baidu/palo/analysis/AccessTestUtil.java b/fe/test/com/baidu/palo/analysis/AccessTestUtil.java index 0b44697a2da69f..51fc208caaa782 100644 --- a/fe/test/com/baidu/palo/analysis/AccessTestUtil.java +++ b/fe/test/com/baidu/palo/analysis/AccessTestUtil.java @@ -20,36 +20,35 @@ package com.baidu.palo.analysis; +import java.util.LinkedList; +import java.util.List; + +import org.easymock.EasyMock; + import com.baidu.palo.alter.RollupHandler; import com.baidu.palo.alter.SchemaChangeHandler; import com.baidu.palo.catalog.AccessPrivilege; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.KeysType; import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.MaterializedIndex.IndexState; +import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.Partition; +import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.catalog.RandomDistributionInfo; import com.baidu.palo.catalog.SinglePartitionInfo; import com.baidu.palo.catalog.UserPropertyMgr; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.catalog.OlapTable; -import com.baidu.palo.catalog.PrimitiveType; -import com.baidu.palo.catalog.MaterializedIndex.IndexState; -import com.baidu.palo.system.SystemInfoService; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.DdlException; import com.baidu.palo.load.Load; import com.baidu.palo.persist.EditLog; import com.baidu.palo.qe.ConnectContext; - +import com.baidu.palo.system.SystemInfoService; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.easymock.EasyMock; - -import java.util.LinkedList; -import java.util.List; - public class AccessTestUtil { public static UserPropertyMgr fetchAdminAccess() { UserPropertyMgr userPropertyMgr = EasyMock.createMock(UserPropertyMgr.class); @@ -116,6 +115,8 @@ public static Catalog fetchAdminCatalog() { return catalog; } catch (DdlException e) { return null; + } catch (AnalysisException e) { + return null; } } @@ -184,6 +185,8 @@ public static Catalog fetchBlockCatalog() { return catalog; } catch (DdlException e) { return null; + } catch (AnalysisException e) { + return null; } } diff --git a/fe/test/com/baidu/palo/analysis/AddColumnClauseTest.java b/fe/test/com/baidu/palo/analysis/AddColumnClauseTest.java index ed402695f2c732..359db24c91c6e6 100644 --- a/fe/test/com/baidu/palo/analysis/AddColumnClauseTest.java +++ b/fe/test/com/baidu/palo/analysis/AddColumnClauseTest.java @@ -20,15 +20,15 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.AggregateType; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.common.AnalysisException; - +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.BeforeClass; -import org.easymock.EasyMock; import org.junit.Test; +import com.baidu.palo.catalog.AggregateType; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.common.AnalysisException; + public class AddColumnClauseTest { private static Analyzer analyzer; @@ -40,7 +40,7 @@ public static void setUp() { @Test public void testNormal() throws AnalysisException { Column definition = EasyMock.createMock(Column.class); - definition.analyze(); + definition.analyze(true); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(definition.toSql()).andReturn("`testCol` INT").anyTimes(); EasyMock.expect(definition.getDefaultValue()).andReturn("").anyTimes(); @@ -79,7 +79,7 @@ public void testNoColDef() throws AnalysisException { @Test(expected = AnalysisException.class) public void testNoDefault() throws AnalysisException { Column definition = EasyMock.createMock(Column.class); - definition.analyze(); + definition.analyze(true); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(definition.toSql()).andReturn("`testCol` INT").anyTimes(); EasyMock.expect(definition.getDefaultValue()).andReturn(null).anyTimes(); @@ -95,7 +95,7 @@ public void testNoDefault() throws AnalysisException { @Test(expected = AnalysisException.class) public void testAggPos() throws AnalysisException { Column definition = EasyMock.createMock(Column.class); - definition.analyze(); + definition.analyze(true); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(definition.toSql()).andReturn("`testCol` INT").anyTimes(); EasyMock.expect(definition.getDefaultValue()).andReturn(null).anyTimes(); @@ -111,7 +111,7 @@ public void testAggPos() throws AnalysisException { @Test(expected = AnalysisException.class) public void testAddValueToFirst() throws AnalysisException { Column definition = EasyMock.createMock(Column.class); - definition.analyze(); + definition.analyze(true); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(definition.toSql()).andReturn("`testCol` INT").anyTimes(); EasyMock.expect(definition.getDefaultValue()).andReturn("2").anyTimes(); diff --git a/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java b/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java index 8a85a4ee9ed88c..0935d82f86d945 100644 --- a/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CancelAlterStmtTest.java @@ -20,21 +20,21 @@ package com.baidu.palo.analysis; -import com.baidu.palo.analysis.ShowAlterStmt.AlterType; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.common.AnalysisException; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.analysis.ShowAlterStmt.AlterType; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @PrepareForTest(Catalog.class) diff --git a/fe/test/com/baidu/palo/analysis/ColumnTest.java b/fe/test/com/baidu/palo/analysis/ColumnTest.java index caaff41d505b22..b2b6ceb0f3a332 100644 --- a/fe/test/com/baidu/palo/analysis/ColumnTest.java +++ b/fe/test/com/baidu/palo/analysis/ColumnTest.java @@ -20,16 +20,16 @@ package com.baidu.palo.analysis; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + import com.baidu.palo.catalog.AggregateType; import com.baidu.palo.catalog.Column; import com.baidu.palo.catalog.ColumnType; import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.common.AnalysisException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - public class ColumnTest { private ColumnType intCol; private ColumnType stringCol; @@ -45,7 +45,7 @@ public void setUp() { @Test public void testNormal() throws AnalysisException { Column column = new Column("col", intCol); - column.analyze(); + column.analyze(true); Assert.assertEquals("`col` int(11) NOT NULL COMMENT \"\"", column.toString()); Assert.assertEquals("col", column.getName()); @@ -55,14 +55,14 @@ public void testNormal() throws AnalysisException { // default column = new Column("col", intCol, true, null, "10", ""); - column.analyze(); + column.analyze(true); Assert.assertNull(column.getAggregationType()); Assert.assertEquals("10", column.getDefaultValue()); Assert.assertEquals("`col` int(11) NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); // agg column = new Column("col", floatCol, false, AggregateType.SUM, "10", ""); - column.analyze(); + column.analyze(true); Assert.assertEquals("10", column.getDefaultValue()); Assert.assertEquals(AggregateType.SUM, column.getAggregationType()); Assert.assertEquals("`col` float SUM NOT NULL DEFAULT \"10\" COMMENT \"\"", column.toSql()); @@ -72,13 +72,13 @@ public void testNormal() throws AnalysisException { public void testFloatKey() throws AnalysisException { Column column = new Column("col", floatCol); column.setIsKey(true); - column.analyze(); + column.analyze(true); } @Test(expected = AnalysisException.class) public void testStrSum() throws AnalysisException { Column column = new Column("col", stringCol, false, AggregateType.SUM, null, ""); - column.analyze(); + column.analyze(true); } } diff --git a/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java b/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java index db0d0617a07d5a..0bd473c746ee85 100644 --- a/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/CreateTableStmtTest.java @@ -1,3 +1,4 @@ + // Modifications copyright (C) 2017, Baidu.com, Inc. // Copyright 2017 The Apache Software Foundation @@ -20,27 +21,26 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.catalog.ColumnType; -import com.baidu.palo.catalog.KeysType; -import com.baidu.palo.catalog.PrimitiveType; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; - -import com.google.common.collect.Lists; +import java.util.List; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.catalog.ColumnType; +import com.baidu.palo.catalog.KeysType; +import com.baidu.palo.catalog.PrimitiveType; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.google.common.collect.Lists; public class CreateTableStmtTest { private static final Logger LOG = LoggerFactory.getLogger(CreateTableStmtTest.class); - + // used to get default db private TableName tblName; private TableName tblNameNoDb; @@ -49,7 +49,7 @@ public class CreateTableStmtTest { private List colsName; private List invalidColsName; private Analyzer analyzer; - + // set default db is 'db1' // table name is table1 // Column: [col1 int; col2 string] @@ -77,12 +77,12 @@ public void setUp() { invalidColsName.add("col2"); invalidColsName.add("col2"); } - + @Test public void testNormal() throws InternalException, AnalysisException { - CreateTableStmt stmt = new CreateTableStmt(false, tblName, cols, "olap", - new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null); + CreateTableStmt stmt = new CreateTableStmt(false, false, tblName, cols, "olap", + new KeysDesc(KeysType.AGG_KEYS, colsName), null, + new RandomDistributionDesc(10), null, null); stmt.analyze(analyzer); Assert.assertEquals("testCluster:db1", stmt.getDbName()); Assert.assertEquals("table1", stmt.getTableName()); @@ -93,12 +93,12 @@ public void testNormal() throws InternalException, AnalysisException { + ") ENGINE = olap\nAGG_KEYS(`col1`, `col2`)\nDISTRIBUTED BY RANDOM\nBUCKETS 10", stmt.toSql()); } - + @Test public void testDefaultDbNormal() throws InternalException, AnalysisException { - CreateTableStmt stmt = new CreateTableStmt(false, tblNameNoDb, cols, "olap", - new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null); + CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", + new KeysDesc(KeysType.AGG_KEYS, colsName), null, + new RandomDistributionDesc(10), null, null); stmt.analyze(analyzer); Assert.assertEquals("testCluster:testDb", stmt.getDbName()); Assert.assertEquals("table1", stmt.getTableName()); @@ -109,46 +109,46 @@ public void testDefaultDbNormal() throws InternalException, AnalysisException { + "`col1` int(11) NOT NULL COMMENT \"\",\n" + "`col2` char(10) NOT NULL COMMENT \"\"\n" + ") ENGINE = olap\nAGG_KEYS(`col1`, `col2`)\nDISTRIBUTED BY RANDOM\nBUCKETS 10", stmt.toSql()); } - + @Test(expected = AnalysisException.class) public void testNoDb() throws InternalException, AnalysisException { // make defalut db return empty; analyzer = EasyMock.createMock(Analyzer.class); EasyMock.expect(analyzer.getDefaultDb()).andReturn("").anyTimes(); EasyMock.replay(analyzer); - CreateTableStmt stmt = new CreateTableStmt(false, tblNameNoDb, cols, "olap", - new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null); + CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", + new KeysDesc(KeysType.AGG_KEYS, colsName), null, + new RandomDistributionDesc(10), null, null); stmt.analyze(analyzer); } - + @Test(expected = AnalysisException.class) public void testEmptyCol() throws InternalException, AnalysisException { // make defalut db return empty; List emptyCols = Lists.newArrayList(); - CreateTableStmt stmt = new CreateTableStmt(false, tblNameNoDb, emptyCols, "olap", - new KeysDesc(), null, - new RandomDistributionDesc(10), null); + CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, emptyCols, "olap", + new KeysDesc(), null, + new RandomDistributionDesc(10), null, null); stmt.analyze(analyzer); } - + @Test(expected = AnalysisException.class) public void testDupCol() throws InternalException, AnalysisException { // make defalut db return empty; - CreateTableStmt stmt = new CreateTableStmt(false, tblNameNoDb, invalidCols, "olap", - new KeysDesc(KeysType.AGG_KEYS, invalidColsName), null, - new RandomDistributionDesc(10), null); + CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, invalidCols, "olap", + new KeysDesc(KeysType.AGG_KEYS, invalidColsName), null, + new RandomDistributionDesc(10), null, null); stmt.analyze(analyzer); } - + @Test(expected = AnalysisException.class) public void testNoPriv() throws InternalException, AnalysisException { // make default db return empty; - CreateTableStmt stmt = new CreateTableStmt(false, tblNameNoDb, cols, "olap", - new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null); + CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap", + new KeysDesc(KeysType.AGG_KEYS, colsName), null, + new RandomDistributionDesc(10), null, null); stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); Assert.fail("No exception throws."); } - -} + +} \ No newline at end of file diff --git a/fe/test/com/baidu/palo/analysis/LoadStmtTest.java b/fe/test/com/baidu/palo/analysis/LoadStmtTest.java index 5868c5d3b2f6d7..75e3fcedfba8e2 100644 --- a/fe/test/com/baidu/palo/analysis/LoadStmtTest.java +++ b/fe/test/com/baidu/palo/analysis/LoadStmtTest.java @@ -20,16 +20,16 @@ package com.baidu.palo.analysis; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.InternalException; +import java.util.List; -import com.google.common.collect.Lists; -import org.junit.Assert; import org.easymock.EasyMock; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.util.List; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.InternalException; +import com.google.common.collect.Lists; public class LoadStmtTest { private DataDescription desc; @@ -51,7 +51,7 @@ public void testNormal() throws InternalException, AnalysisException { EasyMock.expectLastCall().anyTimes(); EasyMock.replay(desc); - LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), dataDescriptions, null, null); + LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), dataDescriptions, null, null, null); stmt.analyze(analyzer); Assert.assertEquals("testCluster:testDb", stmt.getLabel().getDbName()); Assert.assertEquals(dataDescriptions, stmt.getDataDescriptions()); @@ -67,7 +67,7 @@ public void testNoPriv() throws InternalException, AnalysisException { EasyMock.expectLastCall().anyTimes(); EasyMock.replay(desc); - LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), dataDescriptions, null, null); + LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), dataDescriptions, null, null, null); stmt.analyze(AccessTestUtil.fetchBlockAnalyzer()); Assert.fail("No exception throws."); @@ -79,7 +79,7 @@ public void testNoData() throws InternalException, AnalysisException { EasyMock.expectLastCall().anyTimes(); EasyMock.replay(desc); - LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), null, null, null); + LoadStmt stmt = new LoadStmt(new LabelName("testDb", "testLabel"), null, null, null, null); stmt.analyze(analyzer); Assert.fail("No exception throws."); diff --git a/fe/test/com/baidu/palo/analysis/ModifyColumnClauseTest.java b/fe/test/com/baidu/palo/analysis/ModifyColumnClauseTest.java index d0934b8e98371f..56f32d0d71f32f 100644 --- a/fe/test/com/baidu/palo/analysis/ModifyColumnClauseTest.java +++ b/fe/test/com/baidu/palo/analysis/ModifyColumnClauseTest.java @@ -20,14 +20,14 @@ package com.baidu.palo.analysis; -import com.baidu.palo.catalog.Column; -import com.baidu.palo.common.AnalysisException; - +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.BeforeClass; -import org.easymock.EasyMock; import org.junit.Test; +import com.baidu.palo.catalog.Column; +import com.baidu.palo.common.AnalysisException; + public class ModifyColumnClauseTest { private static Analyzer analyzer; @@ -39,7 +39,7 @@ public static void setUp() { @Test public void testNormal() throws AnalysisException { Column definition = EasyMock.createMock(Column.class); - definition.analyze(); + definition.analyze(true); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(definition.toSql()).andReturn("`testCol` INT").anyTimes(); EasyMock.replay(definition); diff --git a/fe/test/com/baidu/palo/catalog/BackendTest.java b/fe/test/com/baidu/palo/catalog/BackendTest.java index 9fe62f37e257af..e1f0471a06ebbc 100644 --- a/fe/test/com/baidu/palo/catalog/BackendTest.java +++ b/fe/test/com/baidu/palo/catalog/BackendTest.java @@ -20,31 +20,31 @@ package com.baidu.palo.catalog; -import com.baidu.palo.analysis.AccessTestUtil; -import com.baidu.palo.system.Backend; -import com.baidu.palo.common.FeConstants; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.analysis.AccessTestUtil; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.system.Backend; import com.baidu.palo.thrift.TDisk; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @PrepareForTest(Catalog.class) @@ -54,7 +54,8 @@ public class BackendTest { private String host = "myhost"; private int heartbeatPort = 21234; private int bePort = 21235; - private int httpPort = 21237; + private int httpPort = 21237; + private int beRpcPort = 21238; private Catalog catalog; @@ -67,7 +68,7 @@ public void setUp() { PowerMock.replay(Catalog.class); backend = new Backend(backendId, host, heartbeatPort); - backend.updateOnce(bePort, httpPort); + backend.updateOnce(bePort, httpPort, beRpcPort); } @Test @@ -80,7 +81,7 @@ public void getMethodTest() { // set new port int newBePort = 31235; int newHttpPort = 31237; - backend.updateOnce(newBePort, newHttpPort); + backend.updateOnce(newBePort, newHttpPort, beRpcPort); Assert.assertEquals(newBePort, backend.getBePort()); // check alive @@ -125,12 +126,12 @@ public void testSerialization() throws Exception { for (int count = 0; count < 100; ++count) { Backend backend = new Backend(count, "10.120.22.32" + count, 6000 + count); - backend.updateOnce(7000 + count, 9000 + count); + backend.updateOnce(7000 + count, 9000 + count, beRpcPort); list1.add(backend); } for (int count = 100; count < 200; count++) { Backend backend = new Backend(count, "10.120.22.32" + count, 6000 + count); - backend.updateOnce(7000 + count, 9000 + count); + backend.updateOnce(7000 + count, 9000 + count, beRpcPort); list1.add(backend); } for (Backend backend : list1) { @@ -164,21 +165,21 @@ public void testSerialization() throws Exception { Assert.assertTrue(list1.get(1).equals(list1.get(1))); Backend back1 = new Backend(1, "a", 1); - back1.updateOnce(1, 1); + back1.updateOnce(1, 1, 1); Backend back2 = new Backend(2, "a", 1); - back2.updateOnce(1, 1); + back2.updateOnce(1, 1, 1); Assert.assertFalse(back1.equals(back2)); back1 = new Backend(1, "a", 1); - back1.updateOnce(1, 1); + back1.updateOnce(1, 1, 1); back2 = new Backend(1, "b", 1); - back2.updateOnce(1, 1); + back2.updateOnce(1, 1, 1); Assert.assertFalse(back1.equals(back2)); back1 = new Backend(1, "a", 1); - back1.updateOnce(1, 1); + back1.updateOnce(1, 1, 1); back2 = new Backend(1, "a", 2); - back2.updateOnce(1, 1); + back2.updateOnce(1, 1, 1); Assert.assertFalse(back1.equals(back2)); Assert.assertEquals("Backend [id=1, host=a, heartbeatPort=1, alive=true]", back1.toString()); diff --git a/fe/test/com/baidu/palo/catalog/CreateTableTest.java b/fe/test/com/baidu/palo/catalog/CreateTableTest.java index 855cc0602d444d..a5c81942a4956c 100644 --- a/fe/test/com/baidu/palo/catalog/CreateTableTest.java +++ b/fe/test/com/baidu/palo/catalog/CreateTableTest.java @@ -20,17 +20,11 @@ package com.baidu.palo.catalog; -import com.baidu.palo.analysis.KeysDesc; -import com.baidu.palo.analysis.Analyzer; -import com.baidu.palo.analysis.CreateTableStmt; -import com.baidu.palo.analysis.RandomDistributionDesc; -import com.baidu.palo.analysis.TableName; -import com.baidu.palo.catalog.KeysType; -import com.baidu.palo.system.Backend; -import com.baidu.palo.system.SystemInfoService; -import com.baidu.palo.common.DdlException; - -import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import org.easymock.EasyMock; import org.easymock.IAnswer; @@ -43,11 +37,15 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; +import com.baidu.palo.analysis.Analyzer; +import com.baidu.palo.analysis.CreateTableStmt; +import com.baidu.palo.analysis.KeysDesc; +import com.baidu.palo.analysis.RandomDistributionDesc; +import com.baidu.palo.analysis.TableName; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.system.Backend; +import com.baidu.palo.system.SystemInfoService; +import com.google.common.collect.Lists; @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @@ -108,9 +106,9 @@ public void createTableTest_invalidColumn() throws Exception { "")); invalidCols1.add(new Column("k1", ColumnType.createVarchar(10))); invalidCols1.get(1).setIsKey(true); - CreateTableStmt stmt1 = new CreateTableStmt(false, dbTableName, invalidCols1, "olap", + CreateTableStmt stmt1 = new CreateTableStmt(false, false, dbTableName, invalidCols1, "olap", new KeysDesc(), null, - new RandomDistributionDesc(10), null); + new RandomDistributionDesc(10), null, null); try { catalog.createTable(stmt1); } catch (DdlException e) { @@ -122,9 +120,9 @@ public void createTableTest_invalidColumn() throws Exception { invalidCols2.add(new Column("v1", ColumnType.createType(PrimitiveType.INT), false, AggregateType.SUM, "1", "")); invalidCols2.add(new Column("v2", ColumnType.createVarchar(10), false, AggregateType.REPLACE, "abc", "")); - CreateTableStmt stmt2 = new CreateTableStmt(false, dbTableName, invalidCols2, "olap", + CreateTableStmt stmt2 = new CreateTableStmt(false, false, dbTableName, invalidCols2, "olap", new KeysDesc(), null, - new RandomDistributionDesc(10), null); + new RandomDistributionDesc(10), null, null); try { catalog.createTable(stmt2); @@ -141,9 +139,9 @@ public void olapDbNotFoundTest() throws Exception { EasyMock.expectLastCall().andReturn(null); EasyMock.replay(catalog); - CreateTableStmt stmt = new CreateTableStmt(false, dbTableName, cols, "olap", + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null); + new RandomDistributionDesc(10), null, null); try { catalog.createTable(stmt); } catch (DdlException e) { @@ -164,9 +162,9 @@ public void olapTableExistsTest() throws Exception { EasyMock.expectLastCall().andReturn(db); EasyMock.replay(catalog); - CreateTableStmt stmt = new CreateTableStmt(false, dbTableName, cols, "olap", + CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(10), null); + new RandomDistributionDesc(10), null, null); try { catalog.createTable(stmt); @@ -217,9 +215,9 @@ public List answer() throws Throwable { .andReturn((short) 2).anyTimes(); PowerMock.replay(Catalog.class); - CreateTableStmt stmt1 = new CreateTableStmt(false, dbTableName, cols, "olap", + CreateTableStmt stmt1 = new CreateTableStmt(false, false, dbTableName, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(1), null); + new RandomDistributionDesc(1), null, null); try { catalog.createTable(stmt1); } catch (DdlException e) { @@ -286,9 +284,9 @@ public List answer() throws Throwable { // 1. larger then indexColumns size properties.put("short_key_num", "3"); - CreateTableStmt stmt1 = new CreateTableStmt(false, dbTableName, cols2, "olap", + CreateTableStmt stmt1 = new CreateTableStmt(false, false, dbTableName, cols2, "olap", new KeysDesc(KeysType.AGG_KEYS, cols2Name), null, - new RandomDistributionDesc(1), null); + new RandomDistributionDesc(1), null, null); try { catalog.createTable(stmt1); } catch (DdlException e) { @@ -302,9 +300,9 @@ public List answer() throws Throwable { cols2.add(new Column("k2_varchar", ColumnType.createType(PrimitiveType.VARCHAR))); cols2.add(new Column("v1", ColumnType.createType(PrimitiveType.INT), true, AggregateType.MAX, "0", "")); - stmt1 = new CreateTableStmt(false, dbTableName, cols2, "olap", + stmt1 = new CreateTableStmt(false, false, dbTableName, cols2, "olap", new KeysDesc(KeysType.AGG_KEYS, cols2Name), null, - new RandomDistributionDesc(1), null); + new RandomDistributionDesc(1), null, null); try { catalog.createTable(stmt1); @@ -354,9 +352,9 @@ public List answer() throws Throwable { .andReturn((short) 2).anyTimes(); PowerMock.replay(Catalog.class); - CreateTableStmt stmt1 = new CreateTableStmt(false, dbTableName, cols, "olap", + CreateTableStmt stmt1 = new CreateTableStmt(false, false, dbTableName, cols, "olap", new KeysDesc(KeysType.AGG_KEYS, colsName), null, - new RandomDistributionDesc(1), null); + new RandomDistributionDesc(1), null, null); try { catalog.createTable(stmt1); diff --git a/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java b/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java index 912ef3aa66e3d0..de5cb59f6c63fa 100644 --- a/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java +++ b/fe/test/com/baidu/palo/cluster/SystemInfoServiceTest.java @@ -19,39 +19,38 @@ package com.baidu.palo.cluster; -import com.baidu.palo.analysis.AccessTestUtil; -import com.baidu.palo.analysis.AddBackendClause; -import com.baidu.palo.analysis.Analyzer; -import com.baidu.palo.analysis.DropBackendClause; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.catalog.Database; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.DdlException; -import com.baidu.palo.common.FeConstants; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.system.SystemInfoService; -import com.baidu.palo.system.Backend; - +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.analysis.AccessTestUtil; +import com.baidu.palo.analysis.AddBackendClause; +import com.baidu.palo.analysis.Analyzer; +import com.baidu.palo.analysis.DropBackendClause; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.catalog.Database; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.DdlException; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.Backend; +import com.baidu.palo.system.SystemInfoService; import com.google.common.collect.Lists; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; - @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @PrepareForTest(Catalog.class) @@ -249,7 +248,7 @@ public void testSaveLoadBackend() throws Exception { DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); SystemInfoService systemInfoService = Catalog.getCurrentSystemInfo(); Backend back1 = new Backend(1L, "localhost", 3); - back1.updateOnce(4, 6); + back1.updateOnce(4, 6, 8); systemInfoService.replayAddBackend(back1); long checksum1 = systemInfoService.saveBackends(dos, 0); catalog.clear(); diff --git a/fe/test/com/baidu/palo/common/GenericPoolTest.java b/fe/test/com/baidu/palo/common/GenericPoolTest.java index 34014b91b9e48c..4b595ee42fcf61 100644 --- a/fe/test/com/baidu/palo/common/GenericPoolTest.java +++ b/fe/test/com/baidu/palo/common/GenericPoolTest.java @@ -20,41 +20,47 @@ package com.baidu.palo.common; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + import com.baidu.palo.thrift.BackendService; import com.baidu.palo.thrift.PaloInternalServiceVersion; import com.baidu.palo.thrift.TAgentPublishRequest; import com.baidu.palo.thrift.TAgentResult; import com.baidu.palo.thrift.TAgentTaskRequest; -import com.baidu.palo.thrift.TMiniLoadEtlStatusRequest; -import com.baidu.palo.thrift.TMiniLoadEtlStatusResult; -import com.baidu.palo.thrift.TMiniLoadEtlTaskRequest; import com.baidu.palo.thrift.TCancelPlanFragmentParams; import com.baidu.palo.thrift.TCancelPlanFragmentResult; import com.baidu.palo.thrift.TDeleteEtlFilesRequest; import com.baidu.palo.thrift.TExecPlanFragmentParams; import com.baidu.palo.thrift.TExecPlanFragmentResult; +import com.baidu.palo.thrift.TExportStatusResult; +import com.baidu.palo.thrift.TExportTaskRequest; +import com.baidu.palo.thrift.TFetchAllPullLoadTaskInfosResult; import com.baidu.palo.thrift.TFetchDataParams; import com.baidu.palo.thrift.TFetchDataResult; +import com.baidu.palo.thrift.TFetchPullLoadTaskInfoResult; +import com.baidu.palo.thrift.TMiniLoadEtlStatusRequest; +import com.baidu.palo.thrift.TMiniLoadEtlStatusResult; +import com.baidu.palo.thrift.TMiniLoadEtlTaskRequest; import com.baidu.palo.thrift.TNetworkAddress; +import com.baidu.palo.thrift.TPullLoadSubTaskInfo; import com.baidu.palo.thrift.TResultBatch; import com.baidu.palo.thrift.TSnapshotRequest; +import com.baidu.palo.thrift.TStatus; import com.baidu.palo.thrift.TTransmitDataParams; import com.baidu.palo.thrift.TTransmitDataResult; import com.baidu.palo.thrift.TUniqueId; -import org.apache.thrift.TException; -import org.junit.Assert; -import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig; -import org.apache.thrift.TProcessor; -import org.junit.Test; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - public class GenericPoolTest { static GenericPool backendService; static ThriftServer service; @@ -156,6 +162,54 @@ public TAgentResult make_snapshot(TSnapshotRequest snapshot_request) throws TExc // TODO Auto-generated method stub return null; } + + @Override + public TStatus register_pull_load_task(TUniqueId id, int num_senders) throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TStatus deregister_pull_load_task(TUniqueId id) throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TStatus report_pull_load_sub_task_info(TPullLoadSubTaskInfo task_info) throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TFetchPullLoadTaskInfoResult fetch_pull_load_task_info(TUniqueId id) throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TFetchAllPullLoadTaskInfosResult fetch_all_pull_load_task_infos() throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TStatus submit_export_task(TExportTaskRequest request) throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TExportStatusResult get_export_status(TUniqueId task_id) throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TStatus erase_export_task(TUniqueId task_id) throws TException { + // TODO Auto-generated method stub + return null; + } } @Test diff --git a/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java b/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java index 6ac12b48a4ebca..87b1032bfeccde 100644 --- a/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java +++ b/fe/test/com/baidu/palo/common/proc/BackendProcNodeTest.java @@ -20,24 +20,23 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.system.Backend; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.persist.EditLog; - +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.Backend; import com.google.common.collect.Lists; -import org.easymock.EasyMock; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.Assert; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @PrepareForTest(Catalog.class) @@ -69,7 +68,7 @@ public void setUp() { PowerMock.replay(Catalog.class); b1 = new Backend(1000, "host1", 10000); - b1.updateOnce(10001, 10003); + b1.updateOnce(10001, 10003, 10005); } @After diff --git a/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java b/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java index 80f9787a245753..22d03fd129935c 100644 --- a/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java +++ b/fe/test/com/baidu/palo/common/proc/BackendsProcDirTest.java @@ -20,26 +20,25 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.system.Backend; -import com.baidu.palo.system.SystemInfoService; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.persist.EditLog; - +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.Backend; +import com.baidu.palo.system.SystemInfoService; import com.google.common.collect.Lists; -import org.easymock.EasyMock; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.Assert; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - @RunWith(PowerMockRunner.class) @PowerMockIgnore("org.apache.log4j.*") @PrepareForTest(Catalog.class) @@ -75,9 +74,9 @@ public static void setUpClass() { PowerMock.replay(Catalog.class); b1 = new Backend(1000, "host1", 10000); - b1.updateOnce(10001, 10003); + b1.updateOnce(10001, 10003, 10005); b2 = new Backend(1001, "host2", 20000); - b2.updateOnce(20001, 20003); + b2.updateOnce(20001, 20003, 20005); } @Before diff --git a/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java b/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java index 5adcc5c3910295..6b21f41e6ee71b 100644 --- a/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java +++ b/fe/test/com/baidu/palo/common/util/RuntimeProfileTest.java @@ -20,23 +20,23 @@ package com.baidu.palo.common.util; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Set; - -import org.junit.Assert; -import org.junit.Test; - -import com.baidu.palo.thrift.TCounter; -import com.baidu.palo.thrift.TCounterType; -import com.baidu.palo.thrift.TRuntimeProfileNode; -import com.baidu.palo.thrift.TRuntimeProfileTree; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Set; + +import org.junit.Assert; +import org.junit.Test; + +import com.baidu.palo.thrift.TCounter; +import com.baidu.palo.thrift.TRuntimeProfileNode; +import com.baidu.palo.thrift.TRuntimeProfileTree; +import com.baidu.palo.thrift.TUnit; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; public class RuntimeProfileTest { @@ -106,7 +106,7 @@ public void testInfoStrings() { @Test public void testCounter() { RuntimeProfile profile = new RuntimeProfile(); - profile.addCounter("key", TCounterType.UNIT, ""); + profile.addCounter("key", TUnit.UNIT, ""); Assert.assertNotNull(profile.getCounterMap().get("key")); Assert.assertNull(profile.getCounterMap().get("key2")); profile.getCounterMap().get("key").setValue(1); @@ -143,13 +143,13 @@ public void testUpdate() throws IOException { tnodeB.counters = Lists.newArrayList(); tnodeASon.counters = Lists.newArrayList(); - tnodeRoot.counters.add(new TCounter("TotalTime", TCounterType.TIME_NS, 3000000000L)); - tnodeA.counters.add(new TCounter("TotalTime", TCounterType.TIME_NS, 1000000000L)); - tnodeB.counters.add(new TCounter("TotalTime", TCounterType.TIME_NS, 1000000000L)); - tnodeASon.counters.add(new TCounter("TotalTime", TCounterType.TIME_NS, 10000000)); - tnodeASon.counters.add(new TCounter("counterA1", TCounterType.UNIT, 1)); - tnodeASon.counters.add(new TCounter("counterA2", TCounterType.BYTES, 1234567L)); - tnodeASon.counters.add(new TCounter("counterA1Son", TCounterType.UNIT, 3)); + tnodeRoot.counters.add(new TCounter("TotalTime", TUnit.TIME_NS, 3000000000L)); + tnodeA.counters.add(new TCounter("TotalTime", TUnit.TIME_NS, 1000000000L)); + tnodeB.counters.add(new TCounter("TotalTime", TUnit.TIME_NS, 1000000000L)); + tnodeASon.counters.add(new TCounter("TotalTime", TUnit.TIME_NS, 10000000)); + tnodeASon.counters.add(new TCounter("counterA1", TUnit.UNIT, 1)); + tnodeASon.counters.add(new TCounter("counterA2", TUnit.BYTES, 1234567L)); + tnodeASon.counters.add(new TCounter("counterA1Son", TUnit.UNIT, 3)); tnodeASon.child_counters_map = Maps.newHashMap(); Set set1 = Sets.newHashSet(); diff --git a/fe/test/com/baidu/palo/common/util/UnitTestUtil.java b/fe/test/com/baidu/palo/common/util/UnitTestUtil.java index 5f5c5b363d8c77..83c248e775cc46 100644 --- a/fe/test/com/baidu/palo/common/util/UnitTestUtil.java +++ b/fe/test/com/baidu/palo/common/util/UnitTestUtil.java @@ -20,6 +20,14 @@ package com.baidu.palo.common.util; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; + import com.baidu.palo.catalog.AggregateType; import com.baidu.palo.catalog.Catalog; import com.baidu.palo.catalog.Column; @@ -28,33 +36,25 @@ import com.baidu.palo.catalog.Database; import com.baidu.palo.catalog.KeysType; import com.baidu.palo.catalog.MaterializedIndex; +import com.baidu.palo.catalog.MaterializedIndex.IndexState; import com.baidu.palo.catalog.OlapTable; import com.baidu.palo.catalog.Partition; import com.baidu.palo.catalog.PartitionInfo; import com.baidu.palo.catalog.PrimitiveType; import com.baidu.palo.catalog.RandomDistributionInfo; import com.baidu.palo.catalog.Replica; -import com.baidu.palo.catalog.SinglePartitionInfo; -import com.baidu.palo.catalog.TabletMeta; -import com.baidu.palo.catalog.MaterializedIndex.IndexState; import com.baidu.palo.catalog.Replica.ReplicaState; +import com.baidu.palo.catalog.SinglePartitionInfo; import com.baidu.palo.catalog.Tablet; -import com.baidu.palo.system.Backend; +import com.baidu.palo.catalog.TabletMeta; import com.baidu.palo.common.Config; import com.baidu.palo.common.LoadException; import com.baidu.palo.load.DppConfig; import com.baidu.palo.load.Load; +import com.baidu.palo.system.Backend; import com.baidu.palo.thrift.TDisk; import com.google.common.collect.Maps; -import org.junit.Assert; - -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - // for unit test public class UnitTestUtil { public static final String DB_NAME = "testDb"; @@ -123,7 +123,7 @@ public static Database createDb(long dbId, long tableId, long partitionId, long public static Backend createBackend(long id, String host, int heartPort, int bePort, int httpPort) { Backend backend = new Backend(id, host, heartPort); - backend.updateOnce(bePort, httpPort); + backend.updateOnce(bePort, httpPort, 10000); return backend; } diff --git a/fe/test/com/baidu/palo/load/LoadTest.java b/fe/test/com/baidu/palo/load/LoadTest.java index 9de12c899f65bf..54549fb6375da3 100644 --- a/fe/test/com/baidu/palo/load/LoadTest.java +++ b/fe/test/com/baidu/palo/load/LoadTest.java @@ -19,6 +19,22 @@ package com.baidu.palo.load; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + import com.baidu.palo.analysis.CancelLoadStmt; import com.baidu.palo.analysis.ColumnSeparator; import com.baidu.palo.analysis.DataDescription; @@ -34,8 +50,6 @@ import com.baidu.palo.catalog.Replica; import com.baidu.palo.catalog.Tablet; import com.baidu.palo.catalog.UserPropertyMgr; -import com.baidu.palo.system.Backend; -import com.baidu.palo.system.SystemInfoService; import com.baidu.palo.common.Config; import com.baidu.palo.common.DdlException; import com.baidu.palo.common.MarkedCountDownLatch; @@ -46,26 +60,10 @@ import com.baidu.palo.load.LoadJob.JobState; import com.baidu.palo.persist.EditLog; import com.baidu.palo.qe.ConnectContext; - +import com.baidu.palo.system.Backend; +import com.baidu.palo.system.SystemInfoService; import com.google.common.collect.Lists; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - @RunWith(PowerMockRunner.class) @PrepareForTest({ Load.class, Catalog.class, ConnectContext.class, SystemInfoService.class }) public class LoadTest { @@ -154,7 +152,7 @@ private void addLoadJob(String label) throws DdlException { Lists.newArrayList(UnitTestUtil.PARTITION_NAME), filePathes, null, new ColumnSeparator(columnSeparator), false, null); dataDescriptions.add(dataDescription); - LoadStmt stmt = new LoadStmt(labelName, dataDescriptions, null, null); + LoadStmt stmt = new LoadStmt(labelName, dataDescriptions, null, null, null); load.addLoadJob(stmt, EtlJobType.HADOOP, 0); } diff --git a/fe/test/com/baidu/palo/qe/CoordinatorTest.java b/fe/test/com/baidu/palo/qe/CoordinatorTest.java index 96ca914ec21449..8cbc5a4875f168 100644 --- a/fe/test/com/baidu/palo/qe/CoordinatorTest.java +++ b/fe/test/com/baidu/palo/qe/CoordinatorTest.java @@ -20,51 +20,50 @@ package com.baidu.palo.qe; -import org.junit.Assert; -import org.apache.thrift.TException; -import org.easymock.EasyMock; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.analysis.Analyzer; -import com.baidu.palo.analysis.TupleDescriptor; -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.system.Backend; -import com.baidu.palo.common.Config; -import com.baidu.palo.common.FeConstants; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.planner.DataPartition; -import com.baidu.palo.planner.ExchangeNode; -import com.baidu.palo.planner.OlapScanNode; -import com.baidu.palo.planner.PlanFragment; -import com.baidu.palo.planner.PlanFragmentId; -import com.baidu.palo.planner.PlanNode; -import com.baidu.palo.planner.PlanNodeId; -import com.baidu.palo.planner.Planner; -import com.baidu.palo.thrift.TExecPlanFragmentParams; -import com.baidu.palo.thrift.TNetworkAddress; -import com.baidu.palo.thrift.TQueryOptions; -import com.baidu.palo.thrift.TScanRange; -import com.baidu.palo.thrift.TScanRangeLocation; -import com.baidu.palo.thrift.TScanRangeLocations; -import com.baidu.palo.thrift.TUniqueId; - +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import org.apache.thrift.TException; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.analysis.Analyzer; +import com.baidu.palo.analysis.TupleDescriptor; +import com.baidu.palo.analysis.TupleId; +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.planner.DataPartition; +import com.baidu.palo.planner.ExchangeNode; +import com.baidu.palo.planner.OlapScanNode; +import com.baidu.palo.planner.PlanFragment; +import com.baidu.palo.planner.PlanFragmentId; +import com.baidu.palo.planner.PlanNode; +import com.baidu.palo.planner.PlanNodeId; +import com.baidu.palo.planner.Planner; +import com.baidu.palo.system.Backend; +import com.baidu.palo.thrift.TExecPlanFragmentParams; +import com.baidu.palo.thrift.TNetworkAddress; +import com.baidu.palo.thrift.TQueryOptions; +import com.baidu.palo.thrift.TScanRange; +import com.baidu.palo.thrift.TScanRangeLocation; +import com.baidu.palo.thrift.TScanRangeLocations; +import com.baidu.palo.thrift.TUniqueId; import com.google.common.collect.ImmutableMap; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; - @RunWith(PowerMockRunner.class) @PowerMockIgnore({"org.apache.log4j.*", "javax.management.*"}) @PrepareForTest(Catalog.class) @@ -110,13 +109,13 @@ public static void beforeTest() throws IllegalAccessException, IllegalArgumentEx FeConstants.heartbeat_interval_second = Integer.MAX_VALUE; backendA = new Backend(0, "machineA", 0); - backendA.updateOnce(10000, 0); + backendA.updateOnce(10000, 0, 0); backendB = new Backend(1, "machineB", 0); - backendB.updateOnce(10000, 0); + backendB.updateOnce(10000, 0, 0); backendC = new Backend(2, "machineC", 0); - backendC.updateOnce(10000, 0); + backendC.updateOnce(10000, 0, 0); backendD = new Backend(3, "machineD", 0); - backendD.updateOnce(10000, 0); + backendD.updateOnce(10000, 0, 0); // private 方法赋值 Field field = coor.getClass().getDeclaredField("idToBackend"); @@ -288,7 +287,7 @@ public void testcomputeFragmentHosts() throws NoSuchMethodException, SecurityExc // 场景1：UNPARTITIONED { PlanFragment fragment = new PlanFragment(new PlanFragmentId(1), - new OlapScanNode(new PlanNodeId(1), new TupleDescriptor(10), "null scanNode"), + new OlapScanNode(new PlanNodeId(1), new TupleDescriptor(new TupleId(10)), "null scanNode"), DataPartition.UNPARTITIONED ); List privateFragments = (ArrayList) getField(coor, "fragments"); @@ -310,7 +309,7 @@ public void testcomputeFragmentHosts() throws NoSuchMethodException, SecurityExc // 场景2： ScanNode { PlanFragment fragment = new PlanFragment(new PlanFragmentId(1), - new OlapScanNode(new PlanNodeId(1), new TupleDescriptor(10), "null scanNode"), + new OlapScanNode(new PlanNodeId(1), new TupleDescriptor(new TupleId(10)), "null scanNode"), DataPartition.RANDOM ); List privateFragments = (ArrayList) getField(coor, "fragments"); @@ -362,7 +361,7 @@ public void testcomputeFragmentHosts() throws NoSuchMethodException, SecurityExc ImmutableMap idToBackendAB = ImmutableMap.copyOf(backendMap); field.set(coor, idToBackendAB); - PlanNode olapNode = new OlapScanNode(new PlanNodeId(1), new TupleDescriptor(10), + PlanNode olapNode = new OlapScanNode(new PlanNodeId(1), new TupleDescriptor(new TupleId(10)), "null scanNode"); PlanFragment fragmentFather = new PlanFragment(new PlanFragmentId(0), new ExchangeNode(new PlanNodeId(10), olapNode, false), diff --git a/fe/test/com/baidu/palo/qe/SimpleSchedulerTest.java b/fe/test/com/baidu/palo/qe/SimpleSchedulerTest.java index 3dd8bd45e66a56..72544c950cdbc1 100644 --- a/fe/test/com/baidu/palo/qe/SimpleSchedulerTest.java +++ b/fe/test/com/baidu/palo/qe/SimpleSchedulerTest.java @@ -20,30 +20,28 @@ package com.baidu.palo.qe; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import com.baidu.palo.catalog.Catalog; -import com.baidu.palo.system.Backend; -import com.baidu.palo.common.Config; -import com.baidu.palo.common.FeConstants; -import com.baidu.palo.common.Reference; -import com.baidu.palo.persist.EditLog; -import com.baidu.palo.thrift.TNetworkAddress; -import com.baidu.palo.thrift.TScanRangeLocation; - -import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.common.Config; +import com.baidu.palo.common.FeConstants; +import com.baidu.palo.common.Reference; +import com.baidu.palo.persist.EditLog; +import com.baidu.palo.system.Backend; +import com.baidu.palo.thrift.TNetworkAddress; +import com.baidu.palo.thrift.TScanRangeLocation; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @RunWith(PowerMockRunner.class) @@ -98,11 +96,11 @@ public void testGetHostWithBackendId() { ImmutableMap emptyBackends = ImmutableMap.of(); Backend backendA = new Backend(0, "addressA", 0); - backendA.updateOnce(0, 0); + backendA.updateOnce(0, 0, 0); Backend backendB = new Backend(1, "addressB", 0); - backendB.updateOnce(0, 0); + backendB.updateOnce(0, 0, 0); Backend backendC = new Backend(2, "addressC", 0); - backendC.updateOnce(0, 0); + backendC.updateOnce(0, 0, 0); Map threeBackends = Maps.newHashMap(); threeBackends.put((long) 0, backendA); @@ -148,11 +146,11 @@ public void testGetHostWithNoParams() { ImmutableMap emptyBackends = ImmutableMap.of(); Backend backendA = new Backend(0, "addressA", 0); - backendA.updateOnce(0, 0); + backendA.updateOnce(0, 0, 0); Backend backendB = new Backend(1, "addressB", 0); - backendB.updateOnce(0, 0); + backendB.updateOnce(0, 0, 0); Backend backendC = new Backend(2, "addressC", 0); - backendC.updateOnce(0, 0); + backendC.updateOnce(0, 0, 0); Map threeBackends = Maps.newHashMap(); threeBackends.put((long) 0, backendA); threeBackends.put((long) 1, backendB); @@ -184,11 +182,11 @@ public void testBlackList() { TNetworkAddress address = null; Backend backendA = new Backend(0, "addressA", 0); - backendA.updateOnce(0, 0); + backendA.updateOnce(0, 0, 0); Backend backendB = new Backend(1, "addressB", 0); - backendB.updateOnce(0, 0); + backendB.updateOnce(0, 0, 0); Backend backendC = new Backend(2, "addressC", 0); - backendC.updateOnce(0, 0); + backendC.updateOnce(0, 0, 0); Map threeBackends = Maps.newHashMap(); threeBackends.put((long) 100, backendA); threeBackends.put((long) 101, backendB); diff --git a/thirdparty/java-libraries.tar.gz b/thirdparty/java-libraries.tar.gz index f4faf115506332..b511d33e473211 100644 Binary files a/thirdparty/java-libraries.tar.gz and b/thirdparty/java-libraries.tar.gz differ