From 354b99e9640f2ab8773e576758b86dc772c1835f Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 2 Sep 2025 13:56:31 +0530 Subject: [PATCH 1/9] Move MetadataStorageActionHandler classes to appropriate modules --- .../sqlserver-metadata-storage/pom.xml | 6 +++ ...SQLServerMetadataStorageActionHandler.java | 0 ...erMetadataStorageActionHandlerFactory.java | 0 .../mysql-metadata-storage/pom.xml | 6 +++ .../MySQLMetadataStorageActionHandler.java | 0 ...QLMetadataStorageActionHandlerFactory.java | 0 .../postgresql-metadata-storage/pom.xml | 6 +++ ...ostgreSQLMetadataStorageActionHandler.java | 0 ...QLMetadataStorageActionHandlerFactory.java | 0 indexing-service/pom.xml | 9 ++-- .../druid/guice/DerbyTaskStorageModule.java | 53 +++++++++++++++++++ .../DerbyMetadataStorageActionHandler.java | 0 ...byMetadataStorageActionHandlerFactory.java | 0 .../MetadataStorageActionHandler.java | 0 .../MetadataStorageActionHandlerFactory.java | 0 .../MetadataStorageActionHandlerTypes.java | 0 .../SQLMetadataStorageActionHandler.java | 9 +--- ...QLMetadataStorageActionHandlerFactory.java | 0 .../MetadataStorageActionHandlerTest.java | 0 .../SQLMetadataStorageActionHandlerTest.java | 0 .../guice/SQLMetadataStorageDruidModule.java | 3 -- .../druid/metadata/SQLMetadataConnector.java | 8 +++ .../druid/metadata/input/SqlEntity.java | 4 +- .../DerbyMetadataStorageDruidModule.java | 7 --- .../org/apache/druid/cli/CliOverlord.java | 2 + 25 files changed, 88 insertions(+), 25 deletions(-) rename {server => extensions-contrib/sqlserver-metadata-storage}/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java (100%) rename {server => extensions-contrib/sqlserver-metadata-storage}/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java (100%) rename {server => extensions-core/mysql-metadata-storage}/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java (100%) rename {server => extensions-core/mysql-metadata-storage}/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java (100%) rename {server => extensions-core/postgresql-metadata-storage}/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java (100%) rename {server => extensions-core/postgresql-metadata-storage}/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java (100%) create mode 100644 indexing-service/src/main/java/org/apache/druid/guice/DerbyTaskStorageModule.java rename {server => indexing-service}/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java (100%) rename {server => indexing-service}/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java (100%) rename {processing => indexing-service}/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java (100%) rename {processing => indexing-service}/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java (100%) rename {processing => indexing-service}/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerTypes.java (100%) rename {server => indexing-service}/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java (99%) rename {server => indexing-service}/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerFactory.java (100%) rename {processing => indexing-service}/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java (100%) rename {server => indexing-service}/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java (100%) diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 2f199a92497d..c91684ac2020 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -46,6 +46,12 @@ ${project.parent.version} provided + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + org.jdbi diff --git a/server/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java rename to extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java diff --git a/server/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java rename to extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 51478c56e16b..29d6170c340e 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -47,6 +47,12 @@ ${project.parent.version} provided + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + com.mysql mysql-connector-j diff --git a/server/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java rename to extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java diff --git a/server/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java rename to extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index f6799a537e9e..4e3eb6bdba9a 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -47,6 +47,12 @@ ${project.parent.version} provided + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + org.postgresql postgresql diff --git a/server/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java rename to extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java diff --git a/server/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java rename to extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index f0da679d4a45..fd886ad0d5aa 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -129,6 +129,10 @@ servlet-api provided + + org.jdbi + jdbi + io.netty netty-handler @@ -262,11 +266,6 @@ maven-resolver-api 1.3.1 - - org.jdbi - jdbi - test - diff --git a/indexing-service/src/main/java/org/apache/druid/guice/DerbyTaskStorageModule.java b/indexing-service/src/main/java/org/apache/druid/guice/DerbyTaskStorageModule.java new file mode 100644 index 000000000000..d8df2f93ce31 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/guice/DerbyTaskStorageModule.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Key; +import com.google.inject.Module; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.annotations.LoadScope; +import org.apache.druid.metadata.DerbyMetadataStorageActionHandlerFactory; +import org.apache.druid.metadata.MetadataStorageActionHandlerFactory; +import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; + +/** + * Module to bind Derby implementation of {@link MetadataStorageActionHandlerFactory} + * used by {@code MetadataTaskStorage}. + */ +@LoadScope(roles = NodeRole.OVERLORD_JSON_NAME) +public class DerbyTaskStorageModule implements Module +{ + @Override + public void configure(Binder binder) + { + PolyBind.createChoiceWithDefault( + binder, + SQLMetadataStorageDruidModule.PROPERTY, + Key.get(MetadataStorageActionHandlerFactory.class), + DerbyMetadataStorageDruidModule.TYPE + ); + + PolyBind.optionBinder(binder, Key.get(MetadataStorageActionHandlerFactory.class)) + .addBinding(DerbyMetadataStorageDruidModule.TYPE) + .to(DerbyMetadataStorageActionHandlerFactory.class) + .in(LazySingleton.class); + } +} diff --git a/server/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java rename to indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java diff --git a/server/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java rename to indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java similarity index 100% rename from processing/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java rename to indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java similarity index 100% rename from processing/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java rename to indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerTypes.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerTypes.java similarity index 100% rename from processing/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerTypes.java rename to indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerTypes.java diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java similarity index 99% rename from server/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java rename to indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java index f8c506d71da9..72dbeb1a33e8 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java @@ -47,7 +47,6 @@ import org.skife.jdbi.v2.Query; import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.exceptions.CallbackFailedException; -import org.skife.jdbi.v2.exceptions.StatementException; import org.skife.jdbi.v2.tweak.HandleCallback; import org.skife.jdbi.v2.tweak.ResultSetMapper; import org.skife.jdbi.v2.util.ByteArrayMapper; @@ -239,12 +238,6 @@ private Void insertEntryWithHandle( } } - public static boolean isStatementException(Throwable e) - { - return e instanceof StatementException || - (e instanceof CallbackFailedException && e.getCause() instanceof StatementException); - } - private boolean isTransientDruidException(Throwable t) { if (t instanceof CallbackFailedException) { @@ -446,7 +439,7 @@ List> getTaskStatusList( */ private DruidException wrapInDruidException(String taskId, Throwable t) { - if (isStatementException(t) && getEntry(taskId).isPresent()) { + if (SQLMetadataConnector.isStatementException(t) && getEntry(taskId).isPresent()) { return InvalidInput.exception("Task [%s] already exists", taskId); } else if (connector.isRootCausePacketTooBigException(t)) { return InvalidInput.exception( diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerFactory.java b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerFactory.java similarity index 100% rename from server/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerFactory.java rename to indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerFactory.java diff --git a/processing/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java b/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java similarity index 100% rename from processing/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java rename to indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java b/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java similarity index 100% rename from server/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java rename to indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java index 8ef0f36a07fa..ed964dcc1bc3 100644 --- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java @@ -23,7 +23,6 @@ import com.google.inject.Key; import com.google.inject.Module; import org.apache.druid.audit.AuditManager; -import org.apache.druid.metadata.MetadataStorageActionHandlerFactory; import org.apache.druid.metadata.MetadataStorageConnector; import org.apache.druid.metadata.MetadataStorageProvider; import org.apache.druid.metadata.SQLMetadataConnector; @@ -54,8 +53,6 @@ public void createBindingChoices(Binder binder, String defaultValue) PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageProvider.class), defaultValue); PolyBind.createChoiceWithDefault(binder, prop, Key.get(SQLMetadataConnector.class), defaultValue); - PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageActionHandlerFactory.class), defaultValue); - configureAuditManager(binder); } diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java index 42b13ab11c89..584a1b6d3584 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java @@ -38,7 +38,9 @@ import org.skife.jdbi.v2.TransactionCallback; import org.skife.jdbi.v2.TransactionIsolationLevel; import org.skife.jdbi.v2.TransactionStatus; +import org.skife.jdbi.v2.exceptions.CallbackFailedException; import org.skife.jdbi.v2.exceptions.DBIException; +import org.skife.jdbi.v2.exceptions.StatementException; import org.skife.jdbi.v2.exceptions.UnableToExecuteStatementException; import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; import org.skife.jdbi.v2.tweak.HandleCallback; @@ -1199,4 +1201,10 @@ private void validateSegmentsTable() ); } } + + public static boolean isStatementException(Throwable e) + { + return e instanceof StatementException || + (e instanceof CallbackFailedException && e.getCause() instanceof StatementException); + } } diff --git a/server/src/main/java/org/apache/druid/metadata/input/SqlEntity.java b/server/src/main/java/org/apache/druid/metadata/input/SqlEntity.java index 0d55e0ed7b59..dcf315b81f87 100644 --- a/server/src/main/java/org/apache/druid/metadata/input/SqlEntity.java +++ b/server/src/main/java/org/apache/druid/metadata/input/SqlEntity.java @@ -28,7 +28,7 @@ import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.metadata.SQLInputSourceDatabaseConnector; -import org.apache.druid.metadata.SQLMetadataStorageActionHandler; +import org.apache.druid.metadata.SQLMetadataConnector; import org.skife.jdbi.v2.ResultIterator; import org.skife.jdbi.v2.exceptions.ResultSetException; @@ -162,7 +162,7 @@ public static CleanableFile openCleanableFile( return null; }, (exception) -> sqlInputSourceDatabaseConnector.isTransientException(exception) - && !(SQLMetadataStorageActionHandler.isStatementException(exception)) + && !(SQLMetadataConnector.isStatementException(exception)) ); return new CleanableFile() { diff --git a/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java index 409c95eb6da4..cb333e974849 100644 --- a/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java +++ b/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java @@ -24,9 +24,7 @@ import org.apache.druid.guice.LazySingleton; import org.apache.druid.guice.PolyBind; import org.apache.druid.guice.SQLMetadataStorageDruidModule; -import org.apache.druid.metadata.DerbyMetadataStorageActionHandlerFactory; import org.apache.druid.metadata.MetadataStorage; -import org.apache.druid.metadata.MetadataStorageActionHandlerFactory; import org.apache.druid.metadata.MetadataStorageConnector; import org.apache.druid.metadata.MetadataStorageProvider; import org.apache.druid.metadata.NoopMetadataStorageProvider; @@ -63,10 +61,5 @@ public void configure(Binder binder) .addBinding(TYPE) .to(DerbyConnector.class) .in(LazySingleton.class); - - PolyBind.optionBinder(binder, Key.get(MetadataStorageActionHandlerFactory.class)) - .addBinding(TYPE) - .to(DerbyMetadataStorageActionHandlerFactory.class) - .in(LazySingleton.class); } } diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index ca175a1286b9..c88233244c3c 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -38,6 +38,7 @@ import com.google.inject.util.Providers; import org.apache.druid.client.indexing.IndexingService; import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.DerbyTaskStorageModule; import org.apache.druid.guice.IndexingServiceInputSourceModule; import org.apache.druid.guice.IndexingServiceModuleHelper; import org.apache.druid.guice.IndexingServiceTaskLogsModule; @@ -198,6 +199,7 @@ public void configure(Properties properties) protected List getModules(final boolean standalone) { return ImmutableList.of( + new DerbyTaskStorageModule(), standalone ? new MetadataManagerModule() : binder -> {}, new Module() { From 237a7cdb3ac692997326e4b10544dacbdc98de62 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 2 Sep 2025 20:42:14 +0530 Subject: [PATCH 2/9] Remove generic params from MetadataStorageActionHandler --- ...SQLServerMetadataStorageActionHandler.java | 9 +- ...erMetadataStorageActionHandlerFactory.java | 15 +- .../CustomStatementRewriterTest.java | 4 +- .../MySQLMetadataStorageActionHandler.java | 9 +- ...QLMetadataStorageActionHandlerFactory.java | 15 +- ...ostgreSQLMetadataStorageActionHandler.java | 9 +- ...QLMetadataStorageActionHandlerFactory.java | 15 +- .../overlord/MetadataTaskStorage.java | 5 +- .../DerbyMetadataStorageActionHandler.java | 9 +- ...byMetadataStorageActionHandlerFactory.java | 14 +- .../MetadataStorageActionHandler.java | 66 ++--- .../MetadataStorageActionHandlerFactory.java | 5 +- .../SQLMetadataStorageActionHandler.java | 192 +++++--------- .../MetadataStorageActionHandlerTest.java | 61 ++--- .../SQLMetadataStorageActionHandlerTest.java | 249 +++++++----------- .../metadata/MetadataStorageTablesConfig.java | 30 --- .../MetadataStorageTablesConfigTest.java | 6 +- .../druid/metadata/SQLMetadataConnector.java | 13 +- ...etadataConnectorSchemaPersistenceTest.java | 9 +- .../metadata/SQLMetadataConnectorTest.java | 8 +- 20 files changed, 249 insertions(+), 494 deletions(-) diff --git a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java index 2c4e93820d25..91cd7d7112bd 100644 --- a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java +++ b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java @@ -21,20 +21,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; -public class SQLServerMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class SQLServerMetadataStorageActionHandler + extends SQLMetadataStorageActionHandler { public SQLServerMetadataStorageActionHandler( SQLMetadataConnector connector, ObjectMapper jsonMapper, - MetadataStorageActionHandlerTypes types, - String entryTypeName, String entryTable, - String logTable, String lockTable ) { - super(connector, jsonMapper, types, entryTypeName, entryTable, logTable, lockTable); + super(connector, jsonMapper, entryTable, lockTable); } @Override diff --git a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java index af6725175340..f2e548a7abce 100644 --- a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java +++ b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandlerFactory.java @@ -35,20 +35,13 @@ public SQLServerMetadataStorageActionHandlerFactory( } @Override - public - MetadataStorageActionHandler create( - String entryType, - MetadataStorageActionHandlerTypes payloadTypes - ) + public MetadataStorageActionHandler create() { - return new SQLServerMetadataStorageActionHandler<>( + return new SQLServerMetadataStorageActionHandler( connector, jsonMapper, - payloadTypes, - entryType, - config.getEntryTable(entryType), - config.getLogTable(entryType), - config.getLockTable(entryType) + config.getTasksTable(), + config.getTaskLockTable() ); } } diff --git a/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/CustomStatementRewriterTest.java b/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/CustomStatementRewriterTest.java index 7dc3ae300827..b0764095c782 100644 --- a/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/CustomStatementRewriterTest.java +++ b/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/CustomStatementRewriterTest.java @@ -20,6 +20,7 @@ package org.apache.druid.metadata.storage.sqlserver; import junit.framework.Assert; +import org.apache.druid.metadata.MetadataStorageActionHandler; import org.apache.druid.metadata.storage.sqlserver.SQLServerConnector.CustomStatementRewriter; import org.junit.Before; import org.junit.Test; @@ -128,8 +129,7 @@ public void testSQLMetadataConnectorCreateTable() /** * - * @see org.apache.druid.metadata.SQLMetadataStorageActionHandler#setStatus(String, - * boolean, Object) + * @see MetadataStorageActionHandler#setStatus(String, boolean, org.apache.druid.indexer.TaskStatus) * */ @Test diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java index 79c028202f41..c30d23ce6ec9 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java @@ -21,20 +21,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; -public class MySQLMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class MySQLMetadataStorageActionHandler + extends SQLMetadataStorageActionHandler { MySQLMetadataStorageActionHandler( SQLMetadataConnector connector, ObjectMapper jsonMapper, - MetadataStorageActionHandlerTypes types, - String entryTypeName, String entryTable, - String logTable, String lockTable ) { - super(connector, jsonMapper, types, entryTypeName, entryTable, logTable, lockTable); + super(connector, jsonMapper, entryTable, lockTable); } @Override diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java index 0af63ba1d631..eb9f04e63a0b 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandlerFactory.java @@ -35,20 +35,13 @@ public MySQLMetadataStorageActionHandlerFactory( } @Override - public - MetadataStorageActionHandler create( - String entryType, - MetadataStorageActionHandlerTypes payloadTypes - ) + public MetadataStorageActionHandler create() { - return new MySQLMetadataStorageActionHandler<>( + return new MySQLMetadataStorageActionHandler( connector, jsonMapper, - payloadTypes, - entryType, - config.getEntryTable(entryType), - config.getLogTable(entryType), - config.getLockTable(entryType) + config.getTasksTable(), + config.getTaskLockTable() ); } } diff --git a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java index 6bed60b89b41..c04cf17c43ff 100644 --- a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java +++ b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java @@ -21,20 +21,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; -public class PostgreSQLMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class PostgreSQLMetadataStorageActionHandler + extends SQLMetadataStorageActionHandler { public PostgreSQLMetadataStorageActionHandler( SQLMetadataConnector connector, ObjectMapper jsonMapper, - MetadataStorageActionHandlerTypes types, - String entryTypeName, String entryTable, - String logTable, String lockTable ) { - super(connector, jsonMapper, types, entryTypeName, entryTable, logTable, lockTable); + super(connector, jsonMapper, entryTable, lockTable); } @Override diff --git a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java index 329b7a52474b..dfc4a411c27a 100644 --- a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java +++ b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandlerFactory.java @@ -35,20 +35,13 @@ public PostgreSQLMetadataStorageActionHandlerFactory( } @Override - public - MetadataStorageActionHandler create( - String entryType, - MetadataStorageActionHandlerTypes payloadTypes - ) + public MetadataStorageActionHandler create() { - return new PostgreSQLMetadataStorageActionHandler<>( + return new PostgreSQLMetadataStorageActionHandler( connector, jsonMapper, - payloadTypes, - entryType, - config.getEntryTable(entryType), - config.getLogTable(entryType), - config.getLockTable(entryType) + config.getTasksTable(), + config.getTaskLockTable() ); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java index ae78bc2c3ff1..9ead5b2516c9 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java @@ -42,7 +42,6 @@ import org.apache.druid.metadata.MetadataStorageActionHandlerFactory; import org.apache.druid.metadata.MetadataStorageActionHandlerTypes; import org.apache.druid.metadata.MetadataStorageConnector; -import org.apache.druid.metadata.MetadataStorageTablesConfig; import org.apache.druid.metadata.TaskLookup; import org.apache.druid.metadata.TaskLookup.ActiveTaskLookup; import org.apache.druid.metadata.TaskLookup.TaskLookupType; @@ -81,7 +80,7 @@ public TypeReference getLockType() private final MetadataStorageConnector metadataStorageConnector; private final TaskStorageConfig config; - private final MetadataStorageActionHandler handler; + private final MetadataStorageActionHandler handler; private static final EmittingLogger log = new EmittingLogger(MetadataTaskStorage.class); @@ -94,7 +93,7 @@ public MetadataTaskStorage( { this.metadataStorageConnector = metadataStorageConnector; this.config = config; - this.handler = factory.create(MetadataStorageTablesConfig.TASK_ENTRY_TYPE, TASK_TYPES); + this.handler = factory.create(); } @LifecycleStart diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java index c3265f388294..7491d5c89bcf 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java @@ -22,21 +22,18 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -public class DerbyMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class DerbyMetadataStorageActionHandler + extends SQLMetadataStorageActionHandler { @VisibleForTesting DerbyMetadataStorageActionHandler( SQLMetadataConnector connector, ObjectMapper jsonMapper, - MetadataStorageActionHandlerTypes types, - String entryTypeName, String entryTable, - String logTable, String lockTable ) { - super(connector, jsonMapper, types, entryTypeName, entryTable, logTable, lockTable); + super(connector, jsonMapper, entryTable, lockTable); } @Override diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java index cf6dc44ed93b..8c480ebff87c 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandlerFactory.java @@ -35,19 +35,13 @@ public DerbyMetadataStorageActionHandlerFactory( } @Override - public MetadataStorageActionHandler create( - final String entryType, - MetadataStorageActionHandlerTypes payloadTypes - ) + public MetadataStorageActionHandler create() { - return new DerbyMetadataStorageActionHandler<>( + return new DerbyMetadataStorageActionHandler( connector, jsonMapper, - payloadTypes, - entryType, - config.getEntryTable(entryType), - config.getLogTable(entryType), - config.getLockTable(entryType) + config.getTasksTable(), + config.getTaskLockTable() ); } } diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java index 4a5a48e8ef78..daf574acf02b 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java @@ -20,10 +20,12 @@ package org.apache.druid.metadata; import com.google.common.base.Optional; -import org.apache.druid.error.DruidException; import org.apache.druid.guice.annotations.ExtensionPoint; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; +import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.task.Task; import org.apache.druid.metadata.TaskLookup.TaskLookupType; import org.joda.time.DateTime; @@ -34,7 +36,7 @@ import java.util.Map; @ExtensionPoint -public interface MetadataStorageActionHandler +public interface MetadataStorageActionHandler { /** * Creates a new entry. @@ -52,9 +54,9 @@ void insert( @NotNull String id, @NotNull DateTime timestamp, @NotNull String dataSource, - @NotNull EntryType entry, + @NotNull Task entry, boolean active, - @Nullable StatusType status, + @Nullable TaskStatus status, @NotNull String type, @NotNull String groupId ); @@ -68,7 +70,7 @@ void insert( * @param status status * @return true if the status was updated, false if the entry did not exist of if the entry was inactive */ - boolean setStatus(String entryId, boolean active, StatusType status); + boolean setStatus(String entryId, boolean active, TaskStatus status); /** * Retrieves the entry with the given id. @@ -76,7 +78,7 @@ void insert( * @param entryId entry id * @return optional entry, absent if the given id does not exist */ - Optional getEntry(String entryId); + Optional getEntry(String entryId); /** * Retrieve the status for the entry with the given id. @@ -84,14 +86,14 @@ void insert( * @param entryId entry id * @return optional status, absent if entry does not exist or status is not set */ - Optional getStatus(String entryId); + Optional getStatus(String entryId); @Nullable - TaskInfo getTaskInfo(String entryId); + TaskInfo getTaskInfo(String entryId); /** * Returns a list of {@link TaskInfo} from metadata store that matches to the given filters. - * + *

* If {@code taskLookups} includes {@link TaskLookupType#ACTIVE}, it returns all active tasks in the metadata store. * If {@code taskLookups} includes {@link TaskLookupType#COMPLETE}, it returns all complete tasks in the metadata * store. For complete tasks, additional filters in {@code CompleteTaskLookup} can be applied. @@ -100,14 +102,14 @@ void insert( * @param taskLookups task lookup type and filters. * @param datasource datasource filter */ - List> getTaskInfos( + List> getTaskInfos( Map taskLookups, @Nullable String datasource ); /** * Returns the statuses of the specified tasks. - * + *

* If {@code taskLookups} includes {@link TaskLookupType#ACTIVE}, it returns all active tasks in the metadata store. * If {@code taskLookups} includes {@link TaskLookupType#COMPLETE}, it returns all complete tasks in the metadata * store. For complete tasks, additional filters in {@code CompleteTaskLookup} can be applied. @@ -116,12 +118,12 @@ List> getTaskInfos( * @param taskLookups task lookup type and filters. * @param datasource datasource filter */ - List> getTaskStatusList( + List> getTaskStatusList( Map taskLookups, @Nullable String datasource ); - default List> getTaskInfos( + default List> getTaskInfos( TaskLookup taskLookup, @Nullable String datasource ) @@ -136,7 +138,7 @@ default List> getTaskInfos( * @param lock lock to add * @return true if the lock was added */ - boolean addLock(String entryId, LockType lock); + boolean addLock(String entryId, TaskLock lock); /** * Replace an existing lock with a new lock. @@ -147,7 +149,7 @@ default List> getTaskInfos( * * @return true if the lock is replaced */ - boolean replaceLock(String entryId, long oldLockId, LockType newLock); + boolean replaceLock(String entryId, long oldLockId, TaskLock newLock); /** * Remove the lock with the given lock id. @@ -163,43 +165,13 @@ default List> getTaskInfos( */ void removeTasksOlderThan(long timestamp); - /** - * Task logs are not used anymore and this method is never called by Druid code. - * It has been retained only for backwards compatibility with older extensions. - * New extensions must not implement this method. - * - * @throws DruidException of category UNSUPPORTED whenever called. - */ - @Deprecated - default boolean addLog(String entryId, LogType log) - { - throw DruidException.defensive() - .ofCategory(DruidException.Category.UNSUPPORTED) - .build("Task actions are not logged anymore."); - } - - /** - * Task logs are not used anymore and this method is never called by Druid code. - * It has been retained only for backwards compatibility with older extensions. - * New extensions must not implement this method. - * - * @throws DruidException of category UNSUPPORTED whenever called. - */ - @Deprecated - default List getLogs(String entryId) - { - throw DruidException.defensive() - .ofCategory(DruidException.Category.UNSUPPORTED) - .build("Task actions are not logged anymore."); - } - /** * Returns the locks for the given entry * * @param entryId entry id * @return map of lockId to lock */ - Map getLocks(String entryId); + Map getLocks(String entryId); /** * Returns the lock id for the given entry and the lock. @@ -207,7 +179,7 @@ default List getLogs(String entryId) * @return lock id if found, otherwise null. */ @Nullable - Long getLockId(String entryId, LockType lock); + Long getLockId(String entryId, TaskLock lock); /** * Utility to migrate existing tasks to the new schema by populating type and groupId asynchronously diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java index a81ecfdbf1c4..2a4ea305696b 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandlerFactory.java @@ -21,8 +21,5 @@ public interface MetadataStorageActionHandlerFactory { - MetadataStorageActionHandler create( - String entryType, - MetadataStorageActionHandlerTypes payloadTypes - ); + MetadataStorageActionHandler create(); } diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java index 72dbeb1a33e8..cf8bcf6e6586 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java @@ -20,7 +20,6 @@ package org.apache.druid.metadata; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -32,6 +31,9 @@ import org.apache.druid.error.InvalidInput; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; +import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.task.Task; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Pair; @@ -41,8 +43,6 @@ import org.apache.druid.metadata.TaskLookup.TaskLookupType; import org.joda.time.DateTime; import org.skife.jdbi.v2.Batch; -import org.skife.jdbi.v2.FoldController; -import org.skife.jdbi.v2.Folder3; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.Query; import org.skife.jdbi.v2.StatementContext; @@ -52,6 +52,7 @@ import org.skife.jdbi.v2.util.ByteArrayMapper; import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; @@ -64,68 +65,38 @@ import java.util.concurrent.Future; import java.util.stream.IntStream; -public abstract class SQLMetadataStorageActionHandler - implements MetadataStorageActionHandler +public abstract class SQLMetadataStorageActionHandler + implements MetadataStorageActionHandler { private static final EmittingLogger log = new EmittingLogger(SQLMetadataStorageActionHandler.class); private static final String CONTEXT_KEY_IS_TRANSIENT = "isTransient"; private final SQLMetadataConnector connector; private final ObjectMapper jsonMapper; - private final TypeReference entryType; - private final TypeReference statusType; - private final TypeReference lockType; - private final String entryTypeName; private final String entryTable; private final String lockTable; - private final TaskInfoMapper taskInfoMapper; + private final TaskInfoMapper taskInfoMapper; private final TaskStatusMapper taskStatusMapper; private final TaskStatusMapperFromPayload taskStatusMapperFromPayload; private final TaskIdentifierMapper taskIdentifierMapper; private Future taskMigrationCompleteFuture; - /** - * @deprecated Use the other constructor without {@code logTable} argument - * since this argument is now unused. - */ - @Deprecated - public SQLMetadataStorageActionHandler( - final SQLMetadataConnector connector, - final ObjectMapper jsonMapper, - final MetadataStorageActionHandlerTypes types, - final String entryTypeName, - final String entryTable, - final String logTable, - final String lockTable - ) - { - this(connector, jsonMapper, types, entryTypeName, entryTable, lockTable); - } - @SuppressWarnings("PMD.UnnecessaryFullyQualifiedName") public SQLMetadataStorageActionHandler( final SQLMetadataConnector connector, final ObjectMapper jsonMapper, - final MetadataStorageActionHandlerTypes types, - final String entryTypeName, final String entryTable, final String lockTable ) { this.connector = connector; - //fully qualified references required below due to identical package names across project modules. - //noinspection UnnecessaryFullyQualifiedName this.jsonMapper = jsonMapper.copy().addMixIn(PasswordProvider.class, PasswordProviderRedactionMixIn.class); - this.entryType = types.getEntryType(); - this.statusType = types.getStatusType(); - this.lockType = types.getLockType(); - this.entryTypeName = entryTypeName; this.entryTable = entryTable; this.lockTable = lockTable; - this.taskInfoMapper = new TaskInfoMapper<>(jsonMapper, entryType, statusType); + this.taskInfoMapper = new TaskInfoMapper(jsonMapper); this.taskStatusMapper = new TaskStatusMapper(jsonMapper); this.taskStatusMapperFromPayload = new TaskStatusMapperFromPayload(jsonMapper); this.taskIdentifierMapper = new TaskIdentifierMapper(jsonMapper); @@ -141,39 +112,19 @@ protected ObjectMapper getJsonMapper() return jsonMapper; } - protected TypeReference getStatusType() - { - return statusType; - } - protected String getEntryTable() { return entryTable; } - protected String getLogTable() - { - throw new UnsupportedOperationException("'tasklogs' table is not used anymore"); - } - - protected String getEntryTypeName() - { - return entryTypeName; - } - - public TypeReference getEntryType() - { - return entryType; - } - @Override public void insert( final String id, final DateTime timestamp, final String dataSource, - final EntryType entry, + final @NotNull Task entry, final boolean active, - final StatusType status, + final TaskStatus status, final String type, final String groupId ) @@ -208,9 +159,9 @@ private Void insertEntryWithHandle( String entryId, DateTime timestamp, String dataSource, - EntryType entry, + Object entry, boolean active, - StatusType status, + Object status, String type, String groupId ) @@ -250,7 +201,7 @@ private boolean isTransientDruidException(Throwable t) } @Override - public boolean setStatus(final String entryId, final boolean active, final StatusType status) + public boolean setStatus(final String entryId, final boolean active, final TaskStatus status) { return connector.retryWithHandle( new HandleCallback<>() @@ -274,13 +225,13 @@ public Boolean withHandle(Handle handle) throws Exception } @Override - public Optional getEntry(final String entryId) + public Optional getEntry(final String entryId) { return connector.retryWithHandle( new HandleCallback<>() { @Override - public Optional withHandle(Handle handle) throws Exception + public Optional withHandle(Handle handle) throws Exception { byte[] res = handle.createQuery( StringUtils.format("SELECT payload FROM %s WHERE id = :id", entryTable) @@ -290,7 +241,7 @@ public Optional withHandle(Handle handle) throws Exception .first(); return Optional.fromNullable( - res == null ? null : jsonMapper.readValue(res, entryType) + res == null ? null : jsonMapper.readValue(res, Task.class) ); } } @@ -299,13 +250,13 @@ public Optional withHandle(Handle handle) throws Exception } @Override - public Optional getStatus(final String entryId) + public Optional getStatus(final String entryId) { return connector.retryWithHandle( new HandleCallback<>() { @Override - public Optional withHandle(Handle handle) throws Exception + public Optional withHandle(Handle handle) throws Exception { byte[] res = handle.createQuery( StringUtils.format("SELECT status_payload FROM %s WHERE id = :id", entryTable) @@ -315,7 +266,7 @@ public Optional withHandle(Handle handle) throws Exception .first(); return Optional.fromNullable( - res == null ? null : jsonMapper.readValue(res, statusType) + res == null ? null : jsonMapper.readValue(res, TaskStatus.class) ); } } @@ -324,7 +275,7 @@ public Optional withHandle(Handle handle) throws Exception @Override @Nullable - public TaskInfo getTaskInfo(String entryId) + public TaskInfo getTaskInfo(String entryId) { return connector.retryWithHandle(handle -> { final String query = StringUtils.format( @@ -339,14 +290,14 @@ public TaskInfo getTaskInfo(String entryId) } @Override - public List> getTaskInfos( + public List> getTaskInfos( Map taskLookups, @Nullable String dataSource ) { return getConnector().retryTransaction( (handle, status) -> { - final List> tasks = new ArrayList<>(); + final List> tasks = new ArrayList<>(); for (Entry entry : taskLookups.entrySet()) { final Query> query; switch (entry.getKey()) { @@ -376,7 +327,7 @@ public List> getTaskInfos( } @Override - public List> getTaskStatusList( + public List> getTaskStatusList( Map taskLookups, @Nullable String dataSource ) @@ -394,17 +345,17 @@ public List> getTaskStatusList( } @VisibleForTesting - List> getTaskStatusList( + List> getTaskStatusList( Map taskLookups, @Nullable String dataSource, boolean fetchPayload ) { - ResultSetMapper> resultSetMapper = + ResultSetMapper> resultSetMapper = fetchPayload ? taskStatusMapperFromPayload : taskStatusMapper; return getConnector().retryTransaction( (handle, status) -> { - final List> taskMetadataInfos = new ArrayList<>(); + final List> taskMetadataInfos = new ArrayList<>(); for (Entry entry : taskLookups.entrySet()) { final Query> query; switch (entry.getKey()) { @@ -641,7 +592,7 @@ private String getWhereClauseForActiveStatusesQuery(String dataSource) return sql; } - private class TaskStatusMapperFromPayload implements ResultSetMapper> + private class TaskStatusMapperFromPayload implements ResultSetMapper> { private final ObjectMapper objectMapper; @@ -651,14 +602,14 @@ private class TaskStatusMapperFromPayload implements ResultSetMapper map(int index, ResultSet resultSet, StatementContext context) + public TaskInfo map(int index, ResultSet resultSet, StatementContext context) throws SQLException { return toTaskIdentifierInfo(objectMapper, resultSet, true); } } - private class TaskStatusMapper implements ResultSetMapper> + private class TaskStatusMapper implements ResultSetMapper> { private final ObjectMapper objectMapper; @@ -668,16 +619,16 @@ private class TaskStatusMapper implements ResultSetMapper map(int index, ResultSet resultSet, StatementContext context) + public TaskInfo map(int index, ResultSet resultSet, StatementContext context) throws SQLException { return toTaskIdentifierInfo(objectMapper, resultSet, false); } } - private TaskInfo toTaskIdentifierInfo(ObjectMapper objectMapper, - ResultSet resultSet, - boolean usePayload + private TaskInfo toTaskIdentifierInfo(ObjectMapper objectMapper, + ResultSet resultSet, + boolean usePayload ) throws SQLException { String type; @@ -699,9 +650,9 @@ private TaskInfo toTaskIdentifierInfo(ObjectMapper o String id = resultSet.getString("id"); DateTime createdTime = DateTimes.of(resultSet.getString("created_date")); - StatusType status; + TaskStatus status; try { - status = objectMapper.readValue(resultSet.getBytes("status_payload"), statusType); + status = objectMapper.readValue(resultSet.getBytes("status_payload"), TaskStatus.class); } catch (IOException e) { log.error(e, "Encountered exception while deserializing task status_payload"); @@ -744,35 +695,31 @@ public TaskIdentifier map(int index, ResultSet resultSet, StatementContext conte } } - static class TaskInfoMapper implements ResultSetMapper> + static class TaskInfoMapper implements ResultSetMapper> { private final ObjectMapper objectMapper; - private final TypeReference entryType; - private final TypeReference statusType; - TaskInfoMapper(ObjectMapper objectMapper, TypeReference entryType, TypeReference statusType) + TaskInfoMapper(ObjectMapper objectMapper) { this.objectMapper = objectMapper; - this.entryType = entryType; - this.statusType = statusType; } @Override - public TaskInfo map(int index, ResultSet resultSet, StatementContext context) + public TaskInfo map(int index, ResultSet resultSet, StatementContext context) throws SQLException { - final TaskInfo taskInfo; - EntryType task; - StatusType status; + final TaskInfo taskInfo; + Task task; + TaskStatus status; try { - task = objectMapper.readValue(resultSet.getBytes("payload"), entryType); + task = objectMapper.readValue(resultSet.getBytes("payload"), Task.class); } catch (IOException e) { log.warn("Encountered exception[%s] while deserializing task payload, setting payload to null", e.getMessage()); task = null; } try { - status = objectMapper.readValue(resultSet.getBytes("status_payload"), statusType); + status = objectMapper.readValue(resultSet.getBytes("status_payload"), TaskStatus.class); } catch (IOException e) { log.error(e, "Encountered exception while deserializing task status_payload"); @@ -790,25 +737,18 @@ public TaskInfo map(int index, ResultSet resultSet, State } @Override - public boolean addLock(final String entryId, final LockType lock) + public boolean addLock(final String entryId, final TaskLock lock) { return connector.retryWithHandle( - new HandleCallback<>() - { - @Override - public Boolean withHandle(Handle handle) throws Exception - { - return addLock(handle, entryId, lock); - } - } + handle -> addLock(handle, entryId, lock) ); } - private boolean addLock(Handle handle, String entryId, LockType lock) throws JsonProcessingException + private boolean addLock(Handle handle, String entryId, Object lock) throws JsonProcessingException { final String statement = StringUtils.format( - "INSERT INTO %1$s (%2$s_id, lock_payload) VALUES (:entryId, :payload)", - lockTable, entryTypeName + "INSERT INTO %1$s (task_id, lock_payload) VALUES (:entryId, :payload)", + lockTable ); return handle.createStatement(statement) .bind("entryId", entryId) @@ -817,7 +757,7 @@ private boolean addLock(Handle handle, String entryId, LockType lock) throws Jso } @Override - public boolean replaceLock(final String entryId, final long oldLockId, final LockType newLock) + public boolean replaceLock(final String entryId, final long oldLockId, final TaskLock newLock) { return connector.retryTransaction( (handle, transactionStatus) -> { @@ -869,26 +809,26 @@ private int removeLock(Handle handle, long lockId) } @Override - public Map getLocks(final String entryId) + public Map getLocks(final String entryId) { return connector.retryWithHandle( new HandleCallback<>() { @Override - public Map withHandle(Handle handle) + public Map withHandle(Handle handle) { return handle.createQuery( StringUtils.format( - "SELECT id, lock_payload FROM %1$s WHERE %2$s_id = :entryId", - lockTable, entryTypeName + "SELECT id, lock_payload FROM %1$s WHERE task_id = :entryId", + lockTable ) ) .bind("entryId", entryId) .map( - new ResultSetMapper>() + new ResultSetMapper>() { @Override - public Pair map(int index, ResultSet r, StatementContext ctx) + public Pair map(int index, ResultSet r, StatementContext ctx) throws SQLException { try { @@ -896,12 +836,12 @@ public Pair map(int index, ResultSet r, StatementContext ctx) r.getLong("id"), jsonMapper.readValue( r.getBytes("lock_payload"), - lockType + TaskLock.class ) ); } catch (IOException e) { - log.makeAlert(e, "Failed to deserialize " + lockType.getType()) + log.makeAlert(e, "Failed to deserialize lock") .addData("id", r.getLong("id")) .addData( "lockPayload", StringUtils.fromUtf8(r.getBytes("lock_payload")) @@ -914,19 +854,9 @@ public Pair map(int index, ResultSet r, StatementContext ctx) ) .fold( Maps.newLinkedHashMap(), - new Folder3, Pair>() - { - @Override - public Map fold( - Map accumulator, - Pair lock, - FoldController control, - StatementContext ctx - ) - { - accumulator.put(lock.lhs, lock.rhs); - return accumulator; - } + (accumulator, lock, control, ctx) -> { + accumulator.put(lock.lhs, lock.rhs); + return accumulator; } ); } @@ -936,7 +866,7 @@ public Map fold( @Override @Nullable - public Long getLockId(String entryId, LockType lock) + public Long getLockId(String entryId, TaskLock lock) { return getLocks(entryId).entrySet().stream() .filter(entry -> entry.getValue().equals(lock)) diff --git a/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java b/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java index 98d579821f6a..014fe3da2e97 100644 --- a/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java @@ -20,15 +20,16 @@ package org.apache.druid.metadata; import com.google.common.base.Optional; -import org.apache.druid.error.DruidException; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; +import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.task.Task; import org.joda.time.DateTime; -import org.junit.Assert; import org.junit.Before; -import org.junit.Test; import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; import java.util.Collections; import java.util.List; import java.util.Map; @@ -41,21 +42,21 @@ public class MetadataStorageActionHandlerTest { - private MetadataStorageActionHandler handler; + private MetadataStorageActionHandler handler; @Before public void setup() { - this.handler = new MetadataStorageActionHandler<>() + this.handler = new MetadataStorageActionHandler() { @Override public void insert( String id, DateTime timestamp, String dataSource, - String entry, + @NotNull Task entry, boolean active, - @Nullable String status, + @Nullable TaskStatus status, String type, String groupId ) @@ -64,32 +65,32 @@ public void insert( } @Override - public boolean setStatus(String entryId, boolean active, String status) + public boolean setStatus(String entryId, boolean active, TaskStatus status) { return false; } @Override - public Optional getEntry(String entryId) + public Optional getEntry(String entryId) { return null; } @Override - public Optional getStatus(String entryId) + public Optional getStatus(String entryId) { return null; } @Nullable @Override - public TaskInfo getTaskInfo(String entryId) + public TaskInfo getTaskInfo(String entryId) { return null; } @Override - public List> getTaskInfos( + public List> getTaskInfos( Map taskLookups, @Nullable String datasource ) @@ -98,7 +99,7 @@ public List> getTaskInfos( } @Override - public List> getTaskStatusList( + public List> getTaskStatusList( Map taskLookups, @Nullable String datasource ) @@ -107,13 +108,13 @@ public List> getTaskStatusList( } @Override - public boolean addLock(String entryId, String lock) + public boolean addLock(String entryId, TaskLock lock) { return false; } @Override - public boolean replaceLock(String entryId, long oldLockId, String newLock) + public boolean replaceLock(String entryId, long oldLockId, TaskLock newLock) { return false; } @@ -131,13 +132,13 @@ public void removeTasksOlderThan(long timestamp) } @Override - public Map getLocks(String entryId) + public Map getLocks(String entryId) { return Collections.emptyMap(); } @Override - public Long getLockId(String entryId, String lock) + public Long getLockId(String entryId, TaskLock lock) { return 0L; } @@ -149,30 +150,4 @@ public void populateTaskTypeAndGroupIdAsync() } }; } - - @Test - public void testAddLogThrowsUnsupportedException() - { - Exception exception = Assert.assertThrows( - DruidException.class, - () -> handler.addLog("abcd", "logentry") - ); - Assert.assertEquals( - "Task actions are not logged anymore.", - exception.getMessage() - ); - } - - @Test - public void testGetLogsThrowsUnsupportedException() - { - Exception exception = Assert.assertThrows( - DruidException.class, - () -> handler.getLogs("abcd") - ); - Assert.assertEquals( - "Task actions are not logged anymore.", - exception.getMessage() - ); - } } diff --git a/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java b/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java index 56ff7751efb3..1fc9a3e656d4 100644 --- a/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java @@ -19,19 +19,24 @@ package org.apache.druid.metadata; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import org.apache.druid.error.DruidException; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskState; +import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.TaskLock; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.TimeChunkLock; +import org.apache.druid.indexing.common.task.NoopTask; +import org.apache.druid.indexing.common.task.Task; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.TaskLookup.ActiveTaskLookup; @@ -50,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -62,7 +68,7 @@ public class SQLMetadataStorageActionHandlerTest private static final Random RANDOM = new Random(1); - private SQLMetadataStorageActionHandler, Map, Map, Map> handler; + private SQLMetadataStorageActionHandler handler; private final String entryTable = "entries"; @@ -71,38 +77,15 @@ public void setUp() { TestDerbyConnector connector = derbyConnectorRule.getConnector(); - final String entryType = "entry"; final String lockTable = "locks"; connector.prepareTaskEntryTable(entryTable); - connector.createLockTable(lockTable, entryType); + connector.createLockTable(lockTable); - handler = new DerbyMetadataStorageActionHandler<>( + handler = new DerbyMetadataStorageActionHandler( connector, JSON_MAPPER, - new MetadataStorageActionHandlerTypes<>() - { - @Override - public TypeReference> getEntryType() - { - return new TypeReference<>() {}; - } - - @Override - public TypeReference> getStatusType() - { - return new TypeReference<>() {}; - } - - @Override - public TypeReference> getLockType() - { - return new TypeReference<>() {}; - } - }, - entryType, entryTable, - null, lockTable ); } @@ -110,11 +93,11 @@ public TypeReference> getLockType() @Test public void testEntryAndStatus() { - Map entry = ImmutableMap.of("numericId", 1234); - Map status1 = ImmutableMap.of("count", 42); - Map status2 = ImmutableMap.of("count", 42, "temp", 1); + Task entry = NoopTask.create(); + TaskStatus status1 = TaskStatus.running(entry.getId()); + TaskStatus status2 = TaskStatus.success(entry.getId()); - final String entryId = "1234"; + final String entryId = entry.getId(); handler.insert(entryId, DateTimes.of("2014-01-02T00:00:00.123"), "testDataSource", entry, true, null, "type", "group"); @@ -174,21 +157,20 @@ public void testEntryAndStatus() public void testGetRecentStatuses() { for (int i = 1; i < 11; i++) { - final String entryId = "abcd_" + i; - final Map entry = ImmutableMap.of("a", i); - final Map status = ImmutableMap.of("count", i * 10); + final Task entry = NoopTask.create(); + final String entryId = entry.getId(); + final TaskStatus status = TaskStatus.running(entry.getId()); handler.insert(entryId, DateTimes.of(StringUtils.format("2014-01-%02d", i)), "test", entry, false, status, "type", "group"); } - final List, Map>> statuses = handler.getTaskInfos( + final List> statuses = handler.getTaskInfos( CompleteTaskLookup.withTasksCreatedPriorTo(7, DateTimes.of("2014-01-01")), null ); Assert.assertEquals(7, statuses.size()); - int i = 10; - for (TaskInfo, Map> status : statuses) { - Assert.assertEquals(ImmutableMap.of("count", i-- * 10), status.getStatus()); + for (TaskInfo status : statuses) { + Assert.assertEquals(TaskState.RUNNING, status.getStatus().getStatusCode()); } } @@ -196,30 +178,29 @@ public void testGetRecentStatuses() public void testGetRecentStatuses2() { for (int i = 1; i < 6; i++) { - final String entryId = "abcd_" + i; - final Map entry = ImmutableMap.of("a", i); - final Map status = ImmutableMap.of("count", i * 10); + final Task entry = NoopTask.create(); + final String entryId = entry.getId(); + final TaskStatus status = TaskStatus.running(entry.getId()); handler.insert(entryId, DateTimes.of(StringUtils.format("2014-01-%02d", i)), "test", entry, false, status, "type", "group"); } - final List, Map>> statuses = handler.getTaskInfos( + final List> statuses = handler.getTaskInfos( CompleteTaskLookup.withTasksCreatedPriorTo(10, DateTimes.of("2014-01-01")), null ); Assert.assertEquals(5, statuses.size()); - int i = 5; - for (TaskInfo, Map> status : statuses) { - Assert.assertEquals(ImmutableMap.of("count", i-- * 10), status.getStatus()); + for (TaskInfo status : statuses) { + Assert.assertEquals(TaskState.RUNNING, status.getStatus().getStatusCode()); } } @Test(timeout = 60_000L) public void testDuplicateInsertThrowsEntryExistsException() { - final String entryId = "abcd"; - Map entry = ImmutableMap.of("a", 1); - Map status = ImmutableMap.of("count", 42); + Task entry = NoopTask.create(); + final String entryId = entry.getId(); + TaskStatus status = TaskStatus.running(entryId); handler.insert(entryId, DateTimes.of("2014-01-01"), "test", entry, true, status, "type", "group"); @@ -228,65 +209,33 @@ public void testDuplicateInsertThrowsEntryExistsException() () -> handler.insert(entryId, DateTimes.of("2014-01-01"), "test", entry, true, status, "type", "group") ); Assert.assertEquals("invalidInput", exception.getErrorCode()); - Assert.assertEquals("Task [abcd] already exists", exception.getMessage()); - } - - @Test - public void testAddLogThrowsUnsupportedException() - { - Exception exception = Assert.assertThrows( - DruidException.class, - () -> handler.addLog("abcd", ImmutableMap.of("logentry", "created")) - ); - Assert.assertEquals( - "Task actions are not logged anymore.", - exception.getMessage() - ); - } - - @Test - public void testGetLogsThrowsUnsupportedException() - { - Exception exception = Assert.assertThrows( - DruidException.class, - () -> handler.getLogs("abcd") - ); - Assert.assertEquals( - "Task actions are not logged anymore.", - exception.getMessage() - ); + Assert.assertEquals(StringUtils.format("Task [%s] already exists", entryId), exception.getMessage()); } @Test public void testLocks() { - final String entryId = "ABC123"; - Map entry = ImmutableMap.of("a", 1); - Map status = ImmutableMap.of("count", 42); + Task entry = NoopTask.create(); + final String entryId = entry.getId(); + TaskStatus status = TaskStatus.running(entryId); handler.insert(entryId, DateTimes.of("2014-01-01"), "test", entry, true, status, "type", "group"); - Assert.assertEquals( - ImmutableMap.>of(), - handler.getLocks("non_exist_entry") - ); + Assert.assertTrue(handler.getLocks("non_exist_entry").isEmpty()); - Assert.assertEquals( - ImmutableMap.>of(), - handler.getLocks(entryId) - ); + Assert.assertTrue(handler.getLocks(entryId).isEmpty()); - final ImmutableMap lock1 = ImmutableMap.of("lock", 1); - final ImmutableMap lock2 = ImmutableMap.of("lock", 2); + final TaskLock lock1 = createLock(entry); + final TaskLock lock2 = createLock(entry); Assert.assertTrue(handler.addLock(entryId, lock1)); Assert.assertTrue(handler.addLock(entryId, lock2)); - final Map> locks = handler.getLocks(entryId); + final Map locks = handler.getLocks(entryId); Assert.assertEquals(2, locks.size()); Assert.assertEquals( - ImmutableSet.>of(lock1, lock2), + Set.of(lock1, lock2), new HashSet<>(locks.values()) ); @@ -294,7 +243,7 @@ public void testLocks() handler.removeLock(lockId); locks.remove(lockId); - final Map> updated = handler.getLocks(entryId); + final Map updated = handler.getLocks(entryId); Assert.assertEquals( new HashSet<>(locks.values()), new HashSet<>(updated.values()) @@ -305,9 +254,9 @@ public void testLocks() @Test public void testReplaceLock() { - final String entryId = "ABC123"; - Map entry = ImmutableMap.of("a", 1); - Map status = ImmutableMap.of("count", 42); + Task entry = NoopTask.create(); + final String entryId = entry.getId(); + TaskStatus status = TaskStatus.running(entryId); handler.insert(entryId, DateTimes.of("2014-01-01"), "test", entry, true, status, "type", "group"); @@ -321,8 +270,8 @@ public void testReplaceLock() handler.getLocks(entryId) ); - final ImmutableMap lock1 = ImmutableMap.of("lock", 1); - final ImmutableMap lock2 = ImmutableMap.of("lock", 2); + final TaskLock lock1 = createLock(entry); + final TaskLock lock2 = createLock(entry); Assert.assertTrue(handler.addLock(entryId, lock1)); @@ -335,9 +284,9 @@ public void testReplaceLock() @Test public void testGetLockId() { - final String entryId = "ABC123"; - Map entry = ImmutableMap.of("a", 1); - Map status = ImmutableMap.of("count", 42); + Task entry = NoopTask.create(); + final String entryId = entry.getId(); + TaskStatus status = TaskStatus.running(entryId); handler.insert(entryId, DateTimes.of("2014-01-01"), "test", entry, true, status, "type", "group"); @@ -351,8 +300,8 @@ public void testGetLockId() handler.getLocks(entryId) ); - final ImmutableMap lock1 = ImmutableMap.of("lock", 1); - final ImmutableMap lock2 = ImmutableMap.of("lock", 2); + final TaskLock lock1 = createLock(entry); + final TaskLock lock2 = createLock(entry); Assert.assertTrue(handler.addLock(entryId, lock1)); @@ -363,19 +312,19 @@ public void testGetLockId() @Test public void testRemoveTasksOlderThan() { - final String entryId1 = "1234"; - Map entry1 = ImmutableMap.of("numericId", 1234); - Map status1 = ImmutableMap.of("count", 42, "temp", 1); + Task entry1 = NoopTask.create(); + final String entryId1 = entry1.getId(); + TaskStatus status1 = TaskStatus.running(entryId1); handler.insert(entryId1, DateTimes.of("2014-01-01T00:00:00.123"), "testDataSource", entry1, false, status1, "type", "group"); - final String entryId2 = "ABC123"; - Map entry2 = ImmutableMap.of("a", 1); - Map status2 = ImmutableMap.of("count", 42); + Task entry2 = NoopTask.create(); + final String entryId2 = entry2.getId(); + TaskStatus status2 = TaskStatus.running(entryId2); handler.insert(entryId2, DateTimes.of("2014-01-01T00:00:00.123"), "test", entry2, true, status2, "type", "group"); - final String entryId3 = "DEF5678"; - Map entry3 = ImmutableMap.of("numericId", 5678); - Map status3 = ImmutableMap.of("count", 21, "temp", 2); + Task entry3 = NoopTask.create(); + final String entryId3 = entry3.getId(); + TaskStatus status3 = TaskStatus.running(entryId2); handler.insert(entryId3, DateTimes.of("2014-01-02T12:00:00.123"), "testDataSource", entry3, false, status3, "type", "group"); Assert.assertEquals(Optional.of(entry1), handler.getEntry(entryId1)); @@ -440,23 +389,23 @@ public void testMigration() public void testGetTaskStatusPlusListInternal() { // SETUP - TaskInfo, Map> activeUnaltered = createRandomTaskInfo(TaskState.RUNNING); + TaskInfo activeUnaltered = createRandomTaskInfo(TaskState.RUNNING); insertTaskInfo(activeUnaltered, false); - TaskInfo, Map> completedUnaltered = createRandomTaskInfo(TaskState.SUCCESS); + TaskInfo completedUnaltered = createRandomTaskInfo(TaskState.SUCCESS); insertTaskInfo(completedUnaltered, false); - TaskInfo, Map> activeAltered = createRandomTaskInfo(TaskState.RUNNING); + TaskInfo activeAltered = createRandomTaskInfo(TaskState.RUNNING); insertTaskInfo(activeAltered, true); - TaskInfo, Map> completedAltered = createRandomTaskInfo(TaskState.SUCCESS); + TaskInfo completedAltered = createRandomTaskInfo(TaskState.SUCCESS); insertTaskInfo(completedAltered, true); Map taskLookups = new HashMap<>(); taskLookups.put(TaskLookup.TaskLookupType.ACTIVE, ActiveTaskLookup.getInstance()); taskLookups.put(TaskLookup.TaskLookupType.COMPLETE, CompleteTaskLookup.of(null, Duration.millis(86400000))); - List>> taskMetadataInfos; + List> taskMetadataInfos; // BEFORE MIGRATION @@ -511,25 +460,22 @@ private Integer getUnmigratedTaskCount() ); } - private TaskInfo, Map> createRandomTaskInfo(TaskState taskState) + private TaskInfo createRandomTaskInfo(TaskState taskState) { String id = UUID.randomUUID().toString(); DateTime createdTime = DateTime.now(DateTimeZone.UTC); String datasource = UUID.randomUUID().toString(); - String type = UUID.randomUUID().toString(); String groupId = UUID.randomUUID().toString(); - Map payload = new HashMap<>(); - payload.put("id", id); - payload.put("type", type); - payload.put("groupId", groupId); + Task payload = new NoopTask(id, groupId, datasource, 1L, 0L, null); - Map status = new HashMap<>(); - status.put("id", id); - status.put("status", taskState); - status.put("duration", RANDOM.nextLong()); - status.put("location", TaskLocation.create(UUID.randomUUID().toString(), 8080, 995)); - status.put("errorMsg", UUID.randomUUID().toString()); + TaskStatus status = new TaskStatus( + id, + taskState, + RANDOM.nextLong(), + UUID.randomUUID().toString(), + TaskLocation.create(UUID.randomUUID().toString(), 8080, 995) + ); return new TaskInfo<>( id, @@ -540,7 +486,20 @@ private TaskInfo, Map> createRandomTaskInfo( ); } - private void insertTaskInfo(TaskInfo, Map> taskInfo, boolean altered) + private TaskLock createLock(Task task) + { + final long intervalStart = RANDOM.nextLong(); + return new TimeChunkLock( + TaskLockType.APPEND, + task.getGroupId(), + task.getDataSource(), + Intervals.utc(intervalStart, intervalStart + 100), + "v1", + 1 + ); + } + + private void insertTaskInfo(TaskInfo taskInfo, boolean altered) { try { handler.insert( @@ -548,10 +507,10 @@ private void insertTaskInfo(TaskInfo, Map> t taskInfo.getCreatedTime(), taskInfo.getDataSource(), taskInfo.getTask(), - TaskState.RUNNING.equals(taskInfo.getStatus().get("status")), + TaskState.RUNNING.equals(taskInfo.getStatus().getStatusCode()), taskInfo.getStatus(), - altered ? taskInfo.getTask().get("type").toString() : null, - altered ? taskInfo.getTask().get("groupId").toString() : null + altered ? taskInfo.getTask().getType() : null, + altered ? taskInfo.getTask().getGroupId() : null ); } catch (Exception e) { @@ -559,11 +518,11 @@ private void insertTaskInfo(TaskInfo, Map> t } } - private void verifyTaskInfoToMetadataInfo(TaskInfo, Map> taskInfo, - List>> taskMetadataInfos, + private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, + List> taskMetadataInfos, boolean nullNewColumns) { - for (TaskInfo> taskMetadataInfo : taskMetadataInfos) { + for (TaskInfo taskMetadataInfo : taskMetadataInfos) { if (taskMetadataInfo.getId().equals(taskInfo.getId())) { verifyTaskInfoToMetadataInfo(taskInfo, taskMetadataInfo, nullNewColumns); } @@ -572,8 +531,8 @@ private void verifyTaskInfoToMetadataInfo(TaskInfo, Map, Map> taskInfo, - TaskInfo> taskMetadataInfo, + private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, + TaskInfo taskMetadataInfo, boolean nullNewColumns) { Assert.assertEquals(taskInfo.getId(), taskMetadataInfo.getId()); @@ -582,24 +541,20 @@ private void verifyTaskInfoToMetadataInfo(TaskInfo, Map task = taskInfo.getTask(); + Task task = taskInfo.getTask(); TaskIdentifier taskIdentifier = taskMetadataInfo.getTask(); - Assert.assertEquals(task.get("id"), taskIdentifier.getId()); + Assert.assertEquals(task.getId(), taskIdentifier.getId()); if (nullNewColumns) { Assert.assertNull(taskIdentifier.getGroupId()); Assert.assertNull(taskIdentifier.getType()); } else { - Assert.assertEquals(task.get("groupId"), taskIdentifier.getGroupId()); - Assert.assertEquals(task.get("type"), taskIdentifier.getType()); + Assert.assertEquals(task.getGroupId(), taskIdentifier.getGroupId()); + Assert.assertEquals(task.getType(), taskIdentifier.getType()); } } - private void verifyTaskStatus(Map expected, Map actual) + private void verifyTaskStatus(TaskStatus expected, TaskStatus actual) { - Assert.assertEquals(expected.get("id"), actual.get("id")); - Assert.assertEquals(expected.get("duration"), actual.get("duration")); - Assert.assertEquals(expected.get("errorMsg"), actual.get("errorMsg")); - Assert.assertEquals(expected.get("status").toString(), actual.get("status")); - Assert.assertEquals(expected.get("location"), JSON_MAPPER.convertValue(actual.get("location"), TaskLocation.class)); + Assert.assertEquals(expected, actual); } } diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java index b70c6894ba7e..fd6a62143488 100644 --- a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java +++ b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java @@ -23,9 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.StringUtils; -import java.util.HashMap; -import java.util.Map; - /** */ public class MetadataStorageTablesConfig @@ -41,10 +38,6 @@ public static MetadataStorageTablesConfig fromBase(String base) private static final String DEFAULT_BASE = "druid"; - private final Map entryTables = new HashMap<>(); - private final Map logTables = new HashMap<>(); - private final Map lockTables = new HashMap<>(); - @JsonProperty("base") private final String base; @@ -112,9 +105,6 @@ public MetadataStorageTablesConfig( this.tasksTable = makeTableName(tasksTable, "tasks"); this.taskLogTable = makeTableName(taskLogTable, "tasklogs"); this.taskLockTable = makeTableName(taskLockTable, "tasklocks"); - entryTables.put(TASK_ENTRY_TYPE, this.tasksTable); - logTables.put(TASK_ENTRY_TYPE, this.taskLogTable); - lockTables.put(TASK_ENTRY_TYPE, this.taskLockTable); this.auditTable = makeTableName(auditTable, "audit"); this.supervisorTable = makeTableName(supervisorTable, "supervisors"); this.segmentSchemasTable = makeTableName(segmentSchemasTable, "segmentSchemas"); @@ -167,26 +157,6 @@ public String getConfigTable() return configTable; } - public String getEntryTable(final String entryType) - { - return entryTables.get(entryType); - } - - public String getLogTable(final String entryType) - { - return logTables.get(entryType); - } - - public String getLockTable(final String entryType) - { - return lockTables.get(entryType); - } - - public String getTaskEntryType() - { - return TASK_ENTRY_TYPE; - } - public String getAuditTable() { return auditTable; diff --git a/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java b/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java index 6f49e94e4e9c..2df5643e16f4 100644 --- a/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java +++ b/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java @@ -69,15 +69,15 @@ public ObjectMapper jsonMapper() Assert.assertEquals(props.getProperty("druid.metadata.storage.tables.config"), config.getConfigTable()); Assert.assertEquals( props.getProperty("druid.metadata.storage.tables.tasks"), - config.getEntryTable(MetadataStorageTablesConfig.TASK_ENTRY_TYPE) + config.getTasksTable() ); Assert.assertEquals( props.getProperty("druid.metadata.storage.tables.taskLog"), - config.getLogTable(MetadataStorageTablesConfig.TASK_ENTRY_TYPE) + config.getTaskLogTable() ); Assert.assertEquals( props.getProperty("druid.metadata.storage.tables.taskLock"), - config.getLockTable(MetadataStorageTablesConfig.TASK_ENTRY_TYPE) + config.getTaskLockTable() ); Assert.assertEquals(props.getProperty("druid.metadata.storage.tables.dataSource"), config.getDataSourceTable()); Assert.assertEquals(props.getProperty("druid.metadata.storage.tables.supervisors"), config.getSupervisorTable()); diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java index 584a1b6d3584..48401430d7ac 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java @@ -539,7 +539,7 @@ private void alterPendingSegmentsTable(final String tableName) ); } - public void createLockTable(final String tableName, final String entryTypeName) + public void createLockTable(final String tableName) { createTable( tableName, @@ -547,13 +547,13 @@ public void createLockTable(final String tableName, final String entryTypeName) StringUtils.format( "CREATE TABLE %1$s (\n" + " id %2$s NOT NULL,\n" - + " %4$s_id VARCHAR(255) DEFAULT NULL,\n" + + " task_id VARCHAR(255) DEFAULT NULL,\n" + " lock_payload %3$s,\n" + " PRIMARY KEY (id)\n" + ")", - tableName, getSerialType(), getPayloadType(), entryTypeName + tableName, getSerialType(), getPayloadType() ), - StringUtils.format("CREATE INDEX idx_%1$s_%2$s_id ON %1$s(%2$s_id)", tableName, entryTypeName) + StringUtils.format("CREATE INDEX idx_%1$s_task_id ON %1$s(task_id)", tableName) ) ); } @@ -814,9 +814,8 @@ public void createTaskTables() { if (config.get().isCreateTables()) { final MetadataStorageTablesConfig tablesConfig = tablesConfigSupplier.get(); - final String entryType = tablesConfig.getTaskEntryType(); - prepareTaskEntryTable(tablesConfig.getEntryTable(entryType)); - createLockTable(tablesConfig.getLockTable(entryType), entryType); + prepareTaskEntryTable(tablesConfig.getTasksTable()); + createLockTable(tablesConfig.getTaskLockTable()); } } diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorSchemaPersistenceTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorSchemaPersistenceTest.java index 4caff1f1abfb..51904c1ddbd8 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorSchemaPersistenceTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorSchemaPersistenceTest.java @@ -50,13 +50,12 @@ public void setUp() public void testCreateTables() { final List tables = new ArrayList<>(); - final String entryType = tablesConfig.getTaskEntryType(); tables.add(tablesConfig.getConfigTable()); tables.add(tablesConfig.getSegmentSchemasTable()); tables.add(tablesConfig.getSegmentsTable()); tables.add(tablesConfig.getRulesTable()); - tables.add(tablesConfig.getLockTable(entryType)); - tables.add(tablesConfig.getEntryTable(entryType)); + tables.add(tablesConfig.getTaskLockTable()); + tables.add(tablesConfig.getTasksTable()); tables.add(tablesConfig.getAuditTable()); tables.add(tablesConfig.getSupervisorTable()); @@ -65,8 +64,8 @@ public void testCreateTables() dropSequence.add(tablesConfig.getSegmentsTable()); dropSequence.add(tablesConfig.getSegmentSchemasTable()); dropSequence.add(tablesConfig.getRulesTable()); - dropSequence.add(tablesConfig.getLockTable(entryType)); - dropSequence.add(tablesConfig.getEntryTable(entryType)); + dropSequence.add(tablesConfig.getTaskLockTable()); + dropSequence.add(tablesConfig.getTasksTable()); dropSequence.add(tablesConfig.getAuditTable()); dropSequence.add(tablesConfig.getSupervisorTable()); diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java index a40f95c31b92..be273f0fb688 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java @@ -70,12 +70,11 @@ public void setUp() public void testCreateTables() { final List tables = new ArrayList<>(); - final String entryType = tablesConfig.getTaskEntryType(); tables.add(tablesConfig.getConfigTable()); tables.add(tablesConfig.getSegmentsTable()); tables.add(tablesConfig.getRulesTable()); - tables.add(tablesConfig.getLockTable(entryType)); - tables.add(tablesConfig.getEntryTable(entryType)); + tables.add(tablesConfig.getTaskLockTable()); + tables.add(tablesConfig.getTasksTable()); tables.add(tablesConfig.getAuditTable()); tables.add(tablesConfig.getSupervisorTable()); @@ -115,8 +114,7 @@ public void testCreateTables() @Test public void testIndexCreationOnTaskTable() { - final String entryType = tablesConfig.getTaskEntryType(); - String entryTableName = tablesConfig.getEntryTable(entryType); + String entryTableName = tablesConfig.getTasksTable(); connector.createTaskTables(); Set createdIndexSet = connector.getIndexOnTable(entryTableName); Set expectedIndexSet = Sets.newHashSet( From 1ddf6e602c4dac094661f9efda8c8e0c3dfd74b9 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 2 Sep 2025 21:14:48 +0530 Subject: [PATCH 3/9] Simplify TaskInfo and add TaskIdStatus --- .../org/apache/druid/indexer/TaskInfo.java | 37 ++++----- .../druid/indexing/common/task/Task.java | 10 +-- .../overlord/HeapMemoryTaskStorage.java | 28 +++---- .../IndexerMetadataStorageAdapter.java | 6 +- .../overlord/MetadataTaskStorage.java | 16 ++-- .../indexing/overlord/TaskQueryTool.java | 10 +-- .../druid/indexing/overlord/TaskQueue.java | 30 +++---- .../druid/indexing/overlord/TaskStorage.java | 10 +-- .../overlord/http/OverlordResource.java | 7 +- .../MetadataStorageActionHandler.java | 10 +-- .../SQLMetadataStorageActionHandler.java | 37 +++++---- .../overlord/HeapMemoryTaskStorageTest.java | 5 +- .../IndexerMetadataStorageAdapterTest.java | 31 +++---- .../indexing/overlord/TaskQueueTest.java | 8 +- .../overlord/http/OverlordResourceTest.java | 18 ++--- .../MetadataStorageActionHandlerTest.java | 8 +- .../SQLMetadataStorageActionHandlerTest.java | 81 +++++++++---------- .../apache/druid/indexer/TaskIdStatus.java | 66 +++++++++++++++ .../apache/druid/indexer/TaskStatusPlus.java | 8 +- 19 files changed, 227 insertions(+), 199 deletions(-) rename {processing => indexing-service}/src/main/java/org/apache/druid/indexer/TaskInfo.java (64%) create mode 100644 processing/src/main/java/org/apache/druid/indexer/TaskIdStatus.java diff --git a/processing/src/main/java/org/apache/druid/indexer/TaskInfo.java b/indexing-service/src/main/java/org/apache/druid/indexer/TaskInfo.java similarity index 64% rename from processing/src/main/java/org/apache/druid/indexer/TaskInfo.java rename to indexing-service/src/main/java/org/apache/druid/indexer/TaskInfo.java index 2616f9daf02b..00f3cf320850 100644 --- a/processing/src/main/java/org/apache/druid/indexer/TaskInfo.java +++ b/indexing-service/src/main/java/org/apache/druid/indexer/TaskInfo.java @@ -20,40 +20,32 @@ package org.apache.druid.indexer; import com.google.common.base.Preconditions; +import org.apache.druid.indexing.common.task.Task; import org.joda.time.DateTime; -import javax.annotation.Nullable; - /** - * This class is used to store task info from runner query and cache in OverlordResource + * Contains the {@link TaskStatus} of a {@link Task} and its created time. */ -public class TaskInfo +public class TaskInfo { - private final String id; private final DateTime createdTime; - private final StatusType status; - private final String dataSource; - @Nullable - private final EntryType task; + private final TaskStatus status; + private final Task task; public TaskInfo( - String id, DateTime createdTime, - StatusType status, - String dataSource, - @Nullable EntryType task + TaskStatus status, + Task task ) { - this.id = Preconditions.checkNotNull(id, "id"); this.createdTime = Preconditions.checkNotNull(createdTime, "createdTime"); this.status = Preconditions.checkNotNull(status, "status"); - this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); - this.task = task; + this.task = Preconditions.checkNotNull(task, "Task cannot be null"); } public String getId() { - return id; + return task.getId(); } public DateTime getCreatedTime() @@ -61,18 +53,17 @@ public DateTime getCreatedTime() return createdTime; } - public StatusType getStatus() + public TaskStatus getStatus() { return status; } public String getDataSource() { - return dataSource; + return task.getDataSource(); } - @Nullable - public EntryType getTask() + public Task getTask() { return task; } @@ -80,8 +71,8 @@ public EntryType getTask() /** * Returns a copy of this TaskInfo object with the given status. */ - public TaskInfo withStatus(StatusType status) + public TaskInfo withStatus(TaskStatus status) { - return new TaskInfo<>(id, createdTime, status, dataSource, task); + return new TaskInfo(createdTime, status, task); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Task.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Task.java index 3cce9d8a6a54..ac556c86d488 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Task.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Task.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.indexer.TaskIdStatus; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskStatus; @@ -312,14 +313,13 @@ default TaskIdentifier getMetadata() return new TaskIdentifier(this.getId(), this.getGroupId(), this.getType()); } - static TaskInfo toTaskIdentifierInfo(TaskInfo taskInfo) + static TaskIdStatus toTaskIdentifierInfo(TaskInfo taskInfo) { - return new TaskInfo<>( - taskInfo.getId(), - taskInfo.getCreatedTime(), + return new TaskIdStatus( + taskInfo.getTask().getMetadata(), taskInfo.getStatus(), taskInfo.getDataSource(), - taskInfo.getTask().getMetadata() + taskInfo.getCreatedTime() ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java index 5f01a5fcc56a..cef49b9d0506 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java @@ -76,7 +76,7 @@ public HeapMemoryTaskStorage(TaskStorageConfig config) } @Override - public TaskInfo insert(Task task, TaskStatus status) + public TaskInfo insert(Task task, TaskStatus status) { Preconditions.checkNotNull(task, "task"); Preconditions.checkNotNull(status, "status"); @@ -137,7 +137,7 @@ public Optional getStatus(String taskid) @Nullable @Override - public TaskInfo getTaskInfo(String taskId) + public TaskInfo getTaskInfo(String taskId) { Preconditions.checkNotNull(taskId, "taskId"); final TaskStuff taskStuff = tasks.get(taskId); @@ -161,9 +161,9 @@ public List getActiveTasks() } @Override - public List> getActiveTaskInfos() + public List getActiveTaskInfos() { - final ImmutableList.Builder> listBuilder = ImmutableList.builder(); + final ImmutableList.Builder listBuilder = ImmutableList.builder(); for (final TaskStuff taskStuff : tasks.values()) { if (taskStuff.getStatus().isRunnable()) { listBuilder.add(TaskStuff.toTaskInfo(taskStuff)); @@ -184,9 +184,9 @@ public List getActiveTasksByDatasource(String datasource) return listBuilder.build(); } - public List> getActiveTaskInfo(@Nullable String dataSource) + public List getActiveTaskInfo(@Nullable String dataSource) { - final ImmutableList.Builder> listBuilder = ImmutableList.builder(); + final ImmutableList.Builder listBuilder = ImmutableList.builder(); for (final TaskStuff taskStuff : tasks.values()) { if (taskStuff.getStatus().isRunnable()) { if (dataSource == null || dataSource.equals(taskStuff.getDataSource())) { @@ -197,7 +197,7 @@ public List> getActiveTaskInfo(@Nullable String dataS return listBuilder.build(); } - public List> getRecentlyCreatedAlreadyFinishedTaskInfo(CompleteTaskLookup taskLookup) + public List getRecentlyCreatedAlreadyFinishedTaskInfo(CompleteTaskLookup taskLookup) { final Ordering createdDateDesc = new Ordering() { @@ -220,12 +220,12 @@ public int compare(TaskStuff a, TaskStuff b) * This method should be used only for testing. */ @Override - public List> getTaskInfos( + public List getTaskInfos( Map taskLookups, @Nullable String datasource ) { - final List> tasks = new ArrayList<>(); + final List tasks = new ArrayList<>(); final Map processedTaskLookups = TaskStorageUtils.processTaskLookups( taskLookups, @@ -254,7 +254,7 @@ public List getTaskStatusPlusList( .collect(Collectors.toList()); } - private List> getRecentlyCreatedAlreadyFinishedTaskInfoSince( + private List getRecentlyCreatedAlreadyFinishedTaskInfoSince( DateTime start, @Nullable Integer n, Ordering createdDateDesc @@ -268,7 +268,7 @@ private List> getRecentlyCreatedAlreadyFinishedTaskIn if (n != null) { stream = stream.limit(n); } - List> list = stream + List list = stream .map(TaskStuff::toTaskInfo) .collect(Collectors.toList()); return Collections.unmodifiableList(list); @@ -382,13 +382,11 @@ private TaskStuff withStatus(TaskStatus _status) return new TaskStuff(task, _status, createdDate, dataSource); } - static TaskInfo toTaskInfo(TaskStuff taskStuff) + static TaskInfo toTaskInfo(TaskStuff taskStuff) { - return new TaskInfo<>( - taskStuff.getTask().getId(), + return new TaskInfo( taskStuff.getCreatedDate(), taskStuff.getStatus(), - taskStuff.getDataSource(), taskStuff.getTask() ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapter.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapter.java index c3e498aa18f7..7a19859e6094 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapter.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapter.java @@ -22,8 +22,6 @@ import com.google.inject.Inject; import org.apache.druid.error.InvalidInput; import org.apache.druid.indexer.TaskInfo; -import org.apache.druid.indexer.TaskStatus; -import org.apache.druid.indexing.common.task.Task; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.metadata.TaskLookup; import org.joda.time.Interval; @@ -50,13 +48,13 @@ public int deletePendingSegments(String dataSource, Interval deleteInterval) { // Find the earliest active task created for the specified datasource; if one exists, // check if its interval overlaps with the delete interval. - final Optional> earliestActiveTaskOptional = taskStorage + final Optional earliestActiveTaskOptional = taskStorage .getTaskInfos(TaskLookup.activeTasksOnly(), dataSource) .stream() .min(Comparator.comparing(TaskInfo::getCreatedTime)); if (earliestActiveTaskOptional.isPresent()) { - final TaskInfo earliestActiveTask = earliestActiveTaskOptional.get(); + final TaskInfo earliestActiveTask = earliestActiveTaskOptional.get(); final Interval activeTaskInterval = new Interval( earliestActiveTask.getCreatedTime(), DateTimes.MAX diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java index 9ead5b2516c9..f646b552d19c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java @@ -111,7 +111,7 @@ public void stop() } @Override - public TaskInfo insert(final Task task, final TaskStatus status) + public TaskInfo insert(final Task task, final TaskStatus status) { Preconditions.checkNotNull(task, "task"); Preconditions.checkNotNull(status, "status"); @@ -144,11 +144,9 @@ public TaskInfo insert(final Task task, final TaskStatus statu throw new RuntimeException(e); } - return new TaskInfo<>( - task.getId(), + return new TaskInfo( insertionTime, status, - task.getDataSource(), task ); } @@ -181,7 +179,7 @@ public Optional getStatus(final String taskId) @Nullable @Override - public TaskInfo getTaskInfo(String taskId) + public TaskInfo getTaskInfo(String taskId) { return handler.getTaskInfo(taskId); } @@ -197,7 +195,7 @@ public List getActiveTasks() } @Override - public List> getActiveTaskInfos() + public List getActiveTaskInfos() { return handler.getTaskInfos(Map.of(TaskLookupType.ACTIVE, ActiveTaskLookup.getInstance()), null) .stream() @@ -208,12 +206,12 @@ public List> getActiveTaskInfos() @Override public List getActiveTasksByDatasource(String datasource) { - List> activeTaskInfos = handler.getTaskInfos( + List activeTaskInfos = handler.getTaskInfos( Collections.singletonMap(TaskLookupType.ACTIVE, ActiveTaskLookup.getInstance()), datasource ); ImmutableList.Builder tasksBuilder = ImmutableList.builder(); - for (TaskInfo taskInfo : activeTaskInfos) { + for (TaskInfo taskInfo : activeTaskInfos) { if (taskInfo.getStatus().isRunnable() && taskInfo.getTask() != null) { tasksBuilder.add(taskInfo.getTask()); } @@ -222,7 +220,7 @@ public List getActiveTasksByDatasource(String datasource) } @Override - public List> getTaskInfos( + public List getTaskInfos( Map taskLookups, @Nullable String datasource ) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java index d33c0a2769b7..77f9d25f607b 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueryTool.java @@ -102,7 +102,7 @@ public Map> getActiveLocks(List lockFil return taskLockbox.getActiveLocks(lockFilterPolicies); } - public List> getActiveTaskInfo(@Nullable String dataSource) + public List getActiveTaskInfo(@Nullable String dataSource) { return storage.getTaskInfos(TaskLookup.activeTasksOnly(), dataSource); } @@ -143,11 +143,11 @@ public Optional getTaskStatus(final String taskId) } @Nullable - public TaskInfo getTaskInfo(String taskId) + public TaskInfo getTaskInfo(String taskId) { final Optional taskQueue = taskMaster.getTaskQueue(); if (taskQueue.isPresent()) { - final Optional> taskStatus = taskQueue.get().getActiveTaskInfo(taskId); + final Optional taskStatus = taskQueue.get().getActiveTaskInfo(taskId); if (taskStatus.isPresent()) { return taskStatus.get(); } @@ -161,9 +161,9 @@ public List getAllActiveTasks() if (taskQueue.isPresent()) { // Serve active task statuses from memory final List taskStatusPlusList = new ArrayList<>(); - final List> activeTasks = taskQueue.get().getTaskInfos(); + final List activeTasks = taskQueue.get().getTaskInfos(); - for (TaskInfo taskInfo : activeTasks) { + for (TaskInfo taskInfo : activeTasks) { final Task task = taskInfo.getTask(); final Optional statusOptional = taskQueue.get().getTaskStatus(task.getId()); if (statusOptional.isPresent()) { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java index d7024eabfe5d..a7839533e5cb 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskQueue.java @@ -536,7 +536,7 @@ public boolean add(final Task task) // If this throws with any sort of exception, including TaskExistsException, we don't want to // insert the task into our queue. So don't catch it. final DateTime insertTime = DateTimes.nowUtc(); - final TaskInfo taskInfo = taskStorage.insert(task, TaskStatus.running(task.getId())); + final TaskInfo taskInfo = taskStorage.insert(task, TaskStatus.running(task.getId())); // Note: the TaskEntry created for this task doesn't actually use the `insertTime` timestamp, it uses a new // timestamp created in the ctor. This prevents races from occurring while syncFromStorage() is happening. addTaskInternal(taskInfo, insertTime); @@ -549,7 +549,7 @@ public boolean add(final Task task) } @GuardedBy("startStopLock") - private void addTaskInternal(final TaskInfo taskInfo, final DateTime updateTime) + private void addTaskInternal(final TaskInfo taskInfo, final DateTime updateTime) { final AtomicBoolean added = new AtomicBoolean(false); final TaskEntry entry = addOrUpdateTaskEntry( @@ -585,7 +585,7 @@ private void addTaskInternal(final TaskInfo taskInfo, final Da @GuardedBy("startStopLock") private boolean removeTaskInternal(final String taskId, final DateTime deleteTime) { - final AtomicReference> removedTask = new AtomicReference<>(); + final AtomicReference removedTask = new AtomicReference<>(); addOrUpdateTaskEntry( taskId, @@ -837,26 +837,26 @@ void syncFromStorage() try { if (active) { - final Map> newTasks = + final Map newTasks = CollectionUtils.toMap(taskStorage.getActiveTaskInfos(), TaskInfo::getId, Function.identity()); - final Map> oldTasks = + final Map oldTasks = CollectionUtils.mapValues(activeTasks, entry -> entry.taskInfo); // Identify the tasks that have been added or removed from the storage - final MapDifference> mapDifference = Maps.difference(oldTasks, newTasks); - final Collection> addedTasks = mapDifference.entriesOnlyOnRight().values(); - final Collection> removedTasks = mapDifference.entriesOnlyOnLeft().values(); + final MapDifference mapDifference = Maps.difference(oldTasks, newTasks); + final Collection addedTasks = mapDifference.entriesOnlyOnRight().values(); + final Collection removedTasks = mapDifference.entriesOnlyOnLeft().values(); // Remove tasks not present in metadata store if their lastUpdatedTime is before syncStartTime int numTasksRemoved = 0; - for (TaskInfo task : removedTasks) { + for (TaskInfo task : removedTasks) { if (removeTaskInternal(task.getId(), syncStartTime)) { ++numTasksRemoved; } } // Add new tasks present in metadata store if their lastUpdatedTime is before syncStartTime - for (TaskInfo task : addedTasks) { + for (TaskInfo task : addedTasks) { addTaskInternal(task, syncStartTime); } @@ -991,7 +991,7 @@ public CoordinatorRunStats getQueueStats() */ public Optional getActiveTask(String id) { - final Optional> taskInfo = getActiveTaskInfo(id); + final Optional taskInfo = getActiveTaskInfo(id); if (!taskInfo.isPresent()) { return Optional.absent(); } @@ -1016,7 +1016,7 @@ public Optional getActiveTask(String id) * Gets the {@link TaskInfo} for the given {@code taskId} from {@link #activeTasks} if present, * otherwise returns an empty optional. */ - public Optional> getActiveTaskInfo(String taskId) + public Optional getActiveTaskInfo(String taskId) { final TaskEntry entry = activeTasks.get(taskId); return entry == null ? Optional.absent() : Optional.of(entry.taskInfo); @@ -1025,7 +1025,7 @@ public Optional> getActiveTaskInfo(String taskId) /** * List of all active and completed task infos currently being managed by this TaskQueue. */ - public List> getTaskInfos() + public List getTaskInfos() { return activeTasks.values().stream().map(entry -> entry.taskInfo).collect(Collectors.toList()); } @@ -1148,13 +1148,13 @@ private void removeTaskLock(Task task) */ static class TaskEntry { - private TaskInfo taskInfo; + private TaskInfo taskInfo; private DateTime lastUpdatedTime; private ListenableFuture future = null; private boolean isComplete = false; - TaskEntry(TaskInfo taskInfo) + TaskEntry(TaskInfo taskInfo) { this.taskInfo = taskInfo; this.lastUpdatedTime = DateTimes.nowUtc(); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskStorage.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskStorage.java index 32f1a9372cb1..49a248ec4b8b 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskStorage.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskStorage.java @@ -42,7 +42,7 @@ public interface TaskStorage * @param status task status * @return a TaskInfo object representing the task information that was committed to the storage facility */ - TaskInfo insert(Task task, TaskStatus status); + TaskInfo insert(Task task, TaskStatus status); /** * Persists task status in the storage facility. This method should throw an exception if the task status lifecycle @@ -108,7 +108,7 @@ public interface TaskStorage Optional getStatus(String taskid); @Nullable - TaskInfo getTaskInfo(String taskId); + TaskInfo getTaskInfo(String taskId); /** * Returns a list of currently running or pending tasks as stored in the storage facility. No particular order @@ -124,7 +124,7 @@ public interface TaskStorage * * @return list of active task infos */ - List> getActiveTaskInfos(); + List getActiveTaskInfos(); /** * Returns a list of currently running or pending tasks as stored in the storage facility. No particular order @@ -161,12 +161,12 @@ List getTaskStatusPlusList( * @param taskLookups lookup types and filters * @param datasource datasource filter */ - List> getTaskInfos( + List getTaskInfos( Map taskLookups, @Nullable String datasource ); - default List> getTaskInfos( + default List getTaskInfos( TaskLookup taskLookup, @Nullable String datasource ) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java index 9da2db68dc50..c33234701336 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java @@ -39,7 +39,6 @@ import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskLocation; -import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.actions.TaskActionHolder; @@ -291,7 +290,7 @@ public Response getTaskPayload(@PathParam("taskid") String taskid) @ResourceFilters(TaskResourceFilter.class) public Response getTaskStatus(@PathParam("taskid") String taskid) { - final TaskInfo taskInfo = taskQueryTool.getTaskInfo(taskid); + final TaskInfo taskInfo = taskQueryTool.getTaskInfo(taskid); TaskStatusResponse response = null; if (taskInfo != null) { @@ -395,11 +394,11 @@ public Response shutdownTasksForDataSource(@PathParam("dataSource") final String return asLeaderWith( taskMaster.getTaskQueue(), taskQueue -> { - final List> tasks = taskQueryTool.getActiveTaskInfo(dataSource); + final List tasks = taskQueryTool.getActiveTaskInfo(dataSource); if (tasks.isEmpty()) { return Response.status(Status.NOT_FOUND).build(); } else { - for (final TaskInfo task : tasks) { + for (final TaskInfo task : tasks) { taskQueue.shutdown(task.getId(), "Shutdown request from user"); } return Response.ok(ImmutableMap.of("dataSource", dataSource)).build(); diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java index daf574acf02b..a70d2ba8f012 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java @@ -21,7 +21,7 @@ import com.google.common.base.Optional; import org.apache.druid.guice.annotations.ExtensionPoint; -import org.apache.druid.indexer.TaskIdentifier; +import org.apache.druid.indexer.TaskIdStatus; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TaskLock; @@ -89,7 +89,7 @@ void insert( Optional getStatus(String entryId); @Nullable - TaskInfo getTaskInfo(String entryId); + TaskInfo getTaskInfo(String entryId); /** * Returns a list of {@link TaskInfo} from metadata store that matches to the given filters. @@ -102,7 +102,7 @@ void insert( * @param taskLookups task lookup type and filters. * @param datasource datasource filter */ - List> getTaskInfos( + List getTaskInfos( Map taskLookups, @Nullable String datasource ); @@ -118,12 +118,12 @@ List> getTaskInfos( * @param taskLookups task lookup type and filters. * @param datasource datasource filter */ - List> getTaskStatusList( + List getTaskStatusList( Map taskLookups, @Nullable String datasource ); - default List> getTaskInfos( + default List getTaskInfos( TaskLookup taskLookup, @Nullable String datasource ) diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java index cf8bcf6e6586..739011152d91 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java @@ -29,6 +29,7 @@ import com.google.common.collect.Maps; import org.apache.druid.error.DruidException; import org.apache.druid.error.InvalidInput; +import org.apache.druid.indexer.TaskIdStatus; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskStatus; @@ -275,7 +276,7 @@ public Optional withHandle(Handle handle) throws Exception @Override @Nullable - public TaskInfo getTaskInfo(String entryId) + public TaskInfo getTaskInfo(String entryId) { return connector.retryWithHandle(handle -> { final String query = StringUtils.format( @@ -290,14 +291,14 @@ public TaskInfo getTaskInfo(String entryId) } @Override - public List> getTaskInfos( + public List getTaskInfos( Map taskLookups, @Nullable String dataSource ) { return getConnector().retryTransaction( (handle, status) -> { - final List> tasks = new ArrayList<>(); + final List tasks = new ArrayList<>(); for (Entry entry : taskLookups.entrySet()) { final Query> query; switch (entry.getKey()) { @@ -327,7 +328,7 @@ public List> getTaskInfos( } @Override - public List> getTaskStatusList( + public List getTaskStatusList( Map taskLookups, @Nullable String dataSource ) @@ -345,17 +346,17 @@ public List> getTaskStatusList( } @VisibleForTesting - List> getTaskStatusList( + List getTaskStatusList( Map taskLookups, @Nullable String dataSource, boolean fetchPayload ) { - ResultSetMapper> resultSetMapper = + ResultSetMapper resultSetMapper = fetchPayload ? taskStatusMapperFromPayload : taskStatusMapper; return getConnector().retryTransaction( (handle, status) -> { - final List> taskMetadataInfos = new ArrayList<>(); + final List taskMetadataInfos = new ArrayList<>(); for (Entry entry : taskLookups.entrySet()) { final Query> query; switch (entry.getKey()) { @@ -592,7 +593,7 @@ private String getWhereClauseForActiveStatusesQuery(String dataSource) return sql; } - private class TaskStatusMapperFromPayload implements ResultSetMapper> + private class TaskStatusMapperFromPayload implements ResultSetMapper { private final ObjectMapper objectMapper; @@ -602,14 +603,14 @@ private class TaskStatusMapperFromPayload implements ResultSetMapper map(int index, ResultSet resultSet, StatementContext context) + public TaskIdStatus map(int index, ResultSet resultSet, StatementContext context) throws SQLException { return toTaskIdentifierInfo(objectMapper, resultSet, true); } } - private class TaskStatusMapper implements ResultSetMapper> + private class TaskStatusMapper implements ResultSetMapper { private final ObjectMapper objectMapper; @@ -619,14 +620,14 @@ private class TaskStatusMapper implements ResultSetMapper map(int index, ResultSet resultSet, StatementContext context) + public TaskIdStatus map(int index, ResultSet resultSet, StatementContext context) throws SQLException { return toTaskIdentifierInfo(objectMapper, resultSet, false); } } - private TaskInfo toTaskIdentifierInfo(ObjectMapper objectMapper, + private TaskIdStatus toTaskIdentifierInfo(ObjectMapper objectMapper, ResultSet resultSet, boolean usePayload ) throws SQLException @@ -661,7 +662,7 @@ private TaskInfo toTaskIdentifierInfo(ObjectMapper o String datasource = resultSet.getString("datasource"); TaskIdentifier taskIdentifier = new TaskIdentifier(id, groupId, type); - return new TaskInfo<>(id, createdTime, status, datasource, taskIdentifier); + return new TaskIdStatus(taskIdentifier, status, datasource, createdTime); } static class TaskIdentifierMapper implements ResultSetMapper @@ -695,7 +696,7 @@ public TaskIdentifier map(int index, ResultSet resultSet, StatementContext conte } } - static class TaskInfoMapper implements ResultSetMapper> + static class TaskInfoMapper implements ResultSetMapper { private final ObjectMapper objectMapper; @@ -705,10 +706,10 @@ static class TaskInfoMapper implements ResultSetMapper map(int index, ResultSet resultSet, StatementContext context) + public TaskInfo map(int index, ResultSet resultSet, StatementContext context) throws SQLException { - final TaskInfo taskInfo; + final TaskInfo taskInfo; Task task; TaskStatus status; try { @@ -725,11 +726,9 @@ public TaskInfo map(int index, ResultSet resultSet, StatementC log.error(e, "Encountered exception while deserializing task status_payload"); throw new SQLException(e); } - taskInfo = new TaskInfo<>( - resultSet.getString("id"), + taskInfo = new TaskInfo( DateTimes.of(resultSet.getString("created_date")), status, - resultSet.getString("datasource"), task ); return taskInfo; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java index d300ceefb137..7021d23fb50a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorageTest.java @@ -24,7 +24,6 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.config.TaskStorageConfig; import org.apache.druid.indexing.common.task.NoopTask; -import org.apache.druid.indexing.common.task.Task; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.metadata.TaskLookup; import org.joda.time.Period; @@ -70,7 +69,7 @@ public void testGetTaskInfos() storage.insert(task2, TaskStatus.running(task2.getId())); // Active statuses - final List> taskInfosActive = storage.getTaskInfos( + final List taskInfosActive = storage.getTaskInfos( ImmutableMap.of( TaskLookup.TaskLookupType.ACTIVE, TaskLookup.ActiveTaskLookup.getInstance(), @@ -84,7 +83,7 @@ public void testGetTaskInfos() Assert.assertEquals(task2.getId(), taskInfosActive.get(0).getTask().getId()); // Complete statuses - final List> taskInfosComplete = storage.getTaskInfos( + final List taskInfosComplete = storage.getTaskInfos( ImmutableMap.of( TaskLookup.TaskLookupType.COMPLETE, new TaskLookup.CompleteTaskLookup(null, DateTimes.of("1970")) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java index 5226c9d735b8..7aad62310b11 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java @@ -25,7 +25,6 @@ import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.task.NoopTask; -import org.apache.druid.indexing.common.task.Task; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.metadata.TaskLookup; @@ -58,19 +57,15 @@ public void setup() @Test public void testDeletePendingSegments() { - final List> taskInfos = ImmutableList.of( - new TaskInfo<>( - "id1", + final List taskInfos = ImmutableList.of( + new TaskInfo( DateTimes.of("2017-12-01"), TaskStatus.running("id1"), - "dataSource", NoopTask.create() ), - new TaskInfo<>( - "id2", + new TaskInfo( DateTimes.of("2017-12-02"), TaskStatus.running("id2"), - "dataSource", NoopTask.create() ) ); @@ -93,19 +88,15 @@ public void testDeletePendingSegments() @Test public void testDeletePendingSegmentsOfOneOverlappingRunningTask() { - final ImmutableList> taskInfos = ImmutableList.of( - new TaskInfo<>( - "id1", + final ImmutableList taskInfos = ImmutableList.of( + new TaskInfo( DateTimes.of("2017-11-01"), TaskStatus.running("id1"), - "dataSource", NoopTask.create() ), - new TaskInfo<>( - "id2", + new TaskInfo( DateTimes.of("2017-12-02"), TaskStatus.running("id2"), - "dataSource", NoopTask.create() ) ); @@ -140,19 +131,15 @@ public void testDeletePendingSegmentsOfOneOverlappingRunningTask() @Test public void testDeletePendingSegmentsOfMultipleOverlappingRunningTasks() { - final ImmutableList> taskInfos = ImmutableList.of( - new TaskInfo<>( - "id1", + final ImmutableList taskInfos = ImmutableList.of( + new TaskInfo( DateTimes.of("2017-12-01"), TaskStatus.running("id1"), - "dataSource", NoopTask.create() ), - new TaskInfo<>( - "id2", + new TaskInfo( DateTimes.of("2017-11-01"), TaskStatus.running("id2"), - "dataSource", NoopTask.create() ) ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java index 0cb35ba39175..3ee1fa065b28 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java @@ -633,13 +633,13 @@ public void testTaskShutdownUpdatesTaskStatusInTaskQueue() final TestTask task = new TestTask("shutdown-test-task", Intervals.of("2021-01-01/P1D")); taskQueue.add(task); - final Optional> activeInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); + final Optional activeInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); Assert.assertTrue(activeInfoOpt.isPresent()); Assert.assertEquals(TaskState.RUNNING, activeInfoOpt.get().getStatus().getStatusCode()); taskQueue.shutdown(task.getId(), shutdownReason); - final Optional> afterShutdownInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); + final Optional afterShutdownInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); Assert.assertTrue(afterShutdownInfoOpt.isPresent()); Assert.assertEquals(shutdownStatus, afterShutdownInfoOpt.get().getStatus()); Assert.assertEquals(shutdownStatus, getTaskStorage().getStatus(task.getId()).get()); @@ -657,7 +657,7 @@ public void testTaskSuccessUpdatesTaskStatusInTaskQueue() throws Exception Thread.sleep(100); Assert.assertTrue(task.isDone()); - final Optional> activeInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); + final Optional activeInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); Assert.assertTrue(activeInfoOpt.isPresent()); Assert.assertEquals(successStatus, activeInfoOpt.get().getStatus()); Assert.assertEquals(successStatus, getTaskStorage().getStatus(task.getId()).get()); @@ -684,7 +684,7 @@ public TaskStatus runTask(TaskToolbox toolbox) Thread.sleep(100); Assert.assertTrue(task.isDone()); - final Optional> activeInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); + final Optional activeInfoOpt = taskQueue.getActiveTaskInfo(task.getId()); Assert.assertTrue(activeInfoOpt.isPresent()); Assert.assertEquals(failedStatus, activeInfoOpt.get().getStatus()); Assert.assertEquals(failedStatus, getTaskStorage().getStatus(task.getId()).get()); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java index a6988feeeb1f..e52255e5c703 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordResourceTest.java @@ -999,11 +999,9 @@ public void testGetTaskStatus() throws Exception final Task task = NoopTask.create(); final String taskId = task.getId(); final TaskStatus status = TaskStatus.running(taskId); - final TaskInfo taskInfo = new TaskInfo<>( - task.getId(), + final TaskInfo taskInfo = new TaskInfo( DateTimes.of("2018-01-01"), status, - task.getDataSource(), task ); @@ -1194,19 +1192,15 @@ public void testShutdownAllTasks() EasyMock.expect( taskStorage.getTaskInfos(TaskLookup.activeTasksOnly(), "datasource") ).andStubReturn(ImmutableList.of( - new TaskInfo<>( - "id_1", + new TaskInfo( DateTime.now(ISOChronology.getInstanceUTC()), TaskStatus.success("id_1"), - "datasource", - NoopTask.create() + new NoopTask("id_1", null, "datasource", 1L, 0L, null) ), - new TaskInfo<>( - "id_2", + new TaskInfo( DateTime.now(ISOChronology.getInstanceUTC()), TaskStatus.success("id_2"), - "datasource", - NoopTask.create() + new NoopTask("id_2", null, "datasource", 1L, 0L, null) ) )); mockQueue.shutdown("id_1", "Shutdown request from user"); @@ -1217,7 +1211,7 @@ public void testShutdownAllTasks() replayAll(); EasyMock.replay(mockQueue); - final Map response = (Map) overlordResource + final Map response = (Map) overlordResource .shutdownTasksForDataSource("datasource") .getEntity(); Assert.assertEquals("datasource", response.get("dataSource")); diff --git a/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java b/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java index 014fe3da2e97..88db9e86f7ea 100644 --- a/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/metadata/MetadataStorageActionHandlerTest.java @@ -20,7 +20,7 @@ package org.apache.druid.metadata; import com.google.common.base.Optional; -import org.apache.druid.indexer.TaskIdentifier; +import org.apache.druid.indexer.TaskIdStatus; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TaskLock; @@ -84,13 +84,13 @@ public Optional getStatus(String entryId) @Nullable @Override - public TaskInfo getTaskInfo(String entryId) + public TaskInfo getTaskInfo(String entryId) { return null; } @Override - public List> getTaskInfos( + public List getTaskInfos( Map taskLookups, @Nullable String datasource ) @@ -99,7 +99,7 @@ public List> getTaskInfos( } @Override - public List> getTaskStatusList( + public List getTaskStatusList( Map taskLookups, @Nullable String datasource ) diff --git a/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java b/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java index 1fc9a3e656d4..604f682fb66a 100644 --- a/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/metadata/SQLMetadataStorageActionHandlerTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.error.DruidException; +import org.apache.druid.indexer.TaskIdStatus; import org.apache.druid.indexer.TaskIdentifier; import org.apache.druid.indexer.TaskInfo; import org.apache.druid.indexer.TaskLocation; @@ -164,12 +165,12 @@ public void testGetRecentStatuses() handler.insert(entryId, DateTimes.of(StringUtils.format("2014-01-%02d", i)), "test", entry, false, status, "type", "group"); } - final List> statuses = handler.getTaskInfos( + final List statuses = handler.getTaskInfos( CompleteTaskLookup.withTasksCreatedPriorTo(7, DateTimes.of("2014-01-01")), null ); Assert.assertEquals(7, statuses.size()); - for (TaskInfo status : statuses) { + for (TaskInfo status : statuses) { Assert.assertEquals(TaskState.RUNNING, status.getStatus().getStatusCode()); } } @@ -185,12 +186,12 @@ public void testGetRecentStatuses2() handler.insert(entryId, DateTimes.of(StringUtils.format("2014-01-%02d", i)), "test", entry, false, status, "type", "group"); } - final List> statuses = handler.getTaskInfos( + final List statuses = handler.getTaskInfos( CompleteTaskLookup.withTasksCreatedPriorTo(10, DateTimes.of("2014-01-01")), null ); Assert.assertEquals(5, statuses.size()); - for (TaskInfo status : statuses) { + for (TaskInfo status : statuses) { Assert.assertEquals(TaskState.RUNNING, status.getStatus().getStatusCode()); } } @@ -225,8 +226,8 @@ public void testLocks() Assert.assertTrue(handler.getLocks(entryId).isEmpty()); - final TaskLock lock1 = createLock(entry); - final TaskLock lock2 = createLock(entry); + final TaskLock lock1 = createRandomLock(entry); + final TaskLock lock2 = createRandomLock(entry); Assert.assertTrue(handler.addLock(entryId, lock1)); Assert.assertTrue(handler.addLock(entryId, lock2)); @@ -270,8 +271,8 @@ public void testReplaceLock() handler.getLocks(entryId) ); - final TaskLock lock1 = createLock(entry); - final TaskLock lock2 = createLock(entry); + final TaskLock lock1 = createRandomLock(entry); + final TaskLock lock2 = createRandomLock(entry); Assert.assertTrue(handler.addLock(entryId, lock1)); @@ -300,8 +301,8 @@ public void testGetLockId() handler.getLocks(entryId) ); - final TaskLock lock1 = createLock(entry); - final TaskLock lock2 = createLock(entry); + final TaskLock lock1 = createRandomLock(entry); + final TaskLock lock2 = createRandomLock(entry); Assert.assertTrue(handler.addLock(entryId, lock1)); @@ -389,23 +390,23 @@ public void testMigration() public void testGetTaskStatusPlusListInternal() { // SETUP - TaskInfo activeUnaltered = createRandomTaskInfo(TaskState.RUNNING); + TaskInfo activeUnaltered = createRandomTaskInfo(TaskState.RUNNING); insertTaskInfo(activeUnaltered, false); - TaskInfo completedUnaltered = createRandomTaskInfo(TaskState.SUCCESS); + TaskInfo completedUnaltered = createRandomTaskInfo(TaskState.SUCCESS); insertTaskInfo(completedUnaltered, false); - TaskInfo activeAltered = createRandomTaskInfo(TaskState.RUNNING); + TaskInfo activeAltered = createRandomTaskInfo(TaskState.RUNNING); insertTaskInfo(activeAltered, true); - TaskInfo completedAltered = createRandomTaskInfo(TaskState.SUCCESS); + TaskInfo completedAltered = createRandomTaskInfo(TaskState.SUCCESS); insertTaskInfo(completedAltered, true); Map taskLookups = new HashMap<>(); taskLookups.put(TaskLookup.TaskLookupType.ACTIVE, ActiveTaskLookup.getInstance()); taskLookups.put(TaskLookup.TaskLookupType.COMPLETE, CompleteTaskLookup.of(null, Duration.millis(86400000))); - List> taskMetadataInfos; + List taskMetadataInfos; // BEFORE MIGRATION @@ -460,7 +461,20 @@ private Integer getUnmigratedTaskCount() ); } - private TaskInfo createRandomTaskInfo(TaskState taskState) + private TaskLock createRandomLock(Task task) + { + final long intervalStart = RANDOM.nextLong(); + return new TimeChunkLock( + TaskLockType.APPEND, + task.getGroupId(), + task.getDataSource(), + Intervals.utc(intervalStart, intervalStart + 100), + "v1", + 1 + ); + } + + private TaskInfo createRandomTaskInfo(TaskState taskState) { String id = UUID.randomUUID().toString(); DateTime createdTime = DateTime.now(DateTimeZone.UTC); @@ -477,29 +491,14 @@ private TaskInfo createRandomTaskInfo(TaskState taskState) TaskLocation.create(UUID.randomUUID().toString(), 8080, 995) ); - return new TaskInfo<>( - id, + return new TaskInfo( createdTime, status, - datasource, payload ); } - private TaskLock createLock(Task task) - { - final long intervalStart = RANDOM.nextLong(); - return new TimeChunkLock( - TaskLockType.APPEND, - task.getGroupId(), - task.getDataSource(), - Intervals.utc(intervalStart, intervalStart + 100), - "v1", - 1 - ); - } - - private void insertTaskInfo(TaskInfo taskInfo, boolean altered) + private void insertTaskInfo(TaskInfo taskInfo, boolean altered) { try { handler.insert( @@ -518,12 +517,12 @@ private void insertTaskInfo(TaskInfo taskInfo, boolean altered } } - private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, - List> taskMetadataInfos, + private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, + List taskMetadataInfos, boolean nullNewColumns) { - for (TaskInfo taskMetadataInfo : taskMetadataInfos) { - if (taskMetadataInfo.getId().equals(taskInfo.getId())) { + for (TaskIdStatus taskMetadataInfo : taskMetadataInfos) { + if (taskMetadataInfo.getTaskIdentifier().getId().equals(taskInfo.getId())) { verifyTaskInfoToMetadataInfo(taskInfo, taskMetadataInfo, nullNewColumns); } return; @@ -531,18 +530,18 @@ private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, Assert.fail(); } - private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, - TaskInfo taskMetadataInfo, + private void verifyTaskInfoToMetadataInfo(TaskInfo taskInfo, + TaskIdStatus taskMetadataInfo, boolean nullNewColumns) { - Assert.assertEquals(taskInfo.getId(), taskMetadataInfo.getId()); + Assert.assertEquals(taskInfo.getId(), taskMetadataInfo.getTaskIdentifier().getId()); Assert.assertEquals(taskInfo.getCreatedTime(), taskMetadataInfo.getCreatedTime()); Assert.assertEquals(taskInfo.getDataSource(), taskMetadataInfo.getDataSource()); verifyTaskStatus(taskInfo.getStatus(), taskMetadataInfo.getStatus()); Task task = taskInfo.getTask(); - TaskIdentifier taskIdentifier = taskMetadataInfo.getTask(); + TaskIdentifier taskIdentifier = taskMetadataInfo.getTaskIdentifier(); Assert.assertEquals(task.getId(), taskIdentifier.getId()); if (nullNewColumns) { Assert.assertNull(taskIdentifier.getGroupId()); diff --git a/processing/src/main/java/org/apache/druid/indexer/TaskIdStatus.java b/processing/src/main/java/org/apache/druid/indexer/TaskIdStatus.java new file mode 100644 index 000000000000..727eb812ff2c --- /dev/null +++ b/processing/src/main/java/org/apache/druid/indexer/TaskIdStatus.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexer; + +import org.joda.time.DateTime; + +/** + * Contains a {@link TaskIdentifier} and its {@link TaskStatus}. + */ +public class TaskIdStatus +{ + private final TaskIdentifier taskId; + private final TaskStatus taskStatus; + private final String dataSource; + private final DateTime createdTime; + + public TaskIdStatus( + TaskIdentifier taskId, + TaskStatus taskStatus, + String dataSource, + DateTime createdTime + ) + { + this.taskId = taskId; + this.taskStatus = taskStatus; + this.dataSource = dataSource; + this.createdTime = createdTime; + } + + public TaskIdentifier getTaskIdentifier() + { + return taskId; + } + + public TaskStatus getStatus() + { + return taskStatus; + } + + public String getDataSource() + { + return dataSource; + } + + public DateTime getCreatedTime() + { + return createdTime; + } +} diff --git a/processing/src/main/java/org/apache/druid/indexer/TaskStatusPlus.java b/processing/src/main/java/org/apache/druid/indexer/TaskStatusPlus.java index af04db211138..017fa401cadb 100644 --- a/processing/src/main/java/org/apache/druid/indexer/TaskStatusPlus.java +++ b/processing/src/main/java/org/apache/druid/indexer/TaskStatusPlus.java @@ -262,13 +262,13 @@ public String toString() * @param taskIdentifierInfo TaskInfo pair * @return corresponding TaskStatusPlus */ - public static TaskStatusPlus fromTaskIdentifierInfo(TaskInfo taskIdentifierInfo) + public static TaskStatusPlus fromTaskIdentifierInfo(TaskIdStatus taskIdentifierInfo) { TaskStatus status = taskIdentifierInfo.getStatus(); return new TaskStatusPlus( - taskIdentifierInfo.getId(), - taskIdentifierInfo.getTask().getGroupId(), - taskIdentifierInfo.getTask().getType(), + taskIdentifierInfo.getTaskIdentifier().getId(), + taskIdentifierInfo.getTaskIdentifier().getGroupId(), + taskIdentifierInfo.getTaskIdentifier().getType(), taskIdentifierInfo.getCreatedTime(), DateTimes.EPOCH, status.getStatusCode(), From bce2fa68278bd4a24ef9fbab696199a623460365 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 2 Sep 2025 22:57:31 +0530 Subject: [PATCH 4/9] Fix tests --- .../overlord/MetadataTaskStorage.java | 39 +++---------------- .../SQLMetadataStorageActionHandler.java | 24 ++++++------ .../IndexerMetadataStorageAdapterTest.java | 8 ++-- .../org/apache/druid/cli/CliOverlordTest.java | 5 ++- 4 files changed, 26 insertions(+), 50 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java index f646b552d19c..9dba47c62780 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/MetadataTaskStorage.java @@ -19,7 +19,6 @@ package org.apache.druid.indexing.overlord; -import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -30,7 +29,6 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.actions.TaskAction; import org.apache.druid.indexing.common.config.TaskStorageConfig; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.java.util.common.DateTimes; @@ -40,7 +38,6 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.metadata.MetadataStorageActionHandler; import org.apache.druid.metadata.MetadataStorageActionHandlerFactory; -import org.apache.druid.metadata.MetadataStorageActionHandlerTypes; import org.apache.druid.metadata.MetadataStorageConnector; import org.apache.druid.metadata.TaskLookup; import org.apache.druid.metadata.TaskLookup.ActiveTaskLookup; @@ -56,28 +53,6 @@ public class MetadataTaskStorage implements TaskStorage { - - private static final MetadataStorageActionHandlerTypes TASK_TYPES = new MetadataStorageActionHandlerTypes<>() - { - @Override - public TypeReference getEntryType() - { - return new TypeReference<>() {}; - } - - @Override - public TypeReference getStatusType() - { - return new TypeReference<>() {}; - } - - @Override - public TypeReference getLockType() - { - return new TypeReference<>() {}; - } - }; - private final MetadataStorageConnector metadataStorageConnector; private final TaskStorageConfig config; private final MetadataStorageActionHandler handler; @@ -199,7 +174,7 @@ public List getActiveTaskInfos() { return handler.getTaskInfos(Map.of(TaskLookupType.ACTIVE, ActiveTaskLookup.getInstance()), null) .stream() - .filter(taskInfo -> taskInfo.getStatus().isRunnable() && taskInfo.getTask() != null) + .filter(taskInfo -> taskInfo.getStatus().isRunnable()) .collect(Collectors.toList()); } @@ -212,7 +187,7 @@ public List getActiveTasksByDatasource(String datasource) ); ImmutableList.Builder tasksBuilder = ImmutableList.builder(); for (TaskInfo taskInfo : activeTaskInfos) { - if (taskInfo.getStatus().isRunnable() && taskInfo.getTask() != null) { + if (taskInfo.getStatus().isRunnable()) { tasksBuilder.add(taskInfo.getTask()); } } @@ -245,12 +220,10 @@ public List getTaskStatusPlusList( DateTimes.nowUtc().minus(config.getRecentlyFinishedThreshold()) ); - return Collections.unmodifiableList( - handler.getTaskStatusList(processedTaskLookups, datasource) - .stream() - .map(TaskStatusPlus::fromTaskIdentifierInfo) - .collect(Collectors.toList()) - ); + return handler.getTaskStatusList(processedTaskLookups, datasource) + .stream() + .map(TaskStatusPlus::fromTaskIdentifierInfo) + .collect(Collectors.toList()); } @Override diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java index 739011152d91..b7190edf6518 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/SQLMetadataStorageActionHandler.java @@ -61,9 +61,11 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.stream.Collectors; import java.util.stream.IntStream; public abstract class SQLMetadataStorageActionHandler @@ -320,7 +322,7 @@ public List getTaskInfos( throw new IAE("Unknown TaskLookupType: [%s]", entry.getKey()); } } - return tasks; + return tasks.stream().filter(Objects::nonNull).collect(Collectors.toList()); }, SQLMetadataConnector.QUIET_RETRIES, SQLMetadataConnector.DEFAULT_MAX_TRIES @@ -696,7 +698,7 @@ public TaskIdentifier map(int index, ResultSet resultSet, StatementContext conte } } - static class TaskInfoMapper implements ResultSetMapper + private static class TaskInfoMapper implements ResultSetMapper { private final ObjectMapper objectMapper; @@ -705,33 +707,31 @@ static class TaskInfoMapper implements ResultSetMapper this.objectMapper = objectMapper; } + @Nullable @Override public TaskInfo map(int index, ResultSet resultSet, StatementContext context) throws SQLException { - final TaskInfo taskInfo; Task task; - TaskStatus status; try { task = objectMapper.readValue(resultSet.getBytes("payload"), Task.class); } catch (IOException e) { log.warn("Encountered exception[%s] while deserializing task payload, setting payload to null", e.getMessage()); - task = null; + return null; } try { - status = objectMapper.readValue(resultSet.getBytes("status_payload"), TaskStatus.class); + TaskStatus status = objectMapper.readValue(resultSet.getBytes("status_payload"), TaskStatus.class); + return new TaskInfo( + DateTimes.of(resultSet.getString("created_date")), + status, + task + ); } catch (IOException e) { log.error(e, "Encountered exception while deserializing task status_payload"); throw new SQLException(e); } - taskInfo = new TaskInfo( - DateTimes.of(resultSet.getString("created_date")), - status, - task - ); - return taskInfo; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java index 7aad62310b11..870f99a309f7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageAdapterTest.java @@ -92,12 +92,12 @@ public void testDeletePendingSegmentsOfOneOverlappingRunningTask() new TaskInfo( DateTimes.of("2017-11-01"), TaskStatus.running("id1"), - NoopTask.create() + new NoopTask("id1", null, "dataSource", 1L, 0L, null) ), new TaskInfo( DateTimes.of("2017-12-02"), TaskStatus.running("id2"), - NoopTask.create() + new NoopTask("id2", null, "dataSource", 1L, 0L, null) ) ); @@ -135,12 +135,12 @@ public void testDeletePendingSegmentsOfMultipleOverlappingRunningTasks() new TaskInfo( DateTimes.of("2017-12-01"), TaskStatus.running("id1"), - NoopTask.create() + new NoopTask("id1", null, "dataSource", 1L, 0L, null) ), new TaskInfo( DateTimes.of("2017-11-01"), TaskStatus.running("id2"), - NoopTask.create() + new NoopTask("id2", null, "dataSource", 1L, 0L, null) ) ); diff --git a/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java b/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java index 5e5d0f5ef78b..9bfbd500f13f 100644 --- a/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java +++ b/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java @@ -20,6 +20,7 @@ package org.apache.druid.cli; import com.google.inject.Injector; +import org.apache.druid.discovery.NodeRole; import org.apache.druid.guice.StartupInjectorBuilder; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.metadata.segment.SqlSegmentsMetadataManagerV2; @@ -28,6 +29,8 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Set; + public class CliOverlordTest { @Test @@ -37,7 +40,7 @@ public void testSegmentMetadataCacheIsBound() final CliOverlord overlord = new CliOverlord(); injector.injectMembers(overlord); - final Injector overlordInjector = overlord.makeInjector(); + final Injector overlordInjector = overlord.makeInjector(Set.of(NodeRole.OVERLORD)); final SegmentMetadataCache segmentMetadataCache = overlordInjector.getInstance(SegmentMetadataCache.class); From a0665b7e8a5db7514952c641e7c0823901edeeea Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Tue, 2 Sep 2025 23:14:17 +0530 Subject: [PATCH 5/9] Clean up formatting --- .../druid/metadata/SQLServerMetadataStorageActionHandler.java | 3 +-- .../druid/metadata/MySQLMetadataStorageActionHandler.java | 3 +-- .../druid/metadata/PostgreSQLMetadataStorageActionHandler.java | 3 +-- .../druid/metadata/DerbyMetadataStorageActionHandler.java | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java index 91cd7d7112bd..cf491c041731 100644 --- a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java +++ b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/SQLServerMetadataStorageActionHandler.java @@ -21,8 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; -public class SQLServerMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class SQLServerMetadataStorageActionHandler extends SQLMetadataStorageActionHandler { public SQLServerMetadataStorageActionHandler( SQLMetadataConnector connector, diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java index c30d23ce6ec9..54a0938dec3f 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/MySQLMetadataStorageActionHandler.java @@ -21,8 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; -public class MySQLMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class MySQLMetadataStorageActionHandler extends SQLMetadataStorageActionHandler { MySQLMetadataStorageActionHandler( SQLMetadataConnector connector, diff --git a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java index c04cf17c43ff..4181633463ea 100644 --- a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java +++ b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/PostgreSQLMetadataStorageActionHandler.java @@ -21,8 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; -public class PostgreSQLMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class PostgreSQLMetadataStorageActionHandler extends SQLMetadataStorageActionHandler { public PostgreSQLMetadataStorageActionHandler( SQLMetadataConnector connector, diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java index 7491d5c89bcf..f60341e293e5 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/DerbyMetadataStorageActionHandler.java @@ -22,8 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -public class DerbyMetadataStorageActionHandler - extends SQLMetadataStorageActionHandler +public class DerbyMetadataStorageActionHandler extends SQLMetadataStorageActionHandler { @VisibleForTesting DerbyMetadataStorageActionHandler( From 7ff00d8ee254290115e1a1a5bb4db7fc0396e422 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 4 Sep 2025 11:19:58 +0530 Subject: [PATCH 6/9] Remove class TaskStuff --- .../overlord/HeapMemoryTaskStorage.java | 134 +++++------------- 1 file changed, 32 insertions(+), 102 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java index cef49b9d0506..b3201ea9314f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/HeapMemoryTaskStorage.java @@ -21,7 +21,6 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; @@ -33,7 +32,6 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexing.common.TaskLock; -import org.apache.druid.indexing.common.actions.TaskAction; import org.apache.druid.indexing.common.config.TaskStorageConfig; import org.apache.druid.indexing.common.task.Task; import org.apache.druid.java.util.common.DateTimes; @@ -45,7 +43,6 @@ import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -60,12 +57,10 @@ public class HeapMemoryTaskStorage implements TaskStorage { private final TaskStorageConfig config; - private final ConcurrentHashMap tasks = new ConcurrentHashMap<>(); + private final ConcurrentHashMap tasks = new ConcurrentHashMap<>(); @GuardedBy("itself") private final Multimap taskLocks = HashMultimap.create(); - @GuardedBy("itself") - private final Multimap taskActions = ArrayListMultimap.create(); private static final Logger log = new Logger(HeapMemoryTaskStorage.class); @@ -87,23 +82,23 @@ public TaskInfo insert(Task task, TaskStatus status) status.getId() ); - TaskStuff newTaskStuff = new TaskStuff(task, status, DateTimes.nowUtc(), task.getDataSource()); - TaskStuff existingTaskStuff = tasks.putIfAbsent(task.getId(), newTaskStuff); - if (existingTaskStuff != null) { + final TaskInfo newTaskInfo = new TaskInfo(DateTimes.nowUtc(), status, task); + final TaskInfo existingTaskInfo = tasks.putIfAbsent(task.getId(), newTaskInfo); + if (existingTaskInfo != null) { throw EntryAlreadyExists.exception("Task[%s] already exists", task.getId()); } log.info("Inserted task[%s] with status[%s]", task.getId(), status); - return TaskStuff.toTaskInfo(newTaskStuff); + return newTaskInfo; } @Override public Optional getTask(String taskid) { Preconditions.checkNotNull(taskid, "taskid"); - TaskStuff taskStuff = tasks.get(taskid); - if (taskStuff != null) { - return Optional.of(taskStuff.getTask()); + TaskInfo taskInfo = tasks.get(taskid); + if (taskInfo != null) { + return Optional.of(taskInfo.getTask()); } else { return Optional.absent(); } @@ -116,9 +111,9 @@ public void setStatus(TaskStatus status) final String taskid = status.getId(); log.info("Updating task %s to status: %s", taskid, status); - TaskStuff updated = tasks.computeIfPresent(taskid, (tid, taskStuff) -> { - Preconditions.checkState(taskStuff.getStatus().isRunnable(), "Task must be runnable: %s", taskid); - return taskStuff.withStatus(status); + TaskInfo updated = tasks.computeIfPresent(taskid, (tid, taskInfo) -> { + Preconditions.checkState(taskInfo.getStatus().isRunnable(), "Task must be runnable: %s", taskid); + return taskInfo.withStatus(status); }); Preconditions.checkNotNull(updated, "Task ID must already be present: %s", taskid); } @@ -127,7 +122,7 @@ public void setStatus(TaskStatus status) public Optional getStatus(String taskid) { Preconditions.checkNotNull(taskid, "taskid"); - TaskStuff existing = tasks.get(taskid); + TaskInfo existing = tasks.get(taskid); if (existing != null) { return Optional.of(existing.getStatus()); } else { @@ -140,21 +135,16 @@ public Optional getStatus(String taskid) public TaskInfo getTaskInfo(String taskId) { Preconditions.checkNotNull(taskId, "taskId"); - final TaskStuff taskStuff = tasks.get(taskId); - if (taskStuff != null) { - return TaskStuff.toTaskInfo(taskStuff); - } else { - return null; - } + return tasks.get(taskId); } @Override public List getActiveTasks() { final ImmutableList.Builder listBuilder = ImmutableList.builder(); - for (final TaskStuff taskStuff : tasks.values()) { - if (taskStuff.getStatus().isRunnable()) { - listBuilder.add(taskStuff.getTask()); + for (final TaskInfo taskInfo : tasks.values()) { + if (taskInfo.getStatus().isRunnable()) { + listBuilder.add(taskInfo.getTask()); } } return listBuilder.build(); @@ -164,9 +154,9 @@ public List getActiveTasks() public List getActiveTaskInfos() { final ImmutableList.Builder listBuilder = ImmutableList.builder(); - for (final TaskStuff taskStuff : tasks.values()) { - if (taskStuff.getStatus().isRunnable()) { - listBuilder.add(TaskStuff.toTaskInfo(taskStuff)); + for (final TaskInfo taskInfo : tasks.values()) { + if (taskInfo.getStatus().isRunnable()) { + listBuilder.add(taskInfo); } } return listBuilder.build(); @@ -176,7 +166,7 @@ public List getActiveTaskInfos() public List getActiveTasksByDatasource(String datasource) { final ImmutableList.Builder listBuilder = ImmutableList.builder(); - for (Map.Entry entry : tasks.entrySet()) { + for (Map.Entry entry : tasks.entrySet()) { if (entry.getValue().getStatus().isRunnable() && entry.getValue().getDataSource().equals(datasource)) { listBuilder.add(entry.getValue().getTask()); } @@ -187,10 +177,10 @@ public List getActiveTasksByDatasource(String datasource) public List getActiveTaskInfo(@Nullable String dataSource) { final ImmutableList.Builder listBuilder = ImmutableList.builder(); - for (final TaskStuff taskStuff : tasks.values()) { - if (taskStuff.getStatus().isRunnable()) { - if (dataSource == null || dataSource.equals(taskStuff.getDataSource())) { - listBuilder.add(TaskStuff.toTaskInfo(taskStuff)); + for (final TaskInfo taskInfo : tasks.values()) { + if (taskInfo.getStatus().isRunnable()) { + if (dataSource == null || dataSource.equals(taskInfo.getDataSource())) { + listBuilder.add(taskInfo); } } } @@ -199,12 +189,12 @@ public List getActiveTaskInfo(@Nullable String dataSource) public List getRecentlyCreatedAlreadyFinishedTaskInfo(CompleteTaskLookup taskLookup) { - final Ordering createdDateDesc = new Ordering() + final Ordering createdDateDesc = new Ordering() { @Override - public int compare(TaskStuff a, TaskStuff b) + public int compare(TaskInfo a, TaskInfo b) { - return a.getCreatedDate().compareTo(b.getCreatedDate()); + return a.getCreatedTime().compareTo(b.getCreatedTime()); } }.reverse(); @@ -257,21 +247,18 @@ public List getTaskStatusPlusList( private List getRecentlyCreatedAlreadyFinishedTaskInfoSince( DateTime start, @Nullable Integer n, - Ordering createdDateDesc + Ordering createdDateDesc ) { - Stream stream = tasks + Stream stream = tasks .values() .stream() - .filter(taskStuff -> taskStuff.getStatus().isComplete() && taskStuff.getCreatedDate().isAfter(start)) + .filter(taskInfo -> taskInfo.getStatus().isComplete() && taskInfo.getCreatedTime().isAfter(start)) .sorted(createdDateDesc); if (n != null) { stream = stream.limit(n); } - List list = stream - .map(TaskStuff::toTaskInfo) - .collect(Collectors.toList()); - return Collections.unmodifiableList(list); + return stream.collect(Collectors.toUnmodifiableList()); } @Override @@ -328,67 +315,10 @@ public void removeTasksOlderThan(final long timestamp) // it will not cause any problems. List taskIds = tasks.entrySet().stream() .filter(entry -> entry.getValue().getStatus().isComplete() - && entry.getValue().getCreatedDate().isBefore(timestamp)) + && entry.getValue().getCreatedTime().isBefore(timestamp)) .map(Map.Entry::getKey) .collect(Collectors.toList()); taskIds.forEach(tasks::remove); - synchronized (taskActions) { - for (String taskId : taskIds) { - taskActions.removeAll(taskId); - } - } - } - - private static class TaskStuff - { - final Task task; - final TaskStatus status; - final DateTime createdDate; - final String dataSource; - - private TaskStuff(Task task, TaskStatus status, DateTime createdDate, String dataSource) - { - Preconditions.checkArgument(task.getId().equals(status.getId())); - - this.task = Preconditions.checkNotNull(task, "task"); - this.status = Preconditions.checkNotNull(status, "status"); - this.createdDate = Preconditions.checkNotNull(createdDate, "createdDate"); - this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); - } - - public Task getTask() - { - return task; - } - - public TaskStatus getStatus() - { - return status; - } - - public DateTime getCreatedDate() - { - return createdDate; - } - - public String getDataSource() - { - return dataSource; - } - - private TaskStuff withStatus(TaskStatus _status) - { - return new TaskStuff(task, _status, createdDate, dataSource); - } - - static TaskInfo toTaskInfo(TaskStuff taskStuff) - { - return new TaskInfo( - taskStuff.getCreatedDate(), - taskStuff.getStatus(), - taskStuff.getTask() - ); - } } } From 47b78d1c7d3b29330e95307b6db21334da39a58d Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 4 Sep 2025 11:21:08 +0530 Subject: [PATCH 7/9] Fix compilation --- .../druid/indexing/overlord/TaskQueueConcurrencyTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueConcurrencyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueConcurrencyTest.java index 288ea3a968dc..f9a1d64c877a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueConcurrencyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueConcurrencyTest.java @@ -523,7 +523,7 @@ public void test_shutdown_doesNotWait_forTaskToShutdownOnRunner() // Verify that shutdown on TaskQueue finishes immediately taskQueue.shutdown(task1.getId(), "killed"); - Optional> taskInfo = taskQueue.getActiveTaskInfo(task1.getId()); + Optional taskInfo = taskQueue.getActiveTaskInfo(task1.getId()); Assert.assertTrue(taskInfo.isPresent()); Assert.assertEquals(TaskStatus.failure(task1.getId(), "killed"), taskInfo.get().getStatus()); From 8ec9d876ebf055629b87a6615c08cea99a59078c Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 4 Sep 2025 11:26:28 +0530 Subject: [PATCH 8/9] Add javadoc --- .../apache/druid/metadata/MetadataStorageActionHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java index a70d2ba8f012..23e315aac0c6 100644 --- a/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java +++ b/indexing-service/src/main/java/org/apache/druid/metadata/MetadataStorageActionHandler.java @@ -35,6 +35,9 @@ import java.util.List; import java.util.Map; +/** + * Handles metadata storage of {@link Task}, {@link TaskLock} and {@link TaskStatus} objects. + */ @ExtensionPoint public interface MetadataStorageActionHandler { From 3bf7d67a9933a24be05869ce42aa7c1ee1761950 Mon Sep 17 00:00:00 2001 From: Kashif Faraz Date: Thu, 4 Sep 2025 11:29:43 +0530 Subject: [PATCH 9/9] Remove config taskLog table --- .../updater/MetadataStorageUpdaterJobSpec.java | 1 - .../metadata/MetadataStorageTablesConfig.java | 14 +------------- .../guice/MetadataStorageTablesConfigTest.java | 4 ---- .../metadata/TestMetadataStorageTablesConfig.java | 1 - 4 files changed, 1 insertion(+), 19 deletions(-) diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java index dfbdc3e4e85f..b262e9b4def6 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java @@ -98,7 +98,6 @@ public MetadataStorageTablesConfig getMetadataStorageTablesConfig() null, null, null, - null, null ); } diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java index fd6a62143488..730da1ad81b2 100644 --- a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java +++ b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java @@ -31,11 +31,9 @@ public class MetadataStorageTablesConfig public static MetadataStorageTablesConfig fromBase(String base) { - return new MetadataStorageTablesConfig(base, null, null, null, null, null, null, null, null, null, null, null, null); + return new MetadataStorageTablesConfig(base, null, null, null, null, null, null, null, null, null, null, null); } - public static final String TASK_ENTRY_TYPE = "task"; - private static final String DEFAULT_BASE = "druid"; @JsonProperty("base") @@ -62,9 +60,6 @@ public static MetadataStorageTablesConfig fromBase(String base) @JsonProperty("tasks") private final String tasksTable; - @JsonProperty("taskLog") - private final String taskLogTable; - @JsonProperty("taskLock") private final String taskLockTable; @@ -86,7 +81,6 @@ public MetadataStorageTablesConfig( @JsonProperty("rules") String rulesTable, @JsonProperty("config") String configTable, @JsonProperty("tasks") String tasksTable, - @JsonProperty("taskLog") String taskLogTable, @JsonProperty("taskLock") String taskLockTable, @JsonProperty("audit") String auditTable, @JsonProperty("supervisors") String supervisorTable, @@ -103,7 +97,6 @@ public MetadataStorageTablesConfig( this.configTable = makeTableName(configTable, "config"); this.tasksTable = makeTableName(tasksTable, "tasks"); - this.taskLogTable = makeTableName(taskLogTable, "tasklogs"); this.taskLockTable = makeTableName(taskLockTable, "tasklocks"); this.auditTable = makeTableName(auditTable, "audit"); this.supervisorTable = makeTableName(supervisorTable, "supervisors"); @@ -172,11 +165,6 @@ public String getTasksTable() return tasksTable; } - public String getTaskLogTable() - { - return taskLogTable; - } - public String getTaskLockTable() { return taskLockTable; diff --git a/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java b/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java index 2df5643e16f4..33f886ad56a4 100644 --- a/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java +++ b/processing/src/test/java/org/apache/druid/guice/MetadataStorageTablesConfigTest.java @@ -71,10 +71,6 @@ public ObjectMapper jsonMapper() props.getProperty("druid.metadata.storage.tables.tasks"), config.getTasksTable() ); - Assert.assertEquals( - props.getProperty("druid.metadata.storage.tables.taskLog"), - config.getTaskLogTable() - ); Assert.assertEquals( props.getProperty("druid.metadata.storage.tables.taskLock"), config.getTaskLockTable() diff --git a/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java b/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java index 784b7e2cad69..8a864f425548 100644 --- a/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java +++ b/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java @@ -38,7 +38,6 @@ public TestMetadataStorageTablesConfig() null, null, null, - null, null ); }