From 2573e03352f5d7535ad40c863cfccc2917da56e2 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 18 Sep 2024 12:33:56 +0530 Subject: [PATCH 01/24] HDDS-11465. Introducing Schema Versioning for Recon Derby table to Handle Fresh Installs and Upgrades. --- .../codegen/ReconSchemaGenerationModule.java | 7 +- .../ReconSchemaVersionTableManager.java | 64 +++++++++++++++++++ .../schema/ContainerSchemaDefinition.java | 35 ++++++---- .../recon/schema/ReconSchemaDefinition.java | 6 ++ .../schema/ReconTaskSchemaDefinition.java | 6 ++ .../schema/SchemaVersionTableDefinition.java | 53 +++++++++++++++ .../recon/schema/StatsSchemaDefinition.java | 6 ++ .../schema/UtilizationSchemaDefinition.java | 6 ++ .../hadoop/ozone/recon/ReconConstants.java | 3 + .../ozone/recon/ReconSchemaManager.java | 54 ++++++++++++++-- .../hadoop/ozone/recon/ReconServer.java | 2 +- .../persistence/AbstractReconSqlDBTest.java | 2 +- 12 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java create mode 100644 hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java index 8272c2bd6da9..9a06a20fdffd 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java @@ -17,11 +17,7 @@ */ package org.hadoop.ozone.recon.codegen; -import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition; -import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition; -import org.hadoop.ozone.recon.schema.ReconSchemaDefinition; -import org.hadoop.ozone.recon.schema.StatsSchemaDefinition; -import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; +import org.hadoop.ozone.recon.schema.*; import com.google.inject.AbstractModule; import com.google.inject.multibindings.Multibinder; @@ -40,5 +36,6 @@ protected void configure() { schemaBinder.addBinding().to(ContainerSchemaDefinition.class); schemaBinder.addBinding().to(ReconTaskSchemaDefinition.class); schemaBinder.addBinding().to(StatsSchemaDefinition.class); + schemaBinder.addBinding().to(SchemaVersionTableDefinition.class); } } diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java new file mode 100644 index 000000000000..6c249851cdb1 --- /dev/null +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java @@ -0,0 +1,64 @@ +package org.hadoop.ozone.recon.codegen; + +import com.google.inject.Inject; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.sql.DataSource; +import java.sql.SQLException; + +import static org.jooq.impl.DSL.name; + +public class ReconSchemaVersionTableManager { + + private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); + public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; + private final DSLContext dslContext; + + @Inject + public ReconSchemaVersionTableManager(DataSource dataSource) throws + SQLException { + this.dslContext = DSL.using(dataSource.getConnection()); + } + + /** + * Get the current schema version stored in the RECON_SCHEMA_VERSION_TABLE. + * + * @return The current schema version as a String, or null if no entry exists. + * @throws SQLException if any SQL error occurs. + */ + public String getCurrentSchemaVersion() { + return dslContext.select(DSL.field(name("version_number"))) + .from(RECON_SCHEMA_VERSION_TABLE_NAME) + .fetchOneInto(String.class); // Return the version number or null if no entry exists + } + + /** + * Update the schema version in the RECON_SCHEMA_VERSION_TABLE after all tables are upgraded. + * + * @param newVersion The new version to set. + * @throws SQLException if any SQL error occurs. + */ + public void updateSchemaVersion(String newVersion) { + boolean recordExists = dslContext.fetchExists( + dslContext.selectOne() + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + ); + + if (recordExists) { + dslContext.update(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .set(DSL.field(name("version_number")), newVersion) + .set(DSL.field(name("applied_on")), DSL.currentTimestamp()) + .execute(); + LOG.info("Updated schema version to '{}'.", newVersion); + } else { + dslContext.insertInto(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .columns(DSL.field(name("version_number")), DSL.field(name("applied_on"))) + .values(newVersion, DSL.currentTimestamp()) + .execute(); + LOG.info("Inserted new schema version '{}'.", newVersion); + } + } + +} diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java index 0882de3bf4fa..cad5eaee7f07 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java @@ -71,25 +71,34 @@ public void initializeSchema() throws SQLException { Connection conn = dataSource.getConnection(); dslContext = DSL.using(conn); - if (TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { - // Drop the existing constraint if it exists - String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1"; - dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME) - .dropConstraint(constraintName) - .execute(); - - // Add the updated constraint with all enum states - addUpdatedConstraint(); - } else { - // Create the table if it does not exist + if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { createUnhealthyContainersTable(); } } + @Override + public void upgradeSchema(String fromVersion, String toVersion) + throws SQLException { + Connection conn = dataSource.getConnection(); + if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { + return; + } + // Example upgrade script + if (fromVersion.equals("1.0") && toVersion.equals("2.0")) { + runMigrationToVersion2(conn); + } + } + /** - * Add the updated constraint to the table. + * Run the upgrade to version 2.0. */ - private void addUpdatedConstraint() { + private void runMigrationToVersion2(Connection conn) throws SQLException { + // Drop the existing constraint if it exists + String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1"; + dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME) + .dropConstraint(constraintName) + .execute(); + // Get all enum values as a list of strings String[] enumStates = Arrays.stream(UnHealthyContainerStates.values()) .map(Enum::name) diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java index 72a105e5fff5..a4b4536dc3c0 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java @@ -31,4 +31,10 @@ public interface ReconSchemaDefinition { * Execute DDL that will create Recon schema. */ void initializeSchema() throws SQLException; + + /** + * Upgrade the schema for the table. + * This method will be called during schema upgrades. + */ + void upgradeSchema(String fromVersion, String toVersion) throws SQLException; } diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java index dfa76eac4fa4..cda358a2c9f9 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java @@ -55,6 +55,12 @@ public void initializeSchema() throws SQLException { } } + @Override + public void upgradeSchema(String fromVersion, String toVersion) + throws SQLException { + + } + /** * Create the Recon Task Status table. * @param conn connection diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java new file mode 100644 index 000000000000..20b60a36e589 --- /dev/null +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java @@ -0,0 +1,53 @@ +package org.hadoop.ozone.recon.schema; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; + +/** + * Class for managing the schema of the SchemaVersion table. + */ +@Singleton +public class SchemaVersionTableDefinition implements ReconSchemaDefinition { + + public static final String SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; + private final DataSource dataSource; + private DSLContext dslContext; + + @Inject + public SchemaVersionTableDefinition(DataSource dataSource) { + this.dataSource = dataSource; + } + + @Override + public void initializeSchema() throws SQLException { + Connection conn = dataSource.getConnection(); + dslContext = DSL.using(conn); + + if (!TABLE_EXISTS_CHECK.test(conn, SCHEMA_VERSION_TABLE_NAME)) { + createSchemaVersionTable(); + } + } + + @Override + public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { + // No specific upgrade logic needed for SchemaVersionTable for now + } + + /** + * Create the Schema Version table. + */ + private void createSchemaVersionTable() throws SQLException { + dslContext.createTableIfNotExists(SCHEMA_VERSION_TABLE_NAME) + .column("version_number", SQLDataType.VARCHAR(10).nullable(false)) + .column("applied_on", SQLDataType.TIMESTAMP.defaultValue(DSL.currentTimestamp())) + .execute(); + } +} diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java index 394c9de8df59..ff3644961f45 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java @@ -54,6 +54,12 @@ public void initializeSchema() throws SQLException { } } + @Override + public void upgradeSchema(String fromVersion, String toVersion) + throws SQLException { + + } + /** * Create the Ozone Global Stats table. */ diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java index d33c7daf308b..83e7e8003726 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java @@ -75,6 +75,12 @@ public void initializeSchema() throws SQLException { } } + @Override + public void upgradeSchema(String fromVersion, String toVersion) + throws SQLException { + + } + private void createClusterGrowthTable() { dslContext.createTableIfNotExists(CLUSTER_GROWTH_DAILY_TABLE_NAME) .column("timestamp", SQLDataType.TIMESTAMP.nullable(false)) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index ed657931e034..f587de55fee7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -35,6 +35,9 @@ private ReconConstants() { public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db"; + // Latest Schema Version for all Recon Derby Tables + public static final String LATEST_SCHEMA_VERSION = "2.0"; // Replace this as needed + // By default, limit the number of results returned /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java index 253e37d75abe..e9a7865cc6d8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.Set; +import org.hadoop.ozone.recon.codegen.ReconSchemaVersionTableManager; import org.hadoop.ozone.recon.schema.ReconSchemaDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,21 +38,62 @@ public class ReconSchemaManager { private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaManager.class); private Set reconSchemaDefinitions = new HashSet<>(); + private final ReconSchemaVersionTableManager schemaVersionTableManager; @Inject - public ReconSchemaManager(Set reconSchemaDefinitions) { + public ReconSchemaManager(Set reconSchemaDefinitions, + ReconSchemaVersionTableManager schemaVersionTableManager) { + this.schemaVersionTableManager = schemaVersionTableManager; this.reconSchemaDefinitions.addAll(reconSchemaDefinitions); } @VisibleForTesting - public void createReconSchema() { - reconSchemaDefinitions.forEach(reconSchemaDefinition -> { + public void createAndUpgradeReconSchema() { + + // Initialize all tables + initializeAllSchemas(); + + // Fetch current version from the SchemaVersionTable + String currentVersion = schemaVersionTableManager.getCurrentSchemaVersion(); + String latestVersion = ReconConstants.LATEST_SCHEMA_VERSION; + + // Upgrade schema if necessary. + if (currentVersion == null || !currentVersion.equals(latestVersion)) { + upgradeAllSchemas(currentVersion, latestVersion); + // Update the schema version in the version table + schemaVersionTableManager.updateSchemaVersion(latestVersion); + } else { + LOG.info("Recon Derby Schema is already up to date."); + } + } + + /** + * Initialize all schemas. + */ + private void initializeAllSchemas() { + for (ReconSchemaDefinition reconSchemaDefinition : reconSchemaDefinitions) { try { reconSchemaDefinition.initializeSchema(); } catch (SQLException e) { - LOG.error("Error creating Recon schema {}.", - reconSchemaDefinition.getClass().getSimpleName(), e); + LOG.error("Error initializing schema: {}", reconSchemaDefinition.getClass().getSimpleName(), e); } - }); + } + LOG.info("All Derby table schemas initialized."); } + + /** + * Upgrade all schemas to the latest version. + */ + private void upgradeAllSchemas(String currentVersion, String latestVersion) { + for (ReconSchemaDefinition schemaDefinition : reconSchemaDefinitions) { + try { + // Use the upgrade logic from each schema + schemaDefinition.upgradeSchema(currentVersion, latestVersion); + } catch (SQLException e) { + LOG.error("Error upgrading schema: {}", schemaDefinition.getClass().getSimpleName(), e); + } + } + LOG.info("All schemas upgraded to the latest version: {}", latestVersion); + } + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 3295eb4524c7..dae187f2e138 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -139,7 +139,7 @@ public Void call() throws Exception { ReconSchemaManager reconSchemaManager = injector.getInstance(ReconSchemaManager.class); LOG.info("Creating Recon Schema."); - reconSchemaManager.createReconSchema(); + reconSchemaManager.createAndUpgradeReconSchema(); LOG.debug("Recon schema creation done."); this.reconSafeModeMgr = injector.getInstance(ReconSafeModeManager.class); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java index d007fbb1cf7b..010ec1069d9d 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java @@ -115,7 +115,7 @@ protected void configure() { public void createSchema(Injector inj) { ReconSchemaManager reconSchemaManager = inj.getInstance(ReconSchemaManager.class); - reconSchemaManager.createReconSchema(); + reconSchemaManager.createAndUpgradeReconSchema(); } protected Injector getInjector() { From 3e64282d9a23655a36760f618aac506aa07f778e Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 19 Sep 2024 01:02:23 +0530 Subject: [PATCH 02/24] Covered a few edge cases of upgrades and fresh install --- .../schema/ContainerSchemaDefinition.java | 6 ++ .../schema/ReconTaskSchemaDefinition.java | 2 +- .../schema/SchemaVersionTableDefinition.java | 2 +- .../recon/schema/StatsSchemaDefinition.java | 2 +- .../schema/UtilizationSchemaDefinition.java | 2 +- .../hadoop/ozone/recon/ReconConstants.java | 3 +- .../ozone/recon/ReconSchemaManager.java | 68 +++++++++++++++++-- .../ReconSchemaVersionTableManager.java | 14 ++-- 8 files changed, 84 insertions(+), 15 deletions(-) rename hadoop-ozone/{recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen => recon/src/main/java/org/apache/hadoop/ozone/recon}/ReconSchemaVersionTableManager.java (84%) diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java index cad5eaee7f07..bc9c6483f719 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java @@ -27,6 +27,8 @@ import org.jooq.DSLContext; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.sql.DataSource; import java.sql.Connection; @@ -39,6 +41,9 @@ @Singleton public class ContainerSchemaDefinition implements ReconSchemaDefinition { + private static final Logger LOG = + LoggerFactory.getLogger(ContainerSchemaDefinition.class); + public static final String UNHEALTHY_CONTAINERS_TABLE_NAME = "UNHEALTHY_CONTAINERS"; @@ -86,6 +91,7 @@ public void upgradeSchema(String fromVersion, String toVersion) // Example upgrade script if (fromVersion.equals("1.0") && toVersion.equals("2.0")) { runMigrationToVersion2(conn); + LOG.info("Upgraded schema from version 1.0 to 2.0."); } } diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java index cda358a2c9f9..3e49f3b673b4 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java @@ -58,7 +58,7 @@ public void initializeSchema() throws SQLException { @Override public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { - + // No schema upgrades needed for Recon Task Status table. } /** diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java index 20b60a36e589..e699f30634a0 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java @@ -38,7 +38,7 @@ public void initializeSchema() throws SQLException { @Override public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { - // No specific upgrade logic needed for SchemaVersionTable for now + // No schema upgrades needed for the Schema Version table. } /** diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java index ff3644961f45..ef380a7cfb8b 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java @@ -57,7 +57,7 @@ public void initializeSchema() throws SQLException { @Override public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { - + // No schema upgrades needed for the stats table. } /** diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java index 83e7e8003726..3780d943a3e1 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java @@ -78,7 +78,7 @@ public void initializeSchema() throws SQLException { @Override public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { - + // No schema upgrades needed for the utilization tables. } private void createClusterGrowthTable() { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index f587de55fee7..b598c0157039 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -36,7 +36,8 @@ private ReconConstants() { public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db"; // Latest Schema Version for all Recon Derby Tables - public static final String LATEST_SCHEMA_VERSION = "2.0"; // Replace this as needed + // Bump it up whenever there is a schema change in Recon Derby Tables + public static final String LATEST_SCHEMA_VERSION = "2.0"; // By default, limit the number of results returned diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java index e9a7865cc6d8..74d708361e0e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java @@ -18,11 +18,11 @@ package org.apache.hadoop.ozone.recon; +import java.sql.Connection; import java.sql.SQLException; import java.util.HashSet; import java.util.Set; -import org.hadoop.ozone.recon.codegen.ReconSchemaVersionTableManager; import org.hadoop.ozone.recon.schema.ReconSchemaDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,8 +30,28 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; +import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; + /** - * Class used to create Recon SQL tables. + * Manages the creation and upgrade of Recon SQL tables with schema versioning. + * + * This class handles the following scenarios: + * + * 1. Fresh Installation: + * - No tables, including `schemaVersionTable`, exist. + * - All tables are created with the latest schema version. + * + * 2. Upgrade from Older Version: + * - All Existing tables (e.g., `UNHEALTHY_CONTAINERS`) are present, but `schemaVersionTable` is missing. + * - Indicates an upgrade from a version without schema tracking which was introduced in version 2.0. + * - The `schemaVersionTable` is created and all tables are upgraded to the latest version. + * + * 3. Upgrade with SchemaVersionTable: + * - `schemaVersionTable` exists but is outdated. + * - Migrations are applied to upgrade all tables to the latest schema. + * + * 4. Schema Already Up to Date: + * - All tables and the schema version match the latest version; no action is needed. */ public class ReconSchemaManager { @@ -50,6 +70,8 @@ public ReconSchemaManager(Set reconSchemaDefinitions, @VisibleForTesting public void createAndUpgradeReconSchema() { + boolean isUpgrade = areOtherTablesExisting(); + // Initialize all tables initializeAllSchemas(); @@ -57,10 +79,23 @@ public void createAndUpgradeReconSchema() { String currentVersion = schemaVersionTableManager.getCurrentSchemaVersion(); String latestVersion = ReconConstants.LATEST_SCHEMA_VERSION; - // Upgrade schema if necessary. - if (currentVersion == null || !currentVersion.equals(latestVersion)) { + // Handle cases where currentVersion is null, indicating schemaVersionTable just got created + if (currentVersion == null) { + if (isUpgrade) { + // Case 1: Upgrade from older version where schemaVersionTable was not present + LOG.info("Upgrade from older version detected. Setting current schema version to 1.0."); + currentVersion = "1.0"; // Set current version to the previous version before schemaVersionTable was introduced + } else { + // Case 2: Fresh install + LOG.info("Fresh installation detected. Setting schema version to latest."); + currentVersion = latestVersion; + } + } + + // Upgrade schema if necessary + if (!currentVersion.equals(latestVersion)) { upgradeAllSchemas(currentVersion, latestVersion); - // Update the schema version in the version table + // Update the schema version in the version table after migration schemaVersionTableManager.updateSchemaVersion(latestVersion); } else { LOG.info("Recon Derby Schema is already up to date."); @@ -96,4 +131,27 @@ private void upgradeAllSchemas(String currentVersion, String latestVersion) { LOG.info("All schemas upgraded to the latest version: {}", latestVersion); } + /** + * Checks whether essential tables (other than schemaVersionTable) already exist in the database. + * + * This method is used to distinguish between an upgrade and a fresh installation scenario. + * + * - If essential tables (like UNHEALTHY_CONTAINERS) exist but the schemaVersionTable does not + * have any records, it indicates an upgrade from an older version where schemaVersionTable + * was not present. + * - If none of the essential tables exist, it indicates a fresh installation, where all tables + * will be created for the first time, including the schemaVersionTable. + * + * @return true if essential tables already exist (indicating an upgrade), false if not + * (indicating a fresh installation) + */ + private boolean areOtherTablesExisting() { + try (Connection conn = schemaVersionTableManager.getDataSource().getConnection()) { + // Check if some essential tables, like UNHEALTHY_CONTAINERS, already exist + return TABLE_EXISTS_CHECK.test(conn, "UNHEALTHY_CONTAINERS"); + } catch (SQLException e) { + LOG.error("Error checking table existence for upgrade detection", e); + return false; + } + } } diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java similarity index 84% rename from hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index 6c249851cdb1..8cfb402f0949 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -1,4 +1,4 @@ -package org.hadoop.ozone.recon.codegen; +package org.apache.hadoop.ozone.recon; import com.google.inject.Inject; import org.jooq.DSLContext; @@ -15,10 +15,12 @@ public class ReconSchemaVersionTableManager { private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; private final DSLContext dslContext; + private final DataSource dataSource; @Inject - public ReconSchemaVersionTableManager(DataSource dataSource) throws + public ReconSchemaVersionTableManager(DataSource src) throws SQLException { + this.dataSource = src; this.dslContext = DSL.using(dataSource.getConnection()); } @@ -41,9 +43,8 @@ public String getCurrentSchemaVersion() { * @throws SQLException if any SQL error occurs. */ public void updateSchemaVersion(String newVersion) { - boolean recordExists = dslContext.fetchExists( - dslContext.selectOne() - .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + boolean recordExists = dslContext.fetchExists(dslContext.selectOne() + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) ); if (recordExists) { @@ -61,4 +62,7 @@ public void updateSchemaVersion(String newVersion) { } } + public DataSource getDataSource() { + return dataSource; + } } From a4895e04ddf645aa848311de14f4813e004533b9 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 23 Sep 2024 19:20:36 +0530 Subject: [PATCH 03/24] Made changes for latest discussion --- .../schema/ContainerSchemaDefinition.java | 41 +++++---- .../hadoop/ozone/recon/ReconConstants.java | 26 ++++++ .../ozone/recon/ReconSchemaManager.java | 84 ++++++++++++------- .../recon/ReconSchemaVersionTableManager.java | 7 ++ 4 files changed, 109 insertions(+), 49 deletions(-) diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java index bc9c6483f719..de5ede959b01 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java @@ -73,32 +73,39 @@ public enum UnHealthyContainerStates { @Override public void initializeSchema() throws SQLException { - Connection conn = dataSource.getConnection(); - dslContext = DSL.using(conn); - - if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { - createUnhealthyContainersTable(); + try (Connection conn = dataSource.getConnection()) { + dslContext = DSL.using(conn); + if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { + createUnhealthyContainersTable(); + } + } catch (SQLException e) { + LOG.error("Error initializing schema", e); + throw e; } } @Override - public void upgradeSchema(String fromVersion, String toVersion) - throws SQLException { - Connection conn = dataSource.getConnection(); - if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { - return; - } - // Example upgrade script - if (fromVersion.equals("1.0") && toVersion.equals("2.0")) { - runMigrationToVersion2(conn); - LOG.info("Upgraded schema from version 1.0 to 2.0."); + public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { + try (Connection conn = dataSource.getConnection()) { + if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { + return; + } + if (fromVersion.equals("0") && toVersion.equals("1.0")) { + runMigrationToVersion1(conn); + LOG.info("Upgraded schema from version 0 to 1.0."); + } + // Add more upgrade paths here as needed + } catch (SQLException e) { + LOG.error("Error upgrading schema", e); + throw e; } } + /** - * Run the upgrade to version 2.0. + * Run the upgrade to version 1.0. */ - private void runMigrationToVersion2(Connection conn) throws SQLException { + private void runMigrationToVersion1(Connection conn) throws SQLException { // Drop the existing constraint if it exists String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1"; dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index b598c0157039..932f0a8cbade 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -39,6 +39,32 @@ private ReconConstants() { // Bump it up whenever there is a schema change in Recon Derby Tables public static final String LATEST_SCHEMA_VERSION = "2.0"; + // Schema versions enum + public enum SchemaVersion { + V0("0"), + V1_0("1.0"), + LATEST("1.0"); // Update as necessary for future versions + + private final String version; + + SchemaVersion(String version) { + this.version = version; + } + + public String getVersion() { + return version; + } + + public static SchemaVersion from(String version) { + for (SchemaVersion schemaVersion : SchemaVersion.values()) { + if (schemaVersion.version.equals(version)) { + return schemaVersion; + } + } + throw new IllegalArgumentException("Unsupported schema version: " + version); + } + } + // By default, limit the number of results returned /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java index 74d708361e0e..201ab5263ed9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java @@ -41,10 +41,10 @@ * - No tables, including `schemaVersionTable`, exist. * - All tables are created with the latest schema version. * - * 2. Upgrade from Older Version: - * - All Existing tables (e.g., `UNHEALTHY_CONTAINERS`) are present, but `schemaVersionTable` is missing. - * - Indicates an upgrade from a version without schema tracking which was introduced in version 2.0. - * - The `schemaVersionTable` is created and all tables are upgraded to the latest version. + * 2. Upgrade from Version 0 (pre-versioning): + * - Existing tables (e.g., `UNHEALTHY_CONTAINERS`) are present, but `schemaVersionTable` is missing. + * - Indicates an upgrade from version 0. + * - The `schemaVersionTable` is created, and all tables are upgraded to the latest version. * * 3. Upgrade with SchemaVersionTable: * - `schemaVersionTable` exists but is outdated. @@ -68,34 +68,37 @@ public ReconSchemaManager(Set reconSchemaDefinitions, } @VisibleForTesting - public void createAndUpgradeReconSchema() { - - boolean isUpgrade = areOtherTablesExisting(); - - // Initialize all tables - initializeAllSchemas(); + public void createAndUpgradeReconSchema() throws SQLException { // Fetch current version from the SchemaVersionTable String currentVersion = schemaVersionTableManager.getCurrentSchemaVersion(); String latestVersion = ReconConstants.LATEST_SCHEMA_VERSION; - // Handle cases where currentVersion is null, indicating schemaVersionTable just got created + // Handle cases where schemaVersionTable is missing if (currentVersion == null) { - if (isUpgrade) { - // Case 1: Upgrade from older version where schemaVersionTable was not present - LOG.info("Upgrade from older version detected. Setting current schema version to 1.0."); - currentVersion = "1.0"; // Set current version to the previous version before schemaVersionTable was introduced - } else { - // Case 2: Fresh install - LOG.info("Fresh installation detected. Setting schema version to latest."); - currentVersion = latestVersion; - } + currentVersion = handleMissingVersion(); + // Initialize the schema if this is a fresh install or an upgrade from version 0 + initializeAllSchemas(); } // Upgrade schema if necessary if (!currentVersion.equals(latestVersion)) { - upgradeAllSchemas(currentVersion, latestVersion); - // Update the schema version in the version table after migration + switch (currentVersion) { + case "0": + LOG.info("Upgrading from version 0 to version {}", latestVersion); + upgradeAllSchemas(currentVersion, latestVersion); + break; + case "1.0": + LOG.info("Upgrading from version 1.0 to version {}", latestVersion); + upgradeAllSchemas(currentVersion, latestVersion); + break; + // Additional cases can be added here as we introduce new versions + default: + LOG.warn("Unsupported schema version: {}", currentVersion); + break; + } + + // After migration, update the schemaVersionTable to reflect the latest version schemaVersionTableManager.updateSchemaVersion(latestVersion); } else { LOG.info("Recon Derby Schema is already up to date."); @@ -111,6 +114,7 @@ private void initializeAllSchemas() { reconSchemaDefinition.initializeSchema(); } catch (SQLException e) { LOG.error("Error initializing schema: {}", reconSchemaDefinition.getClass().getSimpleName(), e); + return; } } LOG.info("All Derby table schemas initialized."); @@ -126,28 +130,44 @@ private void upgradeAllSchemas(String currentVersion, String latestVersion) { schemaDefinition.upgradeSchema(currentVersion, latestVersion); } catch (SQLException e) { LOG.error("Error upgrading schema: {}", schemaDefinition.getClass().getSimpleName(), e); + return; } } LOG.info("All schemas upgraded to the latest version: {}", latestVersion); } /** - * Checks whether essential tables (other than schemaVersionTable) already exist in the database. + * Handle the case where the current schema version is missing from the `schemaVersionTable`. * - * This method is used to distinguish between an upgrade and a fresh installation scenario. + * @param isUpgrade whether or not this is an upgrade from an older version. + * @return the assumed current version. + */ + private String handleMissingVersion() { + boolean isUpgrade = areOtherTablesExisting(); + if (isUpgrade) { + // Case: Upgrade from version 0 (pre-schema versioning) + LOG.info("Detected upgrade from version 0 (pre-versioning). Setting current schema version to 0"); + return "0"; + } else { + // Case: Fresh install + LOG.info("Fresh installation detected. Setting schema version to the latest."); + return ReconConstants.LATEST_SCHEMA_VERSION; + } + } + + /** + * Check if essential tables (other than schemaVersionTable) already exist in the database. * - * - If essential tables (like UNHEALTHY_CONTAINERS) exist but the schemaVersionTable does not - * have any records, it indicates an upgrade from an older version where schemaVersionTable - * was not present. - * - If none of the essential tables exist, it indicates a fresh installation, where all tables - * will be created for the first time, including the schemaVersionTable. + * This method distinguishes between an upgrade and a fresh install. + * - If essential tables (like UNHEALTHY_CONTAINERS) exist but schemaVersionTable does not, + * it indicates an upgrade from version 0 (pre-versioning). + * - If none of the essential tables exist, it indicates a fresh installation. * - * @return true if essential tables already exist (indicating an upgrade), false if not - * (indicating a fresh installation) + * @return true if essential tables already exist (indicating an upgrade), false if not (indicating a fresh installation) */ private boolean areOtherTablesExisting() { try (Connection conn = schemaVersionTableManager.getDataSource().getConnection()) { - // Check if some essential tables, like UNHEALTHY_CONTAINERS, already exist + // Check if essential tables, like UNHEALTHY_CONTAINERS, already exist return TABLE_EXISTS_CHECK.test(conn, "UNHEALTHY_CONTAINERS"); } catch (SQLException e) { LOG.error("Error checking table existence for upgrade detection", e); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index 8cfb402f0949..034bf34b2761 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -6,8 +6,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.sql.DataSource; +import java.sql.Connection; import java.sql.SQLException; +import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; import static org.jooq.impl.DSL.name; public class ReconSchemaVersionTableManager { @@ -16,12 +18,14 @@ public class ReconSchemaVersionTableManager { public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; private final DSLContext dslContext; private final DataSource dataSource; + private final Connection conn; @Inject public ReconSchemaVersionTableManager(DataSource src) throws SQLException { this.dataSource = src; this.dslContext = DSL.using(dataSource.getConnection()); + this.conn = dataSource.getConnection(); } /** @@ -31,6 +35,9 @@ public ReconSchemaVersionTableManager(DataSource src) throws * @throws SQLException if any SQL error occurs. */ public String getCurrentSchemaVersion() { + if (!TABLE_EXISTS_CHECK.test(conn, RECON_SCHEMA_VERSION_TABLE_NAME)) { + return null; + } return dslContext.select(DSL.field(name("version_number"))) .from(RECON_SCHEMA_VERSION_TABLE_NAME) .fetchOneInto(String.class); // Return the version number or null if no entry exists From 3128cd934bab1a63ba7bba5a80874c93919b2ff6 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 29 Sep 2024 20:48:48 +0530 Subject: [PATCH 04/24] Removed the old changes --- .../schema/ContainerSchemaDefinition.java | 56 +++----- .../recon/schema/ReconSchemaDefinition.java | 6 - .../schema/ReconTaskSchemaDefinition.java | 6 - .../recon/schema/StatsSchemaDefinition.java | 6 - .../schema/UtilizationSchemaDefinition.java | 6 - .../hadoop/ozone/recon/ReconConstants.java | 30 ---- .../ozone/recon/ReconSchemaManager.java | 134 +----------------- .../hadoop/ozone/recon/ReconServer.java | 2 +- .../persistence/AbstractReconSqlDBTest.java | 2 +- 9 files changed, 26 insertions(+), 222 deletions(-) diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java index de5ede959b01..0882de3bf4fa 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java @@ -27,8 +27,6 @@ import org.jooq.DSLContext; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.sql.DataSource; import java.sql.Connection; @@ -41,9 +39,6 @@ @Singleton public class ContainerSchemaDefinition implements ReconSchemaDefinition { - private static final Logger LOG = - LoggerFactory.getLogger(ContainerSchemaDefinition.class); - public static final String UNHEALTHY_CONTAINERS_TABLE_NAME = "UNHEALTHY_CONTAINERS"; @@ -73,45 +68,28 @@ public enum UnHealthyContainerStates { @Override public void initializeSchema() throws SQLException { - try (Connection conn = dataSource.getConnection()) { - dslContext = DSL.using(conn); - if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { - createUnhealthyContainersTable(); - } - } catch (SQLException e) { - LOG.error("Error initializing schema", e); - throw e; - } - } - - @Override - public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { - try (Connection conn = dataSource.getConnection()) { - if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { - return; - } - if (fromVersion.equals("0") && toVersion.equals("1.0")) { - runMigrationToVersion1(conn); - LOG.info("Upgraded schema from version 0 to 1.0."); - } - // Add more upgrade paths here as needed - } catch (SQLException e) { - LOG.error("Error upgrading schema", e); - throw e; + Connection conn = dataSource.getConnection(); + dslContext = DSL.using(conn); + + if (TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) { + // Drop the existing constraint if it exists + String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1"; + dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME) + .dropConstraint(constraintName) + .execute(); + + // Add the updated constraint with all enum states + addUpdatedConstraint(); + } else { + // Create the table if it does not exist + createUnhealthyContainersTable(); } } - /** - * Run the upgrade to version 1.0. + * Add the updated constraint to the table. */ - private void runMigrationToVersion1(Connection conn) throws SQLException { - // Drop the existing constraint if it exists - String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1"; - dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME) - .dropConstraint(constraintName) - .execute(); - + private void addUpdatedConstraint() { // Get all enum values as a list of strings String[] enumStates = Arrays.stream(UnHealthyContainerStates.values()) .map(Enum::name) diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java index a4b4536dc3c0..72a105e5fff5 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java @@ -31,10 +31,4 @@ public interface ReconSchemaDefinition { * Execute DDL that will create Recon schema. */ void initializeSchema() throws SQLException; - - /** - * Upgrade the schema for the table. - * This method will be called during schema upgrades. - */ - void upgradeSchema(String fromVersion, String toVersion) throws SQLException; } diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java index 3e49f3b673b4..dfa76eac4fa4 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java @@ -55,12 +55,6 @@ public void initializeSchema() throws SQLException { } } - @Override - public void upgradeSchema(String fromVersion, String toVersion) - throws SQLException { - // No schema upgrades needed for Recon Task Status table. - } - /** * Create the Recon Task Status table. * @param conn connection diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java index ef380a7cfb8b..394c9de8df59 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java @@ -54,12 +54,6 @@ public void initializeSchema() throws SQLException { } } - @Override - public void upgradeSchema(String fromVersion, String toVersion) - throws SQLException { - // No schema upgrades needed for the stats table. - } - /** * Create the Ozone Global Stats table. */ diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java index 3780d943a3e1..d33c7daf308b 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java @@ -75,12 +75,6 @@ public void initializeSchema() throws SQLException { } } - @Override - public void upgradeSchema(String fromVersion, String toVersion) - throws SQLException { - // No schema upgrades needed for the utilization tables. - } - private void createClusterGrowthTable() { dslContext.createTableIfNotExists(CLUSTER_GROWTH_DAILY_TABLE_NAME) .column("timestamp", SQLDataType.TIMESTAMP.nullable(false)) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 932f0a8cbade..ed657931e034 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -35,36 +35,6 @@ private ReconConstants() { public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db"; - // Latest Schema Version for all Recon Derby Tables - // Bump it up whenever there is a schema change in Recon Derby Tables - public static final String LATEST_SCHEMA_VERSION = "2.0"; - - // Schema versions enum - public enum SchemaVersion { - V0("0"), - V1_0("1.0"), - LATEST("1.0"); // Update as necessary for future versions - - private final String version; - - SchemaVersion(String version) { - this.version = version; - } - - public String getVersion() { - return version; - } - - public static SchemaVersion from(String version) { - for (SchemaVersion schemaVersion : SchemaVersion.values()) { - if (schemaVersion.version.equals(version)) { - return schemaVersion; - } - } - throw new IllegalArgumentException("Unsupported schema version: " + version); - } - } - // By default, limit the number of results returned /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java index 201ab5263ed9..253e37d75abe 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.recon; -import java.sql.Connection; import java.sql.SQLException; import java.util.HashSet; import java.util.Set; @@ -30,148 +29,29 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; - /** - * Manages the creation and upgrade of Recon SQL tables with schema versioning. - * - * This class handles the following scenarios: - * - * 1. Fresh Installation: - * - No tables, including `schemaVersionTable`, exist. - * - All tables are created with the latest schema version. - * - * 2. Upgrade from Version 0 (pre-versioning): - * - Existing tables (e.g., `UNHEALTHY_CONTAINERS`) are present, but `schemaVersionTable` is missing. - * - Indicates an upgrade from version 0. - * - The `schemaVersionTable` is created, and all tables are upgraded to the latest version. - * - * 3. Upgrade with SchemaVersionTable: - * - `schemaVersionTable` exists but is outdated. - * - Migrations are applied to upgrade all tables to the latest schema. - * - * 4. Schema Already Up to Date: - * - All tables and the schema version match the latest version; no action is needed. + * Class used to create Recon SQL tables. */ public class ReconSchemaManager { private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaManager.class); private Set reconSchemaDefinitions = new HashSet<>(); - private final ReconSchemaVersionTableManager schemaVersionTableManager; @Inject - public ReconSchemaManager(Set reconSchemaDefinitions, - ReconSchemaVersionTableManager schemaVersionTableManager) { - this.schemaVersionTableManager = schemaVersionTableManager; + public ReconSchemaManager(Set reconSchemaDefinitions) { this.reconSchemaDefinitions.addAll(reconSchemaDefinitions); } @VisibleForTesting - public void createAndUpgradeReconSchema() throws SQLException { - - // Fetch current version from the SchemaVersionTable - String currentVersion = schemaVersionTableManager.getCurrentSchemaVersion(); - String latestVersion = ReconConstants.LATEST_SCHEMA_VERSION; - - // Handle cases where schemaVersionTable is missing - if (currentVersion == null) { - currentVersion = handleMissingVersion(); - // Initialize the schema if this is a fresh install or an upgrade from version 0 - initializeAllSchemas(); - } - - // Upgrade schema if necessary - if (!currentVersion.equals(latestVersion)) { - switch (currentVersion) { - case "0": - LOG.info("Upgrading from version 0 to version {}", latestVersion); - upgradeAllSchemas(currentVersion, latestVersion); - break; - case "1.0": - LOG.info("Upgrading from version 1.0 to version {}", latestVersion); - upgradeAllSchemas(currentVersion, latestVersion); - break; - // Additional cases can be added here as we introduce new versions - default: - LOG.warn("Unsupported schema version: {}", currentVersion); - break; - } - - // After migration, update the schemaVersionTable to reflect the latest version - schemaVersionTableManager.updateSchemaVersion(latestVersion); - } else { - LOG.info("Recon Derby Schema is already up to date."); - } - } - - /** - * Initialize all schemas. - */ - private void initializeAllSchemas() { - for (ReconSchemaDefinition reconSchemaDefinition : reconSchemaDefinitions) { + public void createReconSchema() { + reconSchemaDefinitions.forEach(reconSchemaDefinition -> { try { reconSchemaDefinition.initializeSchema(); } catch (SQLException e) { - LOG.error("Error initializing schema: {}", reconSchemaDefinition.getClass().getSimpleName(), e); - return; + LOG.error("Error creating Recon schema {}.", + reconSchemaDefinition.getClass().getSimpleName(), e); } - } - LOG.info("All Derby table schemas initialized."); - } - - /** - * Upgrade all schemas to the latest version. - */ - private void upgradeAllSchemas(String currentVersion, String latestVersion) { - for (ReconSchemaDefinition schemaDefinition : reconSchemaDefinitions) { - try { - // Use the upgrade logic from each schema - schemaDefinition.upgradeSchema(currentVersion, latestVersion); - } catch (SQLException e) { - LOG.error("Error upgrading schema: {}", schemaDefinition.getClass().getSimpleName(), e); - return; - } - } - LOG.info("All schemas upgraded to the latest version: {}", latestVersion); - } - - /** - * Handle the case where the current schema version is missing from the `schemaVersionTable`. - * - * @param isUpgrade whether or not this is an upgrade from an older version. - * @return the assumed current version. - */ - private String handleMissingVersion() { - boolean isUpgrade = areOtherTablesExisting(); - if (isUpgrade) { - // Case: Upgrade from version 0 (pre-schema versioning) - LOG.info("Detected upgrade from version 0 (pre-versioning). Setting current schema version to 0"); - return "0"; - } else { - // Case: Fresh install - LOG.info("Fresh installation detected. Setting schema version to the latest."); - return ReconConstants.LATEST_SCHEMA_VERSION; - } - } - - /** - * Check if essential tables (other than schemaVersionTable) already exist in the database. - * - * This method distinguishes between an upgrade and a fresh install. - * - If essential tables (like UNHEALTHY_CONTAINERS) exist but schemaVersionTable does not, - * it indicates an upgrade from version 0 (pre-versioning). - * - If none of the essential tables exist, it indicates a fresh installation. - * - * @return true if essential tables already exist (indicating an upgrade), false if not (indicating a fresh installation) - */ - private boolean areOtherTablesExisting() { - try (Connection conn = schemaVersionTableManager.getDataSource().getConnection()) { - // Check if essential tables, like UNHEALTHY_CONTAINERS, already exist - return TABLE_EXISTS_CHECK.test(conn, "UNHEALTHY_CONTAINERS"); - } catch (SQLException e) { - LOG.error("Error checking table existence for upgrade detection", e); - return false; - } + }); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index dae187f2e138..3295eb4524c7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -139,7 +139,7 @@ public Void call() throws Exception { ReconSchemaManager reconSchemaManager = injector.getInstance(ReconSchemaManager.class); LOG.info("Creating Recon Schema."); - reconSchemaManager.createAndUpgradeReconSchema(); + reconSchemaManager.createReconSchema(); LOG.debug("Recon schema creation done."); this.reconSafeModeMgr = injector.getInstance(ReconSafeModeManager.class); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java index 010ec1069d9d..d007fbb1cf7b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java @@ -115,7 +115,7 @@ protected void configure() { public void createSchema(Injector inj) { ReconSchemaManager reconSchemaManager = inj.getInstance(ReconSchemaManager.class); - reconSchemaManager.createAndUpgradeReconSchema(); + reconSchemaManager.createReconSchema(); } protected Injector getInjector() { From a57f9225d3620c0db171c5c12a91477db60892de Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 29 Sep 2024 21:05:53 +0530 Subject: [PATCH 05/24] New things --- .../schema/SchemaVersionTableDefinition.java | 33 +++-- .../recon/ReconSchemaVersionTableManager.java | 51 ++++--- .../recon/upgrade/Feature1UpgradeAction.java | 10 ++ .../recon/upgrade/Feature2UpgradeAction.java | 10 ++ .../recon/upgrade/Feature3UpgradeAction.java | 10 ++ .../recon/upgrade/ReconLayoutFeature.java | 52 +++++++ .../upgrade/ReconLayoutVersionManager.java | 134 ++++++++++++++++++ .../recon/upgrade/ReconUpgradeAction.java | 27 ++++ 8 files changed, 299 insertions(+), 28 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java index e699f30634a0..8a50645f6f5e 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.hadoop.ozone.recon.schema; import com.google.inject.Inject; @@ -5,12 +24,13 @@ import org.jooq.DSLContext; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; -import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; import javax.sql.DataSource; import java.sql.Connection; import java.sql.SQLException; +import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; + /** * Class for managing the schema of the SchemaVersion table. */ @@ -36,18 +56,15 @@ public void initializeSchema() throws SQLException { } } - @Override - public void upgradeSchema(String fromVersion, String toVersion) throws SQLException { - // No schema upgrades needed for the Schema Version table. - } - /** * Create the Schema Version table. */ private void createSchemaVersionTable() throws SQLException { dslContext.createTableIfNotExists(SCHEMA_VERSION_TABLE_NAME) - .column("version_number", SQLDataType.VARCHAR(10).nullable(false)) - .column("applied_on", SQLDataType.TIMESTAMP.defaultValue(DSL.currentTimestamp())) + .column("version_number", SQLDataType.INTEGER.nullable(false)) + .column("applied_on", + SQLDataType.TIMESTAMP.defaultValue(DSL.currentTimestamp())) .execute(); } + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index 034bf34b2761..34665b4c0583 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -1,46 +1,59 @@ -package org.apache.hadoop.ozone.recon; +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.ozone.recon; import com.google.inject.Inject; import org.jooq.DSLContext; import org.jooq.impl.DSL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.sql.DataSource; -import java.sql.Connection; import java.sql.SQLException; -import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; import static org.jooq.impl.DSL.name; +/** + * ReconSchemaVersionTableManager is responsible for managing the schema version of the Recon Metadata. + */ public class ReconSchemaVersionTableManager { - private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; private final DSLContext dslContext; private final DataSource dataSource; - private final Connection conn; @Inject public ReconSchemaVersionTableManager(DataSource src) throws SQLException { this.dataSource = src; this.dslContext = DSL.using(dataSource.getConnection()); - this.conn = dataSource.getConnection(); } /** - * Get the current schema version stored in the RECON_SCHEMA_VERSION_TABLE. - * - * @return The current schema version as a String, or null if no entry exists. - * @throws SQLException if any SQL error occurs. + * Get the current schema version from the RECON_SCHEMA_VERSION_TABLE. + * @return The current schema version. */ - public String getCurrentSchemaVersion() { - if (!TABLE_EXISTS_CHECK.test(conn, RECON_SCHEMA_VERSION_TABLE_NAME)) { - return null; - } + public int getCurrentSchemaVersion() { return dslContext.select(DSL.field(name("version_number"))) - .from(RECON_SCHEMA_VERSION_TABLE_NAME) - .fetchOneInto(String.class); // Return the version number or null if no entry exists + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .fetchOptional() + .map(record -> record.get(DSL.field(name("version_number"), Integer.class))) + .orElse(0); } /** @@ -49,11 +62,10 @@ public String getCurrentSchemaVersion() { * @param newVersion The new version to set. * @throws SQLException if any SQL error occurs. */ - public void updateSchemaVersion(String newVersion) { + public void updateSchemaVersion(int newVersion) { boolean recordExists = dslContext.fetchExists(dslContext.selectOne() .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) ); - if (recordExists) { dslContext.update(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) .set(DSL.field(name("version_number")), newVersion) @@ -68,8 +80,7 @@ public void updateSchemaVersion(String newVersion) { LOG.info("Inserted new schema version '{}'.", newVersion); } } - public DataSource getDataSource() { return dataSource; } -} +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java new file mode 100644 index 000000000000..62e479b8dcbb --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java @@ -0,0 +1,10 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +public class Feature1UpgradeAction implements ReconUpgradeAction { + @Override + public void execute() throws Exception { + // Logic for upgrading to version 1 + System.out.println("Executing Feature 1 upgrade: Adding new column to the table."); + // Implement the database schema update or other upgrade logic here + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java new file mode 100644 index 000000000000..fe7264a5b64a --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java @@ -0,0 +1,10 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +public class Feature2UpgradeAction implements ReconUpgradeAction { + @Override + public void execute() throws Exception { + // Logic for upgrading to version 2 + System.out.println("Executing Feature 2 upgrade: Creating a new table."); + // Implement the database schema update or other upgrade logic here + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java new file mode 100644 index 000000000000..ce1c0db1449b --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java @@ -0,0 +1,10 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +public class Feature3UpgradeAction implements ReconUpgradeAction { + @Override + public void execute() throws Exception { + // Logic for upgrading to version 3 + System.out.println("Executing Feature 3 upgrade: Modifying table schema."); + // Implement the database schema update or other upgrade logic here + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java new file mode 100644 index 000000000000..ecc0a99758d8 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +/** + * Enum representing Recon layout features with their version, description, + * and associated upgrade action to be executed during an upgrade. + */ +public enum ReconLayoutFeature { + FEATURE_1(1, "Description for Feature 1", new Feature1UpgradeAction()), + FEATURE_2(2, "Description for Feature 2", new Feature2UpgradeAction()), + FEATURE_3(3, "Description for Feature 3", new Feature3UpgradeAction()); + + private final int version; + private final String description; + private final ReconUpgradeAction upgradeAction; + + ReconLayoutFeature(int version, String description, ReconUpgradeAction upgradeAction) { + this.version = version; + this.description = description; + this.upgradeAction = upgradeAction; + } + + public int getVersion() { + return version; + } + + public String getDescription() { + return description; + } + + public ReconUpgradeAction getUpgradeAction() { + return upgradeAction; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java new file mode 100644 index 000000000000..57bc049647b8 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * ReconLayoutVersionManager is responsible for managing the layout version of the Recon service. + * It determines the current Metadata Layout Version (MLV) and Schema Layout Version (SLV) of the + * Recon service, and finalizes the layout features that need to be upgraded. + */ +public class ReconLayoutVersionManager { + + private static final Logger LOG = + LoggerFactory.getLogger(ReconLayoutVersionManager.class); + + // Current Metadata Layout Version (MLV) of the Recon service + private int currentMLV; + // Schema Layout Version (SLV) of the Recon service + private final int currentSLV; + private final ReconSchemaVersionTableManager schemaVersionTableManager; + + public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager) { + this.schemaVersionTableManager = schemaVersionTableManager; + this.currentMLV = determineMLV(); + this.currentSLV = determineSLV(); + } + + /** + * Determines the current Metadata Layout Version (MLV) from the version table. + * @return The current Metadata Layout Version (MLV). + */ + private int determineMLV() { + return schemaVersionTableManager.getCurrentSchemaVersion(); + } + + /** + * Determines the Schema Layout Version (SLV) based on the latest feature version. + * @return The Schema Layout Version (SLV). + */ + private int determineSLV() { + return Arrays.stream(ReconLayoutFeature.values()) + .mapToInt(ReconLayoutFeature::getVersion) + .max() + .orElse(0); // Default to 0 if no features are defined + } + + /** + * Finalizes the layout features that need to be upgraded, by executing the upgrade action for each + * feature that is registered for finalization. + */ + public void finalizeLayoutFeatures() { + // Get features that need finalization, sorted by version + List featuresToFinalize = getRegisteredFeatures(); + + for (ReconLayoutFeature feature : featuresToFinalize) { + try { + // Execute the upgrade action for this feature + feature.getUpgradeAction().execute(); + // Update the MLV in the database after successful execution + updateSchemaVersion(feature.getVersion()); + LOG.info("Feature " + feature.getVersion() + " finalized successfully."); + } catch (Exception e) { + LOG.info("Failed to finalize feature " + feature.getVersion() + ": " + e.getMessage()); + break; + } + } + } + + /** + * Returns a list of ReconLayoutFeature objects that are registered for finalization. + * @return List of ReconLayoutFeature objects that are registered for finalization. + */ + private List getRegisteredFeatures() { + List allFeatures = + Arrays.asList(ReconLayoutFeature.values()); + + LOG.info("Current MLV: {}. SLV: {}. Checking features for registration...", currentMLV, currentSLV); + + List registeredFeatures = allFeatures.stream() + .filter(feature -> { + boolean shouldRegister = + feature.getVersion() > currentMLV && feature.getVersion() <= currentSLV; + if (shouldRegister) { + LOG.info("Feature {} (version {}) is registered for finalization.", + feature.name(), feature.getVersion()); + } else { + LOG.info( + "Feature {} (version {}) is NOT registered for finalization.", + feature.name(), feature.getVersion()); + } + return shouldRegister; + }) + .sorted((a, b) -> Integer.compare(a.getVersion(), b.getVersion())) // Sort by version in ascending order + .collect(Collectors.toList()); + + return registeredFeatures; + } + + /** + * Updates the Schema Layout Version (SLV) in the database after finalizing a feature. + * @param newVersion The new Schema Layout Version (SLV) to set. + */ + private void updateSchemaVersion(int newVersion) { + // Logic to update the MLV in the database + this.currentMLV = newVersion; + LOG.info("MLV updated to: " + newVersion); + // Code to update the schema version in the database goes here. + schemaVersionTableManager.updateSchemaVersion(newVersion); + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java new file mode 100644 index 000000000000..36109758ffdd --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +/** + * ReconUpgradeAction is an interface for executing upgrade actions in Recon. + */ +public interface ReconUpgradeAction { + void execute() throws Exception; +} From b80f014775301b0ee9efb2dd0c35e7fe98706057 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 29 Sep 2024 21:06:23 +0530 Subject: [PATCH 06/24] New changes --- .../recon/ReconSchemaVersionTableManager.java | 79 +++++++++---------- .../hadoop/ozone/recon/ReconServer.java | 1 + 2 files changed, 39 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index 34665b4c0583..7948e322bc2b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -18,69 +18,66 @@ */ package org.apache.hadoop.ozone.recon; -import com.google.inject.Inject; + import org.jooq.DSLContext; import org.jooq.impl.DSL; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import javax.sql.DataSource; +import java.sql.Connection; import java.sql.SQLException; -import static org.jooq.impl.DSL.name; - /** - * ReconSchemaVersionTableManager is responsible for managing the schema version of the Recon Metadata. + * Class for managing the schema of the SchemaVersion table. */ public class ReconSchemaVersionTableManager { - private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); - public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; private final DSLContext dslContext; private final DataSource dataSource; - @Inject - public ReconSchemaVersionTableManager(DataSource src) throws - SQLException { - this.dataSource = src; + public ReconSchemaVersionTableManager(DataSource dataSource) throws SQLException { + this.dataSource = dataSource; this.dslContext = DSL.using(dataSource.getConnection()); } /** - * Get the current schema version from the RECON_SCHEMA_VERSION_TABLE. - * @return The current schema version. + * Fetch the current schema version from the database. + * @return the current schema version */ public int getCurrentSchemaVersion() { - return dslContext.select(DSL.field(name("version_number"))) - .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) - .fetchOptional() - .map(record -> record.get(DSL.field(name("version_number"), Integer.class))) - .orElse(0); + // Fetch the current schema version from the database + try { + return dslContext.select(DSL.field("version_number")) + .from("RECON_SCHEMA_VERSION") + .fetchOneInto(Integer.class); + } catch (Exception e) { + System.err.println("Error fetching current schema version: " + e.getMessage()); + return 0; // Assuming 0 if the table does not exist or no version is found + } } /** - * Update the schema version in the RECON_SCHEMA_VERSION_TABLE after all tables are upgraded. - * - * @param newVersion The new version to set. - * @throws SQLException if any SQL error occurs. + * Update the schema version in the database. + * @param newVersion the new schema version */ public void updateSchemaVersion(int newVersion) { - boolean recordExists = dslContext.fetchExists(dslContext.selectOne() - .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) - ); - if (recordExists) { - dslContext.update(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) - .set(DSL.field(name("version_number")), newVersion) - .set(DSL.field(name("applied_on")), DSL.currentTimestamp()) - .execute(); - LOG.info("Updated schema version to '{}'.", newVersion); - } else { - dslContext.insertInto(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) - .columns(DSL.field(name("version_number")), DSL.field(name("applied_on"))) - .values(newVersion, DSL.currentTimestamp()) - .execute(); - LOG.info("Inserted new schema version '{}'.", newVersion); + try (Connection conn = dataSource.getConnection()) { + boolean recordExists = dslContext.fetchExists( + dslContext.selectOne().from("RECON_SCHEMA_VERSION") + ); + + if (recordExists) { + dslContext.update(DSL.table("RECON_SCHEMA_VERSION")) + .set(DSL.field("version_number"), newVersion) + .execute(); + System.out.println("Schema version updated to " + newVersion); + } else { + dslContext.insertInto(DSL.table("RECON_SCHEMA_VERSION")) + .columns(DSL.field("version_number")) + .values(newVersion) + .execute(); + System.out.println("Inserted new schema version: " + newVersion); + } + } catch (SQLException e) { + System.err.println("Error updating schema version: " + e.getMessage()); } } - public DataSource getDataSource() { - return dataSource; - } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 3295eb4524c7..fb79210cc9a1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; +import org.apache.hadoop.ozone.recon.upgrade.ReconLayoutVersionManager; import org.apache.hadoop.ozone.util.OzoneNetUtils; import org.apache.hadoop.ozone.util.OzoneVersionInfo; import org.apache.hadoop.ozone.util.ShutdownHookManager; From ea5946daed04e77ef3ba0f9b066278b2042b3f85 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 30 Sep 2024 00:10:57 +0530 Subject: [PATCH 07/24] Fixed a bug --- .../java/org/apache/hadoop/ozone/recon/ReconServer.java | 9 +++++++++ .../ozone/recon/upgrade/ReconLayoutVersionManager.java | 1 + 2 files changed, 10 insertions(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index fb79210cc9a1..56dcfdafd63f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -143,6 +143,15 @@ public Void call() throws Exception { reconSchemaManager.createReconSchema(); LOG.debug("Recon schema creation done."); + // Handle Recon Schema Versioning + ReconSchemaVersionTableManager versionTableManager = + injector.getInstance(ReconSchemaVersionTableManager.class); + + ReconLayoutVersionManager layoutVersionManager = + new ReconLayoutVersionManager(versionTableManager); + // Run the upgrade framework to finalize layout features if needed + layoutVersionManager.finalizeLayoutFeatures(); + this.reconSafeModeMgr = injector.getInstance(ReconSafeModeManager.class); this.reconSafeModeMgr.setInSafeMode(true); httpServer = injector.getInstance(ReconHttpServer.class); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index 57bc049647b8..e9586bb53eaa 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; From 27f15385a6d547abf8ef23081a2ec645f5e67a63 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 30 Sep 2024 01:00:34 +0530 Subject: [PATCH 08/24] Finalised the new approach --- .../recon/ReconSchemaVersionTableManager.java | 76 ++++++++++++------- .../hadoop/ozone/recon/ReconServer.java | 1 + .../upgrade/ReconLayoutVersionManager.java | 2 +- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index 7948e322bc2b..dd2b52580683 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,65 +19,89 @@ package org.apache.hadoop.ozone.recon; +import com.google.inject.Inject; import org.jooq.DSLContext; import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.sql.DataSource; -import java.sql.Connection; import java.sql.SQLException; +import static org.jooq.impl.DSL.name; + /** - * Class for managing the schema of the SchemaVersion table. + * Manager for handling the Recon Schema Version table. + * This class provides methods to get and update the current schema version. */ public class ReconSchemaVersionTableManager { + + private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); + public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; private final DSLContext dslContext; private final DataSource dataSource; + @Inject public ReconSchemaVersionTableManager(DataSource dataSource) throws SQLException { this.dataSource = dataSource; this.dslContext = DSL.using(dataSource.getConnection()); } /** - * Fetch the current schema version from the database. - * @return the current schema version + * Get the current schema version from the RECON_SCHEMA_VERSION table. + * If the table is empty, or if it does not exist, it will return 0. + * @return The current schema version. */ public int getCurrentSchemaVersion() { - // Fetch the current schema version from the database try { - return dslContext.select(DSL.field("version_number")) - .from("RECON_SCHEMA_VERSION") - .fetchOneInto(Integer.class); + return dslContext.select(DSL.field(name("version_number"))) + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .fetchOptional() + .map(record -> record.get( + DSL.field(name("version_number"), Integer.class))) + .orElse(0); // Return 0 if no version is found } catch (Exception e) { - System.err.println("Error fetching current schema version: " + e.getMessage()); - return 0; // Assuming 0 if the table does not exist or no version is found + LOG.error("Failed to fetch the current schema version.", e); + return 0; // Return 0 if there is an exception } } /** - * Update the schema version in the database. - * @param newVersion the new schema version + * Update the schema version in the RECON_SCHEMA_VERSION table after all tables are upgraded. + * + * @param newVersion The new version to set. */ public void updateSchemaVersion(int newVersion) { - try (Connection conn = dataSource.getConnection()) { - boolean recordExists = dslContext.fetchExists( - dslContext.selectOne().from("RECON_SCHEMA_VERSION") - ); + try { + boolean recordExists = dslContext.fetchExists(dslContext.selectOne() + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME))); if (recordExists) { - dslContext.update(DSL.table("RECON_SCHEMA_VERSION")) - .set(DSL.field("version_number"), newVersion) + // Update the existing schema version record + dslContext.update(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .set(DSL.field(name("version_number")), newVersion) + .set(DSL.field(name("applied_on")), DSL.currentTimestamp()) .execute(); - System.out.println("Schema version updated to " + newVersion); + LOG.info("Updated schema version to '{}'.", newVersion); } else { - dslContext.insertInto(DSL.table("RECON_SCHEMA_VERSION")) - .columns(DSL.field("version_number")) - .values(newVersion) + // Insert a new schema version record + dslContext.insertInto(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .columns(DSL.field(name("version_number")), + DSL.field(name("applied_on"))) + .values(newVersion, DSL.currentTimestamp()) .execute(); - System.out.println("Inserted new schema version: " + newVersion); + LOG.info("Inserted new schema version '{}'.", newVersion); } - } catch (SQLException e) { - System.err.println("Error updating schema version: " + e.getMessage()); + } catch (Exception e) { + LOG.error("Failed to update schema version to '{}'.", newVersion, e); } } + + /** + * Provides the data source used by this manager. + * @return The DataSource instance. + */ + public DataSource getDataSource() { + return dataSource; + } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 56dcfdafd63f..9dad9daf8445 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -139,6 +139,7 @@ public Void call() throws Exception { ReconSchemaManager reconSchemaManager = injector.getInstance(ReconSchemaManager.class); + LOG.info("Creating Recon Schema."); reconSchemaManager.createReconSchema(); LOG.debug("Recon schema creation done."); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index e9586bb53eaa..b67fcad1027a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -128,8 +128,8 @@ private List getRegisteredFeatures() { private void updateSchemaVersion(int newVersion) { // Logic to update the MLV in the database this.currentMLV = newVersion; - LOG.info("MLV updated to: " + newVersion); // Code to update the schema version in the database goes here. schemaVersionTableManager.updateSchemaVersion(newVersion); + LOG.info("MLV updated to: " + newVersion); } } From e4b4b27d5e9824612cae3f285e8c575481500087 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 30 Sep 2024 01:40:57 +0530 Subject: [PATCH 09/24] Added log comments for testing --- .../hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java | 6 +++++- .../hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java | 7 ++++++- .../hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java | 6 +++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java index 62e479b8dcbb..dc7f17c0cfb6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java @@ -1,10 +1,14 @@ package org.apache.hadoop.ozone.recon.upgrade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class Feature1UpgradeAction implements ReconUpgradeAction { + private static final Logger LOG = LoggerFactory.getLogger(Feature1UpgradeAction.class); @Override public void execute() throws Exception { // Logic for upgrading to version 1 - System.out.println("Executing Feature 1 upgrade: Adding new column to the table."); + LOG.info("Executing Feature 1 upgrade:"); // Implement the database schema update or other upgrade logic here } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java index fe7264a5b64a..1159bbc20306 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java @@ -1,10 +1,15 @@ package org.apache.hadoop.ozone.recon.upgrade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class Feature2UpgradeAction implements ReconUpgradeAction { + private static final Logger LOG = LoggerFactory.getLogger(Feature2UpgradeAction.class); + @Override public void execute() throws Exception { // Logic for upgrading to version 2 - System.out.println("Executing Feature 2 upgrade: Creating a new table."); + LOG.info("Executing Feature 2 upgrade"); // Implement the database schema update or other upgrade logic here } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java index ce1c0db1449b..c92fe3510457 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java @@ -1,10 +1,14 @@ package org.apache.hadoop.ozone.recon.upgrade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class Feature3UpgradeAction implements ReconUpgradeAction { + private static final Logger LOG = LoggerFactory.getLogger(Feature3UpgradeAction.class); @Override public void execute() throws Exception { // Logic for upgrading to version 3 - System.out.println("Executing Feature 3 upgrade: Modifying table schema."); + LOG.info("Executing Feature 3 upgrade"); // Implement the database schema update or other upgrade logic here } } \ No newline at end of file From 262a93d21a881c17468216060f8f57edcd9b7141 Mon Sep 17 00:00:00 2001 From: arafat Date: Sat, 5 Oct 2024 19:42:00 +0530 Subject: [PATCH 10/24] Added an initial version to Feature enum --- .../upgrade/InitialLayoutUpgradeAction.java | 30 +++++++++++++++++++ .../recon/upgrade/ReconLayoutFeature.java | 3 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java new file mode 100644 index 000000000000..99b79bb032dc --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +public class InitialLayoutUpgradeAction implements ReconUpgradeAction { + + @Override + public void execute() throws Exception { + // No actions are performed for version 0. + // This version represents the introduction of schema versioning for Recon. + // Therefore, we do not perform any upgrade actions for now. + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java index ecc0a99758d8..7c46af95d556 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,6 +24,7 @@ * and associated upgrade action to be executed during an upgrade. */ public enum ReconLayoutFeature { + INITIAL_VERSION(0, "Initial Layout Version", new InitialLayoutUpgradeAction()), FEATURE_1(1, "Description for Feature 1", new Feature1UpgradeAction()), FEATURE_2(2, "Description for Feature 2", new Feature2UpgradeAction()), FEATURE_3(3, "Description for Feature 3", new Feature3UpgradeAction()); From 929d453e4f0a8c8aa6a0d70a164a7f25b1071dba Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 6 Oct 2024 01:54:16 +0530 Subject: [PATCH 11/24] Added tests for ReconLayoutVersionManager --- .../upgrade/ReconLayoutVersionManager.java | 13 +- .../TestReconLayoutVersionManager.java | 197 ++++++++++++++++++ 2 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index b67fcad1027a..00cbe9b02a8d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -50,6 +50,14 @@ public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTab this.currentSLV = determineSLV(); } + public int getCurrentMLV() { + return currentMLV; + } + + public int getCurrentSLV() { + return currentSLV; + } + /** * Determines the current Metadata Layout Version (MLV) from the version table. * @return The current Metadata Layout Version (MLV). @@ -95,7 +103,7 @@ public void finalizeLayoutFeatures() { * Returns a list of ReconLayoutFeature objects that are registered for finalization. * @return List of ReconLayoutFeature objects that are registered for finalization. */ - private List getRegisteredFeatures() { + protected List getRegisteredFeatures() { List allFeatures = Arrays.asList(ReconLayoutFeature.values()); @@ -103,8 +111,7 @@ private List getRegisteredFeatures() { List registeredFeatures = allFeatures.stream() .filter(feature -> { - boolean shouldRegister = - feature.getVersion() > currentMLV && feature.getVersion() <= currentSLV; + boolean shouldRegister = feature.getVersion() > currentMLV; if (shouldRegister) { LOG.info("Feature {} (version {}) is registered for finalization.", feature.name(), feature.getVersion()); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java new file mode 100644 index 000000000000..39da62a693d4 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -0,0 +1,197 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager; +import org.junit.jupiter.api.*; +import org.mockito.InOrder; +import org.mockito.MockedStatic; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +/** + * Tests for ReconLayoutVersionManager. + */ +public class TestReconLayoutVersionManager { + + private ReconSchemaVersionTableManager schemaVersionTableManager; + private ReconLayoutVersionManager layoutVersionManager; + private static MockedStatic mockedEnum; + + @BeforeEach + public void setUp() { + schemaVersionTableManager = mock(ReconSchemaVersionTableManager.class); + when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(0); + + // Mocking ReconLayoutFeature.values() to return custom enum instances + mockedEnum = mockStatic(ReconLayoutFeature.class); + ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); + when(feature1.getVersion()).thenReturn(1); + when(feature1.getUpgradeAction()).thenReturn(() -> { + // No-op for testing + }); + ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); + when(feature2.getVersion()).thenReturn(2); + when(feature2.getUpgradeAction()).thenReturn(() -> { + // No-op for testing + }); + + // Define the custom features to be returned + mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2}); + + layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager); + } + + @AfterEach + public void tearDown() { + // Close the static mock after each test to deregister it + mockedEnum.close(); + } + + /** + * Tests the initialization of layout version manager to ensure + * that the MLV (Metadata Layout Version) is set correctly to 0, + * and SLV (Software Layout Version) reflects the maximum available version. + */ + @Test + public void testInitializationWithMockedValues() { + assertEquals(0, layoutVersionManager.getCurrentMLV()); + assertEquals(2, layoutVersionManager.getCurrentSLV()); + } + + /** + * Tests the finalization of layout features and ensure that the updateSchemaVersion for + * the schemaVersionTable is triggered for each feature version. + */ + @Test + public void testFinalizeLayoutFeaturesWithMockedValues() { + layoutVersionManager.finalizeLayoutFeatures(); + + // Verify that schema versions are updated for our custom features + verify(schemaVersionTableManager, times(1)).updateSchemaVersion(1); + verify(schemaVersionTableManager, times(1)).updateSchemaVersion(2); + } + + /** + * Tests the retrieval of registered features to ensure that the correct + * layout features are returned according to the mocked values. + */ + @Test + public void testGetRegisteredFeaturesWithMockedValues() { + // Fetch the registered features + List registeredFeatures = layoutVersionManager.getRegisteredFeatures(); + + // Verify that the registered features match the mocked ones + ReconLayoutFeature feature1 = ReconLayoutFeature.values()[0]; + ReconLayoutFeature feature2 = ReconLayoutFeature.values()[1]; + List expectedFeatures = Arrays.asList(feature1, feature2); + assertEquals(expectedFeatures, registeredFeatures); + } + + /** + * Tests the scenario where no layout features are present. Ensures that no schema + * version updates are attempted when there are no features to finalize. + */ + @Test + public void testNoLayoutFeatures() { + mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{}); + layoutVersionManager.finalizeLayoutFeatures(); + verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt()); + } + + /** + * Handling Invalid Versions + * Higher MLV than Available SLV: Test the behavior when the current MLV (Metadata Layout Version) + * is set higher than the maximum version of ReconLayoutFeature. This scenario simulates + * an inconsistent state where the system’s metadata versioning is beyond the known layout features. + */ + @Test + public void testHigherMLVThanSLV() { + when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(5); + layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager); + assertEquals(2, layoutVersionManager.getCurrentSLV()); // Assuming 2 is the max SLV in mocked features + assertEquals(5, layoutVersionManager.getCurrentMLV()); + } + + /** + * Tests the scenario where an upgrade action fails. Ensures that if an upgrade action + * throws an exception, the schema version is not updated. + */ + @Test + public void testUpgradeActionFailure() throws Exception { + // Reset existing mocks and set up new features for this specific test + mockedEnum.reset(); + + // Mock ReconLayoutFeature instances + ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); + when(feature1.getVersion()).thenReturn(1); + ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); + + // Simulate an exception being thrown during the upgrade action execution + doThrow(new RuntimeException("Upgrade failed")).when(action1).execute(); + when(feature1.getUpgradeAction()).thenReturn(action1); + + // Mock the static values method to return the custom feature + mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1}); + + // Execute the layout feature finalization + layoutVersionManager.finalizeLayoutFeatures(); + // Verify that schema version update was never called due to the exception + verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt()); + } + + /** + * Tests the order of execution for the upgrade actions to ensure that + * they are executed sequentially according to their version numbers. + */ + @Test + public void testUpgradeActionExecutionOrder() throws Exception { + // Reset the existing static mock for this specific test + mockedEnum.reset(); + + // Mock ReconLayoutFeature instances + ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); + when(feature1.getVersion()).thenReturn(1); + ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); + when(feature1.getUpgradeAction()).thenReturn(action1); + + ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); + when(feature2.getVersion()).thenReturn(2); + ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); + when(feature2.getUpgradeAction()).thenReturn(action2); + + ReconLayoutFeature feature3 = mock(ReconLayoutFeature.class); + when(feature3.getVersion()).thenReturn(3); + ReconUpgradeAction action3 = mock(ReconUpgradeAction.class); + when(feature3.getUpgradeAction()).thenReturn(action3); + + // Mock the static values method to return custom features in a jumbled order + mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature2, feature3, feature1}); + + // Execute the layout feature finalization + layoutVersionManager.finalizeLayoutFeatures(); + + // Verify that the actions were executed in the correct order using InOrder + InOrder inOrder = inOrder(action1, action2, action3); + inOrder.verify(action1).execute(); // Should be executed first + inOrder.verify(action2).execute(); // Should be executed second + inOrder.verify(action3).execute(); // Should be executed third + } + + /** + * Tests the scenario where no upgrade actions are needed. Ensures that if the current + * schema version matches the maximum layout version, no upgrade actions are executed. + */ + @Test + public void testNoUpgradeActionsNeeded() { + when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(2); + layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager); + layoutVersionManager.finalizeLayoutFeatures(); + + verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt()); + } + +} \ No newline at end of file From bf68424b429ad1b9954f5829f83af0f2ba8d1244 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 6 Oct 2024 02:10:29 +0530 Subject: [PATCH 12/24] Added more tests for Table definition --- .../schema/SchemaVersionTableDefinition.java | 3 +- .../TestSchemaVersionTableDefinition.java | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java index 8a50645f6f5e..2aaa0d3c1e53 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java @@ -62,8 +62,7 @@ public void initializeSchema() throws SQLException { private void createSchemaVersionTable() throws SQLException { dslContext.createTableIfNotExists(SCHEMA_VERSION_TABLE_NAME) .column("version_number", SQLDataType.INTEGER.nullable(false)) - .column("applied_on", - SQLDataType.TIMESTAMP.defaultValue(DSL.currentTimestamp())) + .column("applied_on", SQLDataType.TIMESTAMP.defaultValue(DSL.currentTimestamp())) .execute(); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java new file mode 100644 index 000000000000..c64ffb91eb40 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java @@ -0,0 +1,104 @@ +package org.apache.hadoop.ozone.recon.persistence; + +import static org.hadoop.ozone.recon.schema.SchemaVersionTableDefinition.SCHEMA_VERSION_TABLE_NAME; +import static org.jooq.impl.DSL.name; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +/** + * Test class for SchemaVersionTableDefinition. + */ +public class TestSchemaVersionTableDefinition extends AbstractReconSqlDBTest { + + public TestSchemaVersionTableDefinition() { + super(); + } + + @Test + public void testSchemaVersionTableCreation() throws Exception { + Connection connection = getConnection(); + // Verify table definition + DatabaseMetaData metaData = connection.getMetaData(); + ResultSet resultSet = metaData.getColumns(null, null, + SCHEMA_VERSION_TABLE_NAME, null); + + List> expectedPairs = new ArrayList<>(); + + expectedPairs.add(new ImmutablePair<>("version_number", Types.INTEGER)); + expectedPairs.add(new ImmutablePair<>("applied_on", Types.TIMESTAMP)); + + List> actualPairs = new ArrayList<>(); + + while (resultSet.next()) { + actualPairs.add(new ImmutablePair<>(resultSet.getString("COLUMN_NAME"), + resultSet.getInt("DATA_TYPE"))); + } + + assertEquals(2, actualPairs.size(), "Unexpected number of columns"); + assertEquals(expectedPairs, actualPairs, "Column definitions do not match expected values."); + } + + @Test + public void testSchemaVersionCRUDOperations() throws SQLException { + Connection connection = getConnection(); + + DatabaseMetaData metaData = connection.getMetaData(); + ResultSet resultSet = metaData.getTables(null, null, + SCHEMA_VERSION_TABLE_NAME, null); + + while (resultSet.next()) { + assertEquals(SCHEMA_VERSION_TABLE_NAME, + resultSet.getString("TABLE_NAME")); + } + + DSLContext dslContext = DSL.using(connection); + + // Insert a new version record + dslContext.insertInto(DSL.table(SCHEMA_VERSION_TABLE_NAME)) + .columns(DSL.field(name("version_number")), DSL.field(name("applied_on"))) + .values(1, new Timestamp(System.currentTimeMillis())) + .execute(); + + // Read the inserted record + Record1 result = dslContext.select(DSL.field(name("version_number"), Integer.class)) + .from(DSL.table(SCHEMA_VERSION_TABLE_NAME)) + .fetchOne(); + + assertEquals(1, result.value1(), "The version number does not match the expected value."); + + // Update the version record + dslContext.update(DSL.table(SCHEMA_VERSION_TABLE_NAME)) + .set(DSL.field(name("version_number")), 2) + .execute(); + + // Read the updated record + result = dslContext.select(DSL.field(name("version_number"), Integer.class)) + .from(DSL.table(SCHEMA_VERSION_TABLE_NAME)) + .fetchOne(); + + assertEquals(2, result.value1(), "The updated version number does not match the expected value."); + + // Delete the version record + dslContext.deleteFrom(DSL.table(SCHEMA_VERSION_TABLE_NAME)) + .execute(); + + // Verify deletion + int count = dslContext.fetchCount(DSL.table(SCHEMA_VERSION_TABLE_NAME)); + assertEquals(0, count, "The table should be empty after deletion."); + } +} From 855ad9279d3281f2eb511ab8ca978ed8e65fd0a2 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 6 Oct 2024 02:18:37 +0530 Subject: [PATCH 13/24] Added missing licence certs --- .../schema/SchemaVersionTableDefinition.java | 2 +- .../TestSchemaVersionTableDefinition.java | 19 +++++++++++++++++++ .../TestReconLayoutVersionManager.java | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java index 2aaa0d3c1e53..f7e538f31ad8 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java index c64ffb91eb40..ab3c4f8e6ecb 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSchemaVersionTableDefinition.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.hadoop.ozone.recon.persistence; import static org.hadoop.ozone.recon.schema.SchemaVersionTableDefinition.SCHEMA_VERSION_TABLE_NAME; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java index 39da62a693d4..e80cb120ab60 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.hadoop.ozone.recon.upgrade; import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager; From 6ea93a9aacad30918d6e3a220400b7a321763a38 Mon Sep 17 00:00:00 2001 From: arafat Date: Sun, 6 Oct 2024 12:52:55 +0530 Subject: [PATCH 14/24] TestCase updation --- .../upgrade/TestReconLayoutVersionManager.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java index e80cb120ab60..5c529b9c94bb 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -121,20 +121,6 @@ public void testNoLayoutFeatures() { verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt()); } - /** - * Handling Invalid Versions - * Higher MLV than Available SLV: Test the behavior when the current MLV (Metadata Layout Version) - * is set higher than the maximum version of ReconLayoutFeature. This scenario simulates - * an inconsistent state where the system’s metadata versioning is beyond the known layout features. - */ - @Test - public void testHigherMLVThanSLV() { - when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(5); - layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager); - assertEquals(2, layoutVersionManager.getCurrentSLV()); // Assuming 2 is the max SLV in mocked features - assertEquals(5, layoutVersionManager.getCurrentMLV()); - } - /** * Tests the scenario where an upgrade action fails. Ensures that if an upgrade action * throws an exception, the schema version is not updated. From d95a1a7ee98152fc2bf8e4e01985247498c4171b Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 7 Oct 2024 12:53:39 +0530 Subject: [PATCH 15/24] Replaced the occurance of Schema Layout Version to Software Layout Version --- .../recon/upgrade/ReconLayoutVersionManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index 00cbe9b02a8d..8dd0e358ab83 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -30,7 +30,7 @@ /** * ReconLayoutVersionManager is responsible for managing the layout version of the Recon service. - * It determines the current Metadata Layout Version (MLV) and Schema Layout Version (SLV) of the + * It determines the current Metadata Layout Version (MLV) and Software Layout Version (SLV) of the * Recon service, and finalizes the layout features that need to be upgraded. */ public class ReconLayoutVersionManager { @@ -40,7 +40,7 @@ public class ReconLayoutVersionManager { // Current Metadata Layout Version (MLV) of the Recon service private int currentMLV; - // Schema Layout Version (SLV) of the Recon service + // Software Layout Version (SLV) of the Recon service private final int currentSLV; private final ReconSchemaVersionTableManager schemaVersionTableManager; @@ -67,8 +67,8 @@ private int determineMLV() { } /** - * Determines the Schema Layout Version (SLV) based on the latest feature version. - * @return The Schema Layout Version (SLV). + * Determines the Software Layout Version (SLV) based on the latest feature version. + * @return The Software Layout Version (SLV). */ private int determineSLV() { return Arrays.stream(ReconLayoutFeature.values()) @@ -129,8 +129,8 @@ protected List getRegisteredFeatures() { } /** - * Updates the Schema Layout Version (SLV) in the database after finalizing a feature. - * @param newVersion The new Schema Layout Version (SLV) to set. + * Updates the Software Layout Version (SLV) in the database after finalizing a feature. + * @param newVersion The new Software Layout Version (SLV) to set. */ private void updateSchemaVersion(int newVersion) { // Logic to update the MLV in the database From 29fe85b7034eefc67f066745a97144f1fb8e613d Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 17 Oct 2024 00:24:56 +0530 Subject: [PATCH 16/24] Refactored Recon layout feature upgrade framework to support annotation-based dynamic upgrade actions --- .../recon/upgrade/Feature1UpgradeAction.java | 14 ---- .../Feature1UpgradeActionForAutoFinalise.java | 25 +++++++ .../recon/upgrade/Feature2UpgradeAction.java | 15 ---- .../Feature2UpgradeActionForAutoFinalise.java | 26 +++++++ .../recon/upgrade/Feature3UpgradeAction.java | 14 ---- .../Feature3UpgradeActionForAutoFinalise.java | 26 +++++++ .../Feature3UpgradeActionForTestAction.java | 23 ++++++ .../upgrade/InitialLayoutUpgradeAction.java | 30 -------- .../recon/upgrade/ReconLayoutFeature.java | 73 ++++++++++++++++--- .../upgrade/ReconLayoutVersionManager.java | 45 +++++++----- .../recon/upgrade/ReconUpgradeAction.java | 26 ++++++- .../recon/upgrade/UpgradeActionRecon.java | 70 ++++++++++++++++++ .../TestReconLayoutVersionManager.java | 31 +++++--- 13 files changed, 304 insertions(+), 114 deletions(-) delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java deleted file mode 100644 index dc7f17c0cfb6..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Feature1UpgradeAction implements ReconUpgradeAction { - private static final Logger LOG = LoggerFactory.getLogger(Feature1UpgradeAction.class); - @Override - public void execute() throws Exception { - // Logic for upgrading to version 1 - LOG.info("Executing Feature 1 upgrade:"); - // Implement the database schema update or other upgrade logic here - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java new file mode 100644 index 000000000000..f22700c69f45 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java @@ -0,0 +1,25 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_1; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE; + +@UpgradeActionRecon(feature = FEATURE_1, type = AUTO_FINALIZE) +public class Feature1UpgradeActionForAutoFinalise implements ReconUpgradeAction { + + private static final Logger LOG = LoggerFactory.getLogger( + Feature1UpgradeActionForAutoFinalise.class); + + @Override + public void execute() throws Exception { + // Logic for Feature 1 upgrade + LOG.info("Executing Feature 1 upgrade script for action type AUTO_FINALIZE..."); + } + + @Override + public UpgradeActionType getType() { + return AUTO_FINALIZE; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java deleted file mode 100644 index 1159bbc20306..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Feature2UpgradeAction implements ReconUpgradeAction { - private static final Logger LOG = LoggerFactory.getLogger(Feature2UpgradeAction.class); - - @Override - public void execute() throws Exception { - // Logic for upgrading to version 2 - LOG.info("Executing Feature 2 upgrade"); - // Implement the database schema update or other upgrade logic here - } -} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java new file mode 100644 index 000000000000..e478e7ed2a1f --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java @@ -0,0 +1,26 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_2; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.TEST_ACTION; + +@UpgradeActionRecon(feature = FEATURE_2, type = AUTO_FINALIZE) +public class Feature2UpgradeActionForAutoFinalise implements ReconUpgradeAction { + + private static final Logger LOG = LoggerFactory.getLogger( + Feature2UpgradeActionForAutoFinalise.class); + + @Override + public void execute() throws Exception { + // Logic for Feature 2 upgrade + LOG.info("Executing Feature 2 upgrade script for action type AUTO_FINALIZE..."); + } + + @Override + public UpgradeActionType getType() { + return TEST_ACTION; + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java deleted file mode 100644 index c92fe3510457..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Feature3UpgradeAction implements ReconUpgradeAction { - private static final Logger LOG = LoggerFactory.getLogger(Feature3UpgradeAction.class); - @Override - public void execute() throws Exception { - // Logic for upgrading to version 3 - LOG.info("Executing Feature 3 upgrade"); - // Implement the database schema update or other upgrade logic here - } -} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java new file mode 100644 index 000000000000..1724daabddfc --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java @@ -0,0 +1,26 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_3; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.TEST_ACTION; + +@UpgradeActionRecon(feature = FEATURE_3, type = AUTO_FINALIZE) +public class Feature3UpgradeActionForAutoFinalise implements ReconUpgradeAction { + + private static final Logger LOG = LoggerFactory.getLogger( + Feature3UpgradeActionForAutoFinalise.class); + + @Override + public void execute() throws Exception { + // Logic for Feature 3 upgrade + LOG.info("Executing Feature 3 upgrade script for action type AUTO_FINALIZE..."); + } + + @Override + public UpgradeActionType getType() { + return AUTO_FINALIZE; + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java new file mode 100644 index 000000000000..dd140e9ab78e --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java @@ -0,0 +1,23 @@ +package org.apache.hadoop.ozone.recon.upgrade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_3; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.TEST_ACTION; + +@UpgradeActionRecon(feature = FEATURE_3, type = TEST_ACTION) +public class Feature3UpgradeActionForTestAction implements ReconUpgradeAction { + private static final Logger LOG = LoggerFactory.getLogger( + Feature3UpgradeActionForTestAction.class); + + @Override + public void execute() throws Exception { + LOG.info("Executing Feature 3 upgrade script for action type TEST_ACTION..."); + } + + @Override + public UpgradeActionType getType() { + return TEST_ACTION; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java deleted file mode 100644 index 99b79bb032dc..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.ozone.recon.upgrade; - -public class InitialLayoutUpgradeAction implements ReconUpgradeAction { - - @Override - public void execute() throws Exception { - // No actions are performed for version 0. - // This version represents the introduction of schema versioning for Recon. - // Therefore, we do not perform any upgrade actions for now. - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java index 7c46af95d556..45a62e7a30b7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java @@ -19,24 +19,31 @@ package org.apache.hadoop.ozone.recon.upgrade; +import org.reflections.Reflections; + +import java.util.EnumMap; +import java.util.Optional; +import java.util.Set; + /** * Enum representing Recon layout features with their version, description, * and associated upgrade action to be executed during an upgrade. */ public enum ReconLayoutFeature { - INITIAL_VERSION(0, "Initial Layout Version", new InitialLayoutUpgradeAction()), - FEATURE_1(1, "Description for Feature 1", new Feature1UpgradeAction()), - FEATURE_2(2, "Description for Feature 2", new Feature2UpgradeAction()), - FEATURE_3(3, "Description for Feature 3", new Feature3UpgradeAction()); + // Represents the starting point for Recon's layout versioning system. + INITIAL_VERSION(0, "Recon Layout Versioning Introduction"), + + FEATURE_1(1, "Description for Feature 1"), + FEATURE_2(2, "Description for Feature 2"), + FEATURE_3(3, "Description for Feature 3"); private final int version; private final String description; - private final ReconUpgradeAction upgradeAction; + private final EnumMap actions = new EnumMap<>(ReconUpgradeAction.UpgradeActionType.class); - ReconLayoutFeature(int version, String description, ReconUpgradeAction upgradeAction) { + ReconLayoutFeature(final int version, String description) { this.version = version; this.description = description; - this.upgradeAction = upgradeAction; } public int getVersion() { @@ -47,7 +54,53 @@ public String getDescription() { return description; } - public ReconUpgradeAction getUpgradeAction() { - return upgradeAction; + /** + * Retrieves the upgrade action for the specified {@link ReconUpgradeAction.UpgradeActionType}. + * + * @param type The type of the upgrade action (e.g., AUTO_FINALIZE). + * @return An {@link Optional} containing the upgrade action if present. + */ + public Optional getAction(ReconUpgradeAction.UpgradeActionType type) { + return Optional.ofNullable(actions.get(type)); + } + + /** + * Associates a given upgrade action with a specific upgrade phase for this feature. + * + * @param type The phase/type of the upgrade action. + * @param action The upgrade action to associate with this feature. + */ + public void addAction(ReconUpgradeAction.UpgradeActionType type, ReconUpgradeAction action) { + actions.put(type, action); + } + + /** + * Scans the classpath for all classes annotated with {@link UpgradeActionRecon} + * and registers their upgrade actions for the corresponding feature and phase. + * This method dynamically loads and registers all upgrade actions based on their + * annotations. + */ + public static void registerUpgradeActions() { + Reflections reflections = new Reflections("org.apache.hadoop.ozone.recon.upgrade"); + Set> actionClasses = reflections.getTypesAnnotatedWith(UpgradeActionRecon.class); + + for (Class actionClass : actionClasses) { + try { + ReconUpgradeAction action = (ReconUpgradeAction) actionClass.getDeclaredConstructor().newInstance(); + UpgradeActionRecon annotation = actionClass.getAnnotation(UpgradeActionRecon.class); + annotation.feature().addAction(annotation.type(), action); + } catch (Exception e) { + throw new RuntimeException("Failed to register upgrade action: " + actionClass.getSimpleName(), e); + } + } + } + + /** + * Returns the list of all layout feature values. + * + * @return An array of all {@link ReconLayoutFeature} values. + */ + public static ReconLayoutFeature[] getValues() { + return values(); } -} +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index 8dd0e358ab83..e1abf7d9dc82 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -26,6 +26,7 @@ import javax.inject.Inject; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -35,27 +36,20 @@ */ public class ReconLayoutVersionManager { - private static final Logger LOG = - LoggerFactory.getLogger(ReconLayoutVersionManager.class); + private static final Logger LOG = LoggerFactory.getLogger(ReconLayoutVersionManager.class); - // Current Metadata Layout Version (MLV) of the Recon service + private final ReconSchemaVersionTableManager schemaVersionTableManager; + + // Metadata Layout Version (MLV) of the Recon Metadata on disk private int currentMLV; // Software Layout Version (SLV) of the Recon service private final int currentSLV; - private final ReconSchemaVersionTableManager schemaVersionTableManager; public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager) { this.schemaVersionTableManager = schemaVersionTableManager; this.currentMLV = determineMLV(); this.currentSLV = determineSLV(); - } - - public int getCurrentMLV() { - return currentMLV; - } - - public int getCurrentSLV() { - return currentSLV; + ReconLayoutFeature.registerUpgradeActions(); // Register actions via annotation } /** @@ -71,7 +65,7 @@ private int determineMLV() { * @return The Software Layout Version (SLV). */ private int determineSLV() { - return Arrays.stream(ReconLayoutFeature.values()) + return Arrays.stream(ReconLayoutFeature.getValues()) .mapToInt(ReconLayoutFeature::getVersion) .max() .orElse(0); // Default to 0 if no features are defined @@ -87,13 +81,16 @@ public void finalizeLayoutFeatures() { for (ReconLayoutFeature feature : featuresToFinalize) { try { - // Execute the upgrade action for this feature - feature.getUpgradeAction().execute(); - // Update the MLV in the database after successful execution - updateSchemaVersion(feature.getVersion()); - LOG.info("Feature " + feature.getVersion() + " finalized successfully."); + // Fetch only the AUTO_FINALIZE action for the feature + Optional action = feature.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE); + if (action.isPresent()) { + // Execute the upgrade action & update the schema version in the DB + action.get().execute(); + updateSchemaVersion(feature.getVersion()); + LOG.info("Feature {} finalized successfully.", feature.getVersion()); + } } catch (Exception e) { - LOG.info("Failed to finalize feature " + feature.getVersion() + ": " + e.getMessage()); + LOG.error("Failed to finalize feature {}: {}", feature.getVersion(), e.getMessage()); break; } } @@ -101,7 +98,6 @@ public void finalizeLayoutFeatures() { /** * Returns a list of ReconLayoutFeature objects that are registered for finalization. - * @return List of ReconLayoutFeature objects that are registered for finalization. */ protected List getRegisteredFeatures() { List allFeatures = @@ -139,4 +135,13 @@ private void updateSchemaVersion(int newVersion) { schemaVersionTableManager.updateSchemaVersion(newVersion); LOG.info("MLV updated to: " + newVersion); } + + public int getCurrentMLV() { + return currentMLV; + } + + public int getCurrentSLV() { + return currentSLV; + } + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java index 36109758ffdd..fd9502a368ea 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -23,5 +23,29 @@ * ReconUpgradeAction is an interface for executing upgrade actions in Recon. */ public interface ReconUpgradeAction { + + /** + * Defines the different phases during which upgrade actions can be executed. + * Each action type corresponds to a specific point in the upgrade process: + * + * - AUTO_FINALIZE: This action is executed automatically during the startup + * of Recon when it finalizes the layout upgrade. It ensures that all necessary + * upgrades or schema changes are applied to bring the system in sync with + * the latest version. + */ + enum UpgradeActionType { + AUTO_FINALIZE, + // For testing purpose only will remove this before merge + TEST_ACTION + } + + /** + * Execute the upgrade action. + */ void execute() throws Exception; + + /** + * Provides the type of upgrade phase (e.g., AUTO_FINALIZE). + */ + UpgradeActionType getType(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java new file mode 100644 index 000000000000..93ce4ec46b56 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * The {@code UpgradeActionRecon} annotation is used to specify + * upgrade actions that should be executed during particular phases + * of the Recon service layout upgrade process. + * + *

This annotation can be used to associate an upgrade action + * class with a specific layout feature and upgrade phase. The + * framework will dynamically discover these annotated upgrade + * actions and execute them based on the feature's version and + * the defined action type (e.g., {@link ReconUpgradeAction.UpgradeActionType#AUTO_FINALIZE}). + * + *

The annotation is retained at runtime, allowing the reflection-based + * mechanism to scan for annotated classes, register the associated actions, + * and execute them as necessary during the layout upgrade process. + * + * Example usage: + * + * 

+ * {@code
+ *
+ * @UpgradeActionRecon(feature = FEATURE_NAME, type = AUTO_FINALIZE)
+ *  public class FeatureNameUpgradeAction implements ReconUpgradeAction {
+ *     @Override
+ *     public void execute() throws Exception {
+ *       // Custom upgrade logic for FEATURE_1
+ *     }
+ *  }
+ * }
+ *
+ */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface UpgradeActionRecon { + + /** + * Defines the layout feature this upgrade action is associated with. + */ + ReconLayoutFeature feature(); + + /** + * Defines the type of upgrade phase during which the action should be executed. + */ + ReconUpgradeAction.UpgradeActionType type(); +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java index 5c529b9c94bb..58aa1cb7e924 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.*; @@ -39,6 +40,7 @@ public class TestReconLayoutVersionManager { private ReconSchemaVersionTableManager schemaVersionTableManager; private ReconLayoutVersionManager layoutVersionManager; private static MockedStatic mockedEnum; + private static MockedStatic mockedEnumUpgradeActionType; @BeforeEach public void setUp() { @@ -47,16 +49,18 @@ public void setUp() { // Mocking ReconLayoutFeature.values() to return custom enum instances mockedEnum = mockStatic(ReconLayoutFeature.class); + ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); when(feature1.getVersion()).thenReturn(1); - when(feature1.getUpgradeAction()).thenReturn(() -> { - // No-op for testing - }); + ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action1)); + ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); when(feature2.getVersion()).thenReturn(2); - when(feature2.getUpgradeAction()).thenReturn(() -> { - // No-op for testing - }); + ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); + when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action2)); // Define the custom features to be returned mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2}); @@ -68,6 +72,9 @@ public void setUp() { public void tearDown() { // Close the static mock after each test to deregister it mockedEnum.close(); + if (mockedEnumUpgradeActionType != null) { + mockedEnumUpgradeActionType.close(); + } } /** @@ -137,7 +144,8 @@ public void testUpgradeActionFailure() throws Exception { // Simulate an exception being thrown during the upgrade action execution doThrow(new RuntimeException("Upgrade failed")).when(action1).execute(); - when(feature1.getUpgradeAction()).thenReturn(action1); + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action1)); // Mock the static values method to return the custom feature mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1}); @@ -161,17 +169,20 @@ public void testUpgradeActionExecutionOrder() throws Exception { ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); when(feature1.getVersion()).thenReturn(1); ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); - when(feature1.getUpgradeAction()).thenReturn(action1); + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action1)); ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); when(feature2.getVersion()).thenReturn(2); ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); - when(feature2.getUpgradeAction()).thenReturn(action2); + when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action2)); ReconLayoutFeature feature3 = mock(ReconLayoutFeature.class); when(feature3.getVersion()).thenReturn(3); ReconUpgradeAction action3 = mock(ReconUpgradeAction.class); - when(feature3.getUpgradeAction()).thenReturn(action3); + when(feature3.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action3)); // Mock the static values method to return custom features in a jumbled order mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature2, feature3, feature1}); From 287cafa261e4507b53c5ef70e90274e54f937bf3 Mon Sep 17 00:00:00 2001 From: arafat Date: Sat, 19 Oct 2024 00:56:02 +0530 Subject: [PATCH 17/24] Added a new test case --- .../recon/upgrade/ReconLayoutFeature.java | 5 +- .../upgrade/ReconLayoutVersionManager.java | 2 +- .../TestReconLayoutVersionManager.java | 59 +++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java index 45a62e7a30b7..9c0dc0321350 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java @@ -39,7 +39,8 @@ public enum ReconLayoutFeature { private final int version; private final String description; - private final EnumMap actions = new EnumMap<>(ReconUpgradeAction.UpgradeActionType.class); + private final EnumMap actions = + new EnumMap<>(ReconUpgradeAction.UpgradeActionType.class); ReconLayoutFeature(final int version, String description) { this.version = version; @@ -101,6 +102,6 @@ public static void registerUpgradeActions() { * @return An array of all {@link ReconLayoutFeature} values. */ public static ReconLayoutFeature[] getValues() { - return values(); + return ReconLayoutFeature.values(); } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index e1abf7d9dc82..b004b26c3ab2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -65,7 +65,7 @@ private int determineMLV() { * @return The Software Layout Version (SLV). */ private int determineSLV() { - return Arrays.stream(ReconLayoutFeature.getValues()) + return Arrays.stream(ReconLayoutFeature.values()) .mapToInt(ReconLayoutFeature::getVersion) .max() .orElse(0); // Default to 0 if no features are defined diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java index 58aa1cb7e924..858d7f69f7ff 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -210,4 +210,63 @@ public void testNoUpgradeActionsNeeded() { verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt()); } + /** + * Tests the scenario where the first two features are finalized, + * and then a third feature is introduced. Ensures that only the + * newly introduced feature is finalized while the previously + * finalized features are skipped. + */ + @Test + public void testFinalizingNewFeatureWithoutReFinalizingPreviousFeatures() throws Exception { + // Step 1: Finalize the first two features. + when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(0); + + // Mock the first two features. + ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); + when(feature1.getVersion()).thenReturn(1); + ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action1)); + + ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); + when(feature2.getVersion()).thenReturn(2); + ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); + when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action2)); + + mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2}); + + // Finalize the first two features. + layoutVersionManager.finalizeLayoutFeatures(); + + // Verify that the schema versions for the first two features were updated. + verify(schemaVersionTableManager, times(1)).updateSchemaVersion(1); + verify(schemaVersionTableManager, times(1)).updateSchemaVersion(2); + + // Step 2: Introduce a new feature (Feature 3). + ReconLayoutFeature feature3 = mock(ReconLayoutFeature.class); + when(feature3.getVersion()).thenReturn(3); + ReconUpgradeAction action3 = mock(ReconUpgradeAction.class); + when(feature3.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + .thenReturn(Optional.of(action3)); + + mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2, feature3}); + + // Update schema version to simulate that features 1 and 2 have already been finalized. + when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(2); + + // Finalize again, but only feature 3 should be finalized. + layoutVersionManager.finalizeLayoutFeatures(); + + // Verify that the schema version for feature 3 was updated. + verify(schemaVersionTableManager, times(1)).updateSchemaVersion(3); + + // Verify that action1 and action2 were not executed again. + verify(action1, times(1)).execute(); // Still should have been executed only once + verify(action2, times(1)).execute(); // Still should have been executed only once + + // Verify that the upgrade action for feature 3 was executed. + verify(action3, times(1)).execute(); + } + } \ No newline at end of file From 81e62f07948f77934cc3e6973c38b5ea85aef86b Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 22 Oct 2024 12:18:18 +0530 Subject: [PATCH 18/24] Made final changes --- .../codegen/ReconSchemaGenerationModule.java | 7 ++++- .../recon/ReconSchemaVersionTableManager.java | 2 +- .../Feature1UpgradeActionForAutoFinalise.java | 25 ---------------- .../Feature2UpgradeActionForAutoFinalise.java | 26 ----------------- .../Feature3UpgradeActionForAutoFinalise.java | 26 ----------------- .../Feature3UpgradeActionForTestAction.java | 23 --------------- .../recon/upgrade/ReconLayoutFeature.java | 8 ++--- .../upgrade/ReconLayoutVersionManager.java | 1 - .../ozone/recon/upgrade/package-info.java | 29 +++++++++++++++++++ .../TestReconLayoutVersionManager.java | 20 +++++++++---- 10 files changed, 53 insertions(+), 114 deletions(-) delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/package-info.java diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java index 9a06a20fdffd..d59ab8acd6ba 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java @@ -17,7 +17,12 @@ */ package org.hadoop.ozone.recon.codegen; -import org.hadoop.ozone.recon.schema.*; +import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition; +import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition; +import org.hadoop.ozone.recon.schema.ReconSchemaDefinition; +import org.hadoop.ozone.recon.schema.StatsSchemaDefinition; +import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; +import org.hadoop.ozone.recon.schema.SchemaVersionTableDefinition; import com.google.inject.AbstractModule; import com.google.inject.multibindings.Multibinder; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index dd2b52580683..72855be47720 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -104,4 +104,4 @@ public void updateSchemaVersion(int newVersion) { public DataSource getDataSource() { return dataSource; } -} \ No newline at end of file +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java deleted file mode 100644 index f22700c69f45..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeActionForAutoFinalise.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_1; -import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE; - -@UpgradeActionRecon(feature = FEATURE_1, type = AUTO_FINALIZE) -public class Feature1UpgradeActionForAutoFinalise implements ReconUpgradeAction { - - private static final Logger LOG = LoggerFactory.getLogger( - Feature1UpgradeActionForAutoFinalise.class); - - @Override - public void execute() throws Exception { - // Logic for Feature 1 upgrade - LOG.info("Executing Feature 1 upgrade script for action type AUTO_FINALIZE..."); - } - - @Override - public UpgradeActionType getType() { - return AUTO_FINALIZE; - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java deleted file mode 100644 index e478e7ed2a1f..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeActionForAutoFinalise.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_2; -import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE; -import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.TEST_ACTION; - -@UpgradeActionRecon(feature = FEATURE_2, type = AUTO_FINALIZE) -public class Feature2UpgradeActionForAutoFinalise implements ReconUpgradeAction { - - private static final Logger LOG = LoggerFactory.getLogger( - Feature2UpgradeActionForAutoFinalise.class); - - @Override - public void execute() throws Exception { - // Logic for Feature 2 upgrade - LOG.info("Executing Feature 2 upgrade script for action type AUTO_FINALIZE..."); - } - - @Override - public UpgradeActionType getType() { - return TEST_ACTION; - } -} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java deleted file mode 100644 index 1724daabddfc..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForAutoFinalise.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_3; -import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE; -import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.TEST_ACTION; - -@UpgradeActionRecon(feature = FEATURE_3, type = AUTO_FINALIZE) -public class Feature3UpgradeActionForAutoFinalise implements ReconUpgradeAction { - - private static final Logger LOG = LoggerFactory.getLogger( - Feature3UpgradeActionForAutoFinalise.class); - - @Override - public void execute() throws Exception { - // Logic for Feature 3 upgrade - LOG.info("Executing Feature 3 upgrade script for action type AUTO_FINALIZE..."); - } - - @Override - public UpgradeActionType getType() { - return AUTO_FINALIZE; - } -} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java deleted file mode 100644 index dd140e9ab78e..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeActionForTestAction.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.apache.hadoop.ozone.recon.upgrade; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.FEATURE_3; -import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.TEST_ACTION; - -@UpgradeActionRecon(feature = FEATURE_3, type = TEST_ACTION) -public class Feature3UpgradeActionForTestAction implements ReconUpgradeAction { - private static final Logger LOG = LoggerFactory.getLogger( - Feature3UpgradeActionForTestAction.class); - - @Override - public void execute() throws Exception { - LOG.info("Executing Feature 3 upgrade script for action type TEST_ACTION..."); - } - - @Override - public UpgradeActionType getType() { - return TEST_ACTION; - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java index 9c0dc0321350..6d4f137bbcfb 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java @@ -31,11 +31,7 @@ */ public enum ReconLayoutFeature { // Represents the starting point for Recon's layout versioning system. - INITIAL_VERSION(0, "Recon Layout Versioning Introduction"), - - FEATURE_1(1, "Description for Feature 1"), - FEATURE_2(2, "Description for Feature 2"), - FEATURE_3(3, "Description for Feature 3"); + INITIAL_VERSION(0, "Recon Layout Versioning Introduction"); private final int version; private final String description; @@ -104,4 +100,4 @@ public static void registerUpgradeActions() { public static ReconLayoutFeature[] getValues() { return ReconLayoutFeature.values(); } -} \ No newline at end of file +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index b004b26c3ab2..46332d2b0a28 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -23,7 +23,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; import java.util.Arrays; import java.util.List; import java.util.Optional; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/package-info.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/package-info.java new file mode 100644 index 000000000000..56a94b1f84aa --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * This package contains classes and interfaces for handling + * upgrade actions in Apache Ozone Recon. + * + * The main interface {@link org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction} + * defines the structure for actions that need to be executed during an upgrade + * process in Recon. The actions can be triggered automatically + * during startup to ensure the correct version of the schema or + * layout is applied. + */ +package org.apache.hadoop.ozone.recon.upgrade; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java index 858d7f69f7ff..216491572989 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -20,17 +20,26 @@ package org.apache.hadoop.ozone.recon.upgrade; import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager; -import org.junit.jupiter.api.*; import org.mockito.InOrder; import org.mockito.MockedStatic; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; import java.util.Arrays; import java.util.List; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; + /** * Tests for ReconLayoutVersionManager. @@ -39,8 +48,8 @@ public class TestReconLayoutVersionManager { private ReconSchemaVersionTableManager schemaVersionTableManager; private ReconLayoutVersionManager layoutVersionManager; - private static MockedStatic mockedEnum; - private static MockedStatic mockedEnumUpgradeActionType; + private MockedStatic mockedEnum; + private MockedStatic mockedEnumUpgradeActionType; @BeforeEach public void setUp() { @@ -49,6 +58,7 @@ public void setUp() { // Mocking ReconLayoutFeature.values() to return custom enum instances mockedEnum = mockStatic(ReconLayoutFeature.class); + mockedEnumUpgradeActionType = mockStatic(ReconUpgradeAction.UpgradeActionType.class); ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); when(feature1.getVersion()).thenReturn(1); @@ -269,4 +279,4 @@ public void testFinalizingNewFeatureWithoutReFinalizingPreviousFeatures() throws verify(action3, times(1)).execute(); } -} \ No newline at end of file +} From 0afebffabdbd2b8da4639f594448f5967a310355 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 23 Oct 2024 17:25:36 +0530 Subject: [PATCH 19/24] Final changes for review --- .../recon/ReconSchemaVersionTableManager.java | 2 +- .../upgrade/InitialVersionUpgradeAction.java | 37 +++++++++++++++++++ .../upgrade/ReconLayoutVersionManager.java | 17 ++------- .../recon/upgrade/ReconUpgradeAction.java | 4 +- .../recon/upgrade/UpgradeActionRecon.java | 2 +- .../TestReconLayoutVersionManager.java | 18 ++++----- 6 files changed, 52 insertions(+), 28 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java index 72855be47720..81fefc334acf 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java @@ -59,7 +59,7 @@ public int getCurrentSchemaVersion() { .fetchOptional() .map(record -> record.get( DSL.field(name("version_number"), Integer.class))) - .orElse(0); // Return 0 if no version is found + .orElse(-1); // Return -1 if no version is found } catch (Exception e) { LOG.error("Failed to fetch the current schema version.", e); return 0; // Return 0 if there is an exception diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java new file mode 100644 index 000000000000..b76b7ac28c08 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.INITIAL_VERSION; +import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.FINALIZE; + +@UpgradeActionRecon(feature = INITIAL_VERSION, type = FINALIZE) +public class InitialVersionUpgradeAction implements ReconUpgradeAction { + + @Override + public void execute() throws Exception { + // This action does nothing for now. + } + + @Override + public UpgradeActionType getType() { + return FINALIZE; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index 46332d2b0a28..bb345da8cc20 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -81,12 +81,12 @@ public void finalizeLayoutFeatures() { for (ReconLayoutFeature feature : featuresToFinalize) { try { // Fetch only the AUTO_FINALIZE action for the feature - Optional action = feature.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE); + Optional action = feature.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE); if (action.isPresent()) { // Execute the upgrade action & update the schema version in the DB action.get().execute(); updateSchemaVersion(feature.getVersion()); - LOG.info("Feature {} finalized successfully.", feature.getVersion()); + LOG.info("Feature versioned {} finalized successfully.", feature.getVersion()); } } catch (Exception e) { LOG.error("Failed to finalize feature {}: {}", feature.getVersion(), e.getMessage()); @@ -105,18 +105,7 @@ protected List getRegisteredFeatures() { LOG.info("Current MLV: {}. SLV: {}. Checking features for registration...", currentMLV, currentSLV); List registeredFeatures = allFeatures.stream() - .filter(feature -> { - boolean shouldRegister = feature.getVersion() > currentMLV; - if (shouldRegister) { - LOG.info("Feature {} (version {}) is registered for finalization.", - feature.name(), feature.getVersion()); - } else { - LOG.info( - "Feature {} (version {}) is NOT registered for finalization.", - feature.name(), feature.getVersion()); - } - return shouldRegister; - }) + .filter(feature -> feature.getVersion() > currentMLV) .sorted((a, b) -> Integer.compare(a.getVersion(), b.getVersion())) // Sort by version in ascending order .collect(Collectors.toList()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java index fd9502a368ea..f77ab2086d0a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java @@ -34,9 +34,7 @@ public interface ReconUpgradeAction { * the latest version. */ enum UpgradeActionType { - AUTO_FINALIZE, - // For testing purpose only will remove this before merge - TEST_ACTION + FINALIZE } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java index 93ce4ec46b56..8c50e3e36884 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java @@ -33,7 +33,7 @@ * class with a specific layout feature and upgrade phase. The * framework will dynamically discover these annotated upgrade * actions and execute them based on the feature's version and - * the defined action type (e.g., {@link ReconUpgradeAction.UpgradeActionType#AUTO_FINALIZE}). + * the defined action type (e.g., {@link ReconUpgradeAction.UpgradeActionType#FINALIZE}). * *

The annotation is retained at runtime, allowing the reflection-based * mechanism to scan for annotated classes, register the associated actions, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java index 216491572989..9ab1e3f3ec58 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java @@ -63,13 +63,13 @@ public void setUp() { ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); when(feature1.getVersion()).thenReturn(1); ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); - when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action1)); ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); when(feature2.getVersion()).thenReturn(2); ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); - when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action2)); // Define the custom features to be returned @@ -154,7 +154,7 @@ public void testUpgradeActionFailure() throws Exception { // Simulate an exception being thrown during the upgrade action execution doThrow(new RuntimeException("Upgrade failed")).when(action1).execute(); - when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action1)); // Mock the static values method to return the custom feature @@ -179,19 +179,19 @@ public void testUpgradeActionExecutionOrder() throws Exception { ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); when(feature1.getVersion()).thenReturn(1); ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); - when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action1)); ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); when(feature2.getVersion()).thenReturn(2); ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); - when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action2)); ReconLayoutFeature feature3 = mock(ReconLayoutFeature.class); when(feature3.getVersion()).thenReturn(3); ReconUpgradeAction action3 = mock(ReconUpgradeAction.class); - when(feature3.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature3.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action3)); // Mock the static values method to return custom features in a jumbled order @@ -235,13 +235,13 @@ public void testFinalizingNewFeatureWithoutReFinalizingPreviousFeatures() throws ReconLayoutFeature feature1 = mock(ReconLayoutFeature.class); when(feature1.getVersion()).thenReturn(1); ReconUpgradeAction action1 = mock(ReconUpgradeAction.class); - when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature1.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action1)); ReconLayoutFeature feature2 = mock(ReconLayoutFeature.class); when(feature2.getVersion()).thenReturn(2); ReconUpgradeAction action2 = mock(ReconUpgradeAction.class); - when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature2.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action2)); mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2}); @@ -257,7 +257,7 @@ public void testFinalizingNewFeatureWithoutReFinalizingPreviousFeatures() throws ReconLayoutFeature feature3 = mock(ReconLayoutFeature.class); when(feature3.getVersion()).thenReturn(3); ReconUpgradeAction action3 = mock(ReconUpgradeAction.class); - when(feature3.getAction(ReconUpgradeAction.UpgradeActionType.AUTO_FINALIZE)) + when(feature3.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE)) .thenReturn(Optional.of(action3)); mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2, feature3}); From c538a787492c30e9c0522cf17064a2d79f33631f Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 23 Oct 2024 17:36:11 +0530 Subject: [PATCH 20/24] Removed the occurance of AUTO_FINALIZE in comments and made changes to handle INITIAL_VERSION in ReconLayoutVersionManager --- .../hadoop/ozone/recon/upgrade/ReconLayoutFeature.java | 2 +- .../ozone/recon/upgrade/ReconLayoutVersionManager.java | 9 ++++++++- .../hadoop/ozone/recon/upgrade/ReconUpgradeAction.java | 4 ++-- .../hadoop/ozone/recon/upgrade/UpgradeActionRecon.java | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java index 6d4f137bbcfb..96969c9f3d22 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java @@ -54,7 +54,7 @@ public String getDescription() { /** * Retrieves the upgrade action for the specified {@link ReconUpgradeAction.UpgradeActionType}. * - * @param type The type of the upgrade action (e.g., AUTO_FINALIZE). + * @param type The type of the upgrade action (e.g., FINALIZE). * @return An {@link Optional} containing the upgrade action if present. */ public Optional getAction(ReconUpgradeAction.UpgradeActionType type) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java index bb345da8cc20..34651dde5f60 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java @@ -80,7 +80,14 @@ public void finalizeLayoutFeatures() { for (ReconLayoutFeature feature : featuresToFinalize) { try { - // Fetch only the AUTO_FINALIZE action for the feature + // If the feature is INITIAL_VERSION, skip executing any action and just update the schema version + if (feature == ReconLayoutFeature.INITIAL_VERSION) { + updateSchemaVersion(0); + LOG.info("INITIAL_VERSION feature processed by setting schema version to 0."); + continue; + } + + // Fetch only the FINALIZE action for the feature Optional action = feature.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE); if (action.isPresent()) { // Execute the upgrade action & update the schema version in the DB diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java index f77ab2086d0a..f09cdf8e1f28 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java @@ -28,7 +28,7 @@ public interface ReconUpgradeAction { * Defines the different phases during which upgrade actions can be executed. * Each action type corresponds to a specific point in the upgrade process: * - * - AUTO_FINALIZE: This action is executed automatically during the startup + * - FINALIZE: This action is executed automatically during the startup * of Recon when it finalizes the layout upgrade. It ensures that all necessary * upgrades or schema changes are applied to bring the system in sync with * the latest version. @@ -43,7 +43,7 @@ enum UpgradeActionType { void execute() throws Exception; /** - * Provides the type of upgrade phase (e.g., AUTO_FINALIZE). + * Provides the type of upgrade phase (e.g., FINALIZE). */ UpgradeActionType getType(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java index 8c50e3e36884..749e5f5c0673 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UpgradeActionRecon.java @@ -44,7 +44,7 @@ * 

  * {@code
  *
- * @UpgradeActionRecon(feature = FEATURE_NAME, type = AUTO_FINALIZE)
+ * @UpgradeActionRecon(feature = FEATURE_NAME, type = FINALIZE)
  *  public class FeatureNameUpgradeAction implements ReconUpgradeAction {
  *     @Override
  *     public void execute() throws Exception {

From 4127ab572680ef263d2933b87b16332c687eeb91 Mon Sep 17 00:00:00 2001
From: arafat 
Date: Wed, 23 Oct 2024 20:22:43 +0530
Subject: [PATCH 21/24] Updated recon context for failed upgrades and made
 other review changes

---
 .../hadoop/ozone/recon/ReconContext.java      |  5 ++-
 .../hadoop/ozone/recon/ReconServer.java       |  9 +++--
 .../upgrade/InitialVersionUpgradeAction.java  | 37 -------------------
 .../upgrade/ReconLayoutVersionManager.java    | 17 ++++++---
 .../TestReconLayoutVersionManager.java        |  5 ++-
 5 files changed, 23 insertions(+), 50 deletions(-)
 delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java

diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconContext.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconContext.java
index c9875cb826be..a98603a7e9ca 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconContext.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconContext.java
@@ -128,7 +128,10 @@ public enum ErrorCode {
         Arrays.asList("Overview (OM Data)", "OM DB Insights")),
     GET_SCM_DB_SNAPSHOT_FAILED(
         "SCM DB Snapshot sync failed !!!",
-        Arrays.asList("Containers", "Pipelines"));
+        Arrays.asList("Containers", "Pipelines")),
+    UPGRADE_FAILURE(
+        "Schema upgrade failed. Recon encountered an issue while finalizing the layout upgrade.",
+        Arrays.asList("Recon startup", "Metadata Layout Version"));
 
     private final String message;
     private final List impacts;
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index 9dad9daf8445..8c692db32485 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -106,6 +106,10 @@ public Void call() throws Exception {
             ReconServer.class, originalArgs, LOG, configuration);
     ConfigurationProvider.setConfiguration(configuration);
 
+    ReconStorageContainerManagerFacade reconStorageContainerManagerFacade =
+        (ReconStorageContainerManagerFacade) this.getReconStorageContainerManager();
+    ReconContext reconContext = reconStorageContainerManagerFacade.getReconContext();
+
     injector = Guice.createInjector(new ReconControllerModule(),
         new ReconRestServletModule(configuration),
         new ReconSchemaGenerationModule());
@@ -149,7 +153,7 @@ public Void call() throws Exception {
           injector.getInstance(ReconSchemaVersionTableManager.class);
 
       ReconLayoutVersionManager layoutVersionManager =
-          new ReconLayoutVersionManager(versionTableManager);
+          new ReconLayoutVersionManager(versionTableManager, reconContext);
       // Run the upgrade framework to finalize layout features if needed
       layoutVersionManager.finalizeLayoutFeatures();
 
@@ -173,9 +177,6 @@ public Void call() throws Exception {
       isStarted = true;
       LOG.info("Recon server initialized successfully!");
     } catch (Exception e) {
-      ReconStorageContainerManagerFacade reconStorageContainerManagerFacade =
-          (ReconStorageContainerManagerFacade) this.getReconStorageContainerManager();
-      ReconContext reconContext = reconStorageContainerManagerFacade.getReconContext();
       reconContext.updateHealthStatus(new AtomicBoolean(false));
       reconContext.getErrors().add(ReconContext.ErrorCode.INTERNAL_ERROR);
       LOG.error("Error during initializing Recon server.", e);
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java
deleted file mode 100644
index b76b7ac28c08..000000000000
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialVersionUpgradeAction.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.ozone.recon.upgrade;
-
-import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.INITIAL_VERSION;
-import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.FINALIZE;
-
-@UpgradeActionRecon(feature = INITIAL_VERSION, type = FINALIZE)
-public class InitialVersionUpgradeAction implements ReconUpgradeAction {
-
-  @Override
-  public void execute() throws Exception {
-    // This action does nothing for now.
-  }
-
-  @Override
-  public UpgradeActionType getType() {
-    return FINALIZE;
-  }
-}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java
index 34651dde5f60..5e5e0e776ee5 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java
@@ -19,6 +19,7 @@
 
 package org.apache.hadoop.ozone.recon.upgrade;
 
+import org.apache.hadoop.ozone.recon.ReconContext;
 import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,6 +27,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 /**
@@ -38,16 +40,16 @@ public class ReconLayoutVersionManager {
   private static final Logger LOG = LoggerFactory.getLogger(ReconLayoutVersionManager.class);
 
   private final ReconSchemaVersionTableManager schemaVersionTableManager;
+  private final ReconContext reconContext;
 
   // Metadata Layout Version (MLV) of the Recon Metadata on disk
   private int currentMLV;
-  // Software Layout Version (SLV) of the Recon service
-  private final int currentSLV;
 
-  public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager) {
+  public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager,
+                                   ReconContext reconContext) {
     this.schemaVersionTableManager = schemaVersionTableManager;
     this.currentMLV = determineMLV();
-    this.currentSLV = determineSLV();
+    this.reconContext = reconContext;
     ReconLayoutFeature.registerUpgradeActions();  // Register actions via annotation
   }
 
@@ -96,7 +98,10 @@ public void finalizeLayoutFeatures() {
           LOG.info("Feature versioned {} finalized successfully.", feature.getVersion());
         }
       } catch (Exception e) {
+        // Log the error to both logs and ReconContext
         LOG.error("Failed to finalize feature {}: {}", feature.getVersion(), e.getMessage());
+        reconContext.updateErrors(ReconContext.ErrorCode.UPGRADE_FAILURE);
+        reconContext.updateHealthStatus(new AtomicBoolean(false));
         break;
       }
     }
@@ -109,7 +114,7 @@ protected List getRegisteredFeatures() {
     List allFeatures =
         Arrays.asList(ReconLayoutFeature.values());
 
-    LOG.info("Current MLV: {}. SLV: {}. Checking features for registration...", currentMLV, currentSLV);
+    LOG.info("Current MLV: {}. SLV: {}. Checking features for registration...", currentMLV, determineSLV());
 
     List registeredFeatures = allFeatures.stream()
         .filter(feature -> feature.getVersion() > currentMLV)
@@ -136,7 +141,7 @@ public int getCurrentMLV() {
   }
 
   public int getCurrentSLV() {
-    return currentSLV;
+    return determineSLV();
   }
 
 }
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
index 9ab1e3f3ec58..8299914bd361 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
@@ -19,6 +19,7 @@
 
 package org.apache.hadoop.ozone.recon.upgrade;
 
+import org.apache.hadoop.ozone.recon.ReconContext;
 import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager;
 import org.mockito.InOrder;
 import org.mockito.MockedStatic;
@@ -75,7 +76,7 @@ public void setUp() {
     // Define the custom features to be returned
     mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1, feature2});
 
-    layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager);
+    layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager, mock(ReconContext.class));
   }
 
   @AfterEach
@@ -214,7 +215,7 @@ public void testUpgradeActionExecutionOrder() throws Exception {
   @Test
   public void testNoUpgradeActionsNeeded() {
     when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(2);
-    layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager);
+    layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager, mock(ReconContext.class));
     layoutVersionManager.finalizeLayoutFeatures();
 
     verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt());

From 3609b74faf6ad04c5471ecfd85237ad8ec7a6e6a Mon Sep 17 00:00:00 2001
From: arafat 
Date: Thu, 24 Oct 2024 12:40:46 +0530
Subject: [PATCH 22/24] Fixed the error handling review comments

---
 .../recon/ReconSchemaVersionTableManager.java |  7 ++---
 .../hadoop/ozone/recon/ReconServer.java       |  8 +++---
 .../upgrade/ReconLayoutVersionManager.java    | 26 +++++++------------
 .../TestReconLayoutVersionManager.java        | 10 ++++---
 4 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java
index 81fefc334acf..d7c3c65f2c15 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java
@@ -52,7 +52,7 @@ public ReconSchemaVersionTableManager(DataSource dataSource) throws SQLException
    * If the table is empty, or if it does not exist, it will return 0.
    * @return The current schema version.
    */
-  public int getCurrentSchemaVersion() {
+  public int getCurrentSchemaVersion() throws SQLException {
     try {
       return dslContext.select(DSL.field(name("version_number")))
           .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME))
@@ -62,7 +62,7 @@ public int getCurrentSchemaVersion() {
           .orElse(-1); // Return -1 if no version is found
     } catch (Exception e) {
       LOG.error("Failed to fetch the current schema version.", e);
-      return 0; // Return 0 if there is an exception
+      throw new SQLException("Unable to read schema version from the table.", e);
     }
   }
 
@@ -71,7 +71,7 @@ public int getCurrentSchemaVersion() {
    *
    * @param newVersion The new version to set.
    */
-  public void updateSchemaVersion(int newVersion) {
+  public void updateSchemaVersion(int newVersion) throws SQLException {
     try {
       boolean recordExists = dslContext.fetchExists(dslContext.selectOne()
           .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)));
@@ -94,6 +94,7 @@ public void updateSchemaVersion(int newVersion) {
       }
     } catch (Exception e) {
       LOG.error("Failed to update schema version to '{}'.", newVersion, e);
+      throw new SQLException("Unable to update schema version in the table.", e);
     }
   }
 
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index 8c692db32485..46d8de016bba 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -106,9 +106,6 @@ public Void call() throws Exception {
             ReconServer.class, originalArgs, LOG, configuration);
     ConfigurationProvider.setConfiguration(configuration);
 
-    ReconStorageContainerManagerFacade reconStorageContainerManagerFacade =
-        (ReconStorageContainerManagerFacade) this.getReconStorageContainerManager();
-    ReconContext reconContext = reconStorageContainerManagerFacade.getReconContext();
 
     injector = Guice.createInjector(new ReconControllerModule(),
         new ReconRestServletModule(configuration),
@@ -141,6 +138,8 @@ public Void call() throws Exception {
       this.reconNamespaceSummaryManager =
           injector.getInstance(ReconNamespaceSummaryManager.class);
 
+      ReconContext reconContext = injector.getInstance(ReconContext.class);
+
       ReconSchemaManager reconSchemaManager =
           injector.getInstance(ReconSchemaManager.class);
 
@@ -177,6 +176,9 @@ public Void call() throws Exception {
       isStarted = true;
       LOG.info("Recon server initialized successfully!");
     } catch (Exception e) {
+      ReconStorageContainerManagerFacade reconStorageContainerManagerFacade =
+          (ReconStorageContainerManagerFacade) this.getReconStorageContainerManager();
+      ReconContext reconContext = reconStorageContainerManagerFacade.getReconContext();
       reconContext.updateHealthStatus(new AtomicBoolean(false));
       reconContext.getErrors().add(ReconContext.ErrorCode.INTERNAL_ERROR);
       LOG.error("Error during initializing Recon server.", e);
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java
index 5e5e0e776ee5..b646f6d9a860 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java
@@ -24,6 +24,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -46,7 +47,8 @@ public class ReconLayoutVersionManager {
   private int currentMLV;
 
   public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager,
-                                   ReconContext reconContext) {
+                                   ReconContext reconContext)
+      throws SQLException {
     this.schemaVersionTableManager = schemaVersionTableManager;
     this.currentMLV = determineMLV();
     this.reconContext = reconContext;
@@ -57,7 +59,7 @@ public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTab
    * Determines the current Metadata Layout Version (MLV) from the version table.
    * @return The current Metadata Layout Version (MLV).
    */
-  private int determineMLV() {
+  private int determineMLV() throws SQLException {
     return schemaVersionTableManager.getCurrentSchemaVersion();
   }
 
@@ -82,13 +84,6 @@ public void finalizeLayoutFeatures() {
 
     for (ReconLayoutFeature feature : featuresToFinalize) {
       try {
-        // If the feature is INITIAL_VERSION, skip executing any action and just update the schema version
-        if (feature == ReconLayoutFeature.INITIAL_VERSION) {
-          updateSchemaVersion(0);
-          LOG.info("INITIAL_VERSION feature processed by setting schema version to 0.");
-          continue;
-        }
-
         // Fetch only the FINALIZE action for the feature
         Optional action = feature.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE);
         if (action.isPresent()) {
@@ -102,7 +97,8 @@ public void finalizeLayoutFeatures() {
         LOG.error("Failed to finalize feature {}: {}", feature.getVersion(), e.getMessage());
         reconContext.updateErrors(ReconContext.ErrorCode.UPGRADE_FAILURE);
         reconContext.updateHealthStatus(new AtomicBoolean(false));
-        break;
+        // Stop further upgrades as an error occurred
+        throw new RuntimeException("Recon failed to finalize layout feature. Startup halted.");
       }
     }
   }
@@ -125,14 +121,12 @@ protected List getRegisteredFeatures() {
   }
 
   /**
-   * Updates the Software Layout Version (SLV) in the database after finalizing a feature.
-   * @param newVersion The new Software Layout Version (SLV) to set.
+   * Updates the Metadata Layout Version (MLV) in the database after finalizing a feature.
+   * @param newVersion The new Metadata Layout Version (MLV) to set.
    */
-  private void updateSchemaVersion(int newVersion) {
-    // Logic to update the MLV in the database
-    this.currentMLV = newVersion;
-    // Code to update the schema version in the database goes here.
+  private void updateSchemaVersion(int newVersion) throws SQLException {
     schemaVersionTableManager.updateSchemaVersion(newVersion);
+    this.currentMLV = newVersion;
     LOG.info("MLV updated to: " + newVersion);
   }
 
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
index 8299914bd361..d7b63c28d97b 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
@@ -26,6 +26,8 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.AfterEach;
+
+import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -53,7 +55,7 @@ public class TestReconLayoutVersionManager {
   private MockedStatic mockedEnumUpgradeActionType;
 
   @BeforeEach
-  public void setUp() {
+  public void setUp() throws SQLException {
     schemaVersionTableManager = mock(ReconSchemaVersionTableManager.class);
     when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(0);
 
@@ -104,7 +106,7 @@ public void testInitializationWithMockedValues() {
    * the schemaVersionTable is triggered for each feature version.
    */
   @Test
-  public void testFinalizeLayoutFeaturesWithMockedValues() {
+  public void testFinalizeLayoutFeaturesWithMockedValues() throws SQLException {
     layoutVersionManager.finalizeLayoutFeatures();
 
     // Verify that schema versions are updated for our custom features
@@ -133,7 +135,7 @@ public void testGetRegisteredFeaturesWithMockedValues() {
    * version updates are attempted when there are no features to finalize.
    */
   @Test
-  public void testNoLayoutFeatures() {
+  public void testNoLayoutFeatures() throws SQLException {
     mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{});
     layoutVersionManager.finalizeLayoutFeatures();
     verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt());
@@ -213,7 +215,7 @@ public void testUpgradeActionExecutionOrder() throws Exception {
    * schema version matches the maximum layout version, no upgrade actions are executed.
    */
   @Test
-  public void testNoUpgradeActionsNeeded() {
+  public void testNoUpgradeActionsNeeded() throws SQLException {
     when(schemaVersionTableManager.getCurrentSchemaVersion()).thenReturn(2);
     layoutVersionManager = new ReconLayoutVersionManager(schemaVersionTableManager, mock(ReconContext.class));
     layoutVersionManager.finalizeLayoutFeatures();

From defd31331446c7ceb1a0034da2be3c8d2f954188 Mon Sep 17 00:00:00 2001
From: arafat 
Date: Thu, 24 Oct 2024 13:42:06 +0530
Subject: [PATCH 23/24] Fixed a failing uT

---
 .../ozone/recon/upgrade/TestReconLayoutVersionManager.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
index d7b63c28d97b..1da4c48a94c7 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/upgrade/TestReconLayoutVersionManager.java
@@ -164,7 +164,10 @@ public void testUpgradeActionFailure() throws Exception {
     mockedEnum.when(ReconLayoutFeature::values).thenReturn(new ReconLayoutFeature[]{feature1});
 
     // Execute the layout feature finalization
-    layoutVersionManager.finalizeLayoutFeatures();
+    try {
+      layoutVersionManager.finalizeLayoutFeatures();
+    } catch (Exception e) {
+    }
     // Verify that schema version update was never called due to the exception
     verify(schemaVersionTableManager, never()).updateSchemaVersion(anyInt());
   }

From f75c952f27ba6f9a95c3e3cb127d02588b4f13b4 Mon Sep 17 00:00:00 2001
From: arafat 
Date: Mon, 28 Oct 2024 11:29:11 +0530
Subject: [PATCH 24/24] Fixed stack strace problem

---
 .../apache/hadoop/ozone/recon/ReconServer.java | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index 46d8de016bba..7c9564c23b16 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -147,15 +147,6 @@ public Void call() throws Exception {
       reconSchemaManager.createReconSchema();
       LOG.debug("Recon schema creation done.");
 
-      // Handle Recon Schema Versioning
-      ReconSchemaVersionTableManager versionTableManager =
-          injector.getInstance(ReconSchemaVersionTableManager.class);
-
-      ReconLayoutVersionManager layoutVersionManager =
-          new ReconLayoutVersionManager(versionTableManager, reconContext);
-      // Run the upgrade framework to finalize layout features if needed
-      layoutVersionManager.finalizeLayoutFeatures();
-
       this.reconSafeModeMgr = injector.getInstance(ReconSafeModeManager.class);
       this.reconSafeModeMgr.setInSafeMode(true);
       httpServer = injector.getInstance(ReconHttpServer.class);
@@ -167,6 +158,15 @@ public Void call() throws Exception {
       this.reconTaskStatusMetrics =
           injector.getInstance(ReconTaskStatusMetrics.class);
 
+      // Handle Recon Schema Versioning
+      ReconSchemaVersionTableManager versionTableManager =
+          injector.getInstance(ReconSchemaVersionTableManager.class);
+
+      ReconLayoutVersionManager layoutVersionManager =
+          new ReconLayoutVersionManager(versionTableManager, reconContext);
+      // Run the upgrade framework to finalize layout features if needed
+      layoutVersionManager.finalizeLayoutFeatures();
+
       LOG.info("Initializing support of Recon Features...");
       FeatureProvider.initFeatureSupport(configuration);