From 7e3afaea1705c508a8353463d4dd0c731a088bae Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Mon, 11 Sep 2023 21:25:06 +0800 Subject: [PATCH] test(plugins): add test for plugins --- plugins/build.gradle | 3 + .../java/org/tron/plugins/DbConvertTest.java | 92 ++------- .../java/org/tron/plugins/DbCopyTest.java | 53 ++++++ .../org/tron/plugins/DbLiteLevelDbTest.java | 13 ++ .../org/tron/plugins/DbLiteLevelDbV2Test.java | 12 ++ .../org/tron/plugins/DbLiteRocksDbTest.java | 12 ++ .../java/org/tron/plugins/DbLiteTest.java | 176 ++++++++++++++++++ .../test/java/org/tron/plugins/DbTest.java | 89 +++++++++ 8 files changed, 372 insertions(+), 78 deletions(-) create mode 100644 plugins/src/test/java/org/tron/plugins/DbCopyTest.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbLiteTest.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbTest.java diff --git a/plugins/build.gradle b/plugins/build.gradle index 9f4cb3ef59b..d8578f576b0 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -29,6 +29,8 @@ dependencies { testCompile group: 'junit', name: 'junit', version: '4.13.2' testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0' testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1' + testCompile project(":framework") + testCompile project(":framework").sourceSets.test.output compile group: 'info.picocli', name: 'picocli', version: '4.6.3' compile group: 'com.typesafe', name: 'config', version: '1.3.2' compile group: 'me.tongfei', name: 'progressbar', version: '0.9.3' @@ -75,6 +77,7 @@ test { testLogging { exceptionFormat = 'full' } + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 jacoco { destinationFile = file("$buildDir/jacoco/jacocoTest.exec") classDumpDir = file("$buildDir/jacoco/classpathdumps") diff --git a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java b/plugins/src/test/java/org/tron/plugins/DbConvertTest.java index 91996815c01..150e47c9f65 100644 --- a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbConvertTest.java @@ -1,85 +1,24 @@ package org.tron.plugins; -import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.UUID; -import org.iq80.leveldb.DB; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; -import org.tron.plugins.utils.ByteArray; -import org.tron.plugins.utils.DBUtils; -import org.tron.plugins.utils.FileUtils; -import org.tron.plugins.utils.MarketUtils; import picocli.CommandLine; -public class DbConvertTest { +public class DbConvertTest extends DbTest { - - private static final String INPUT_DIRECTORY = "output-directory/convert-database/"; - private static final String OUTPUT_DIRECTORY = "output-directory/convert-database-dest/"; - private static final String ACCOUNT = "account"; - private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; - CommandLine cli = new CommandLine(new Toolkit()); - - - @BeforeClass - public static void init() throws IOException { - if (new File(INPUT_DIRECTORY).mkdirs()) { - initDB(new File(INPUT_DIRECTORY,ACCOUNT)); - initDB(new File(INPUT_DIRECTORY,MARKET)); - } - } - - private static void initDB(File file) throws IOException { - try (DB db = DBUtils.newLevelDb(file.toPath())) { - if (MARKET.equalsIgnoreCase(file.getName())) { - byte[] sellTokenID1 = ByteArray.fromString("100"); - byte[] buyTokenID1 = ByteArray.fromString("200"); - byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( - sellTokenID1, - buyTokenID1, - 1000L, - 2001L - ); - byte[] pairPriceKey2 = MarketUtils.createPairPriceKey( - sellTokenID1, - buyTokenID1, - 1000L, - 2002L - ); - byte[] pairPriceKey3 = MarketUtils.createPairPriceKey( - sellTokenID1, - buyTokenID1, - 1000L, - 2003L - ); - - - //Use out-of-order insertion，key in store should be 1,2,3 - db.put(pairPriceKey1, "1".getBytes(StandardCharsets.UTF_8)); - db.put(pairPriceKey2, "2".getBytes(StandardCharsets.UTF_8)); - db.put(pairPriceKey3, "3".getBytes(StandardCharsets.UTF_8)); - } else { - for (int i = 0; i < 100; i++) { - byte[] bytes = UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8); - db.put(bytes, bytes); - } - } - } - } - - @AfterClass - public static void destroy() { - FileUtils.deleteDir(new File(INPUT_DIRECTORY)); - FileUtils.deleteDir(new File(OUTPUT_DIRECTORY)); + @Test + public void testRun() throws IOException { + String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, + temporaryFolder.newFolder().toString() }; + Assert.assertEquals(0, cli.execute(args)); } @Test - public void testRun() { - String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, OUTPUT_DIRECTORY }; + public void testRunWithSafe() throws IOException { + String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, + temporaryFolder.newFolder().toString(),"--safe" }; Assert.assertEquals(0, cli.execute(args)); } @@ -92,18 +31,15 @@ public void testHelp() { @Test public void testNotExist() { - String[] args = new String[] {"db", "convert", - OUTPUT_DIRECTORY + File.separator + UUID.randomUUID(), - OUTPUT_DIRECTORY}; + String[] args = new String[] {"db", "convert", UUID.randomUUID().toString(), + UUID.randomUUID().toString()}; Assert.assertEquals(404, cli.execute(args)); } @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {"db", "convert", file.toString(), OUTPUT_DIRECTORY}; + public void testEmpty() throws IOException { + String[] args = new String[] {"db", "convert", temporaryFolder.newFolder().toString(), + temporaryFolder.newFolder().toString()}; Assert.assertEquals(0, cli.execute(args)); } diff --git a/plugins/src/test/java/org/tron/plugins/DbCopyTest.java b/plugins/src/test/java/org/tron/plugins/DbCopyTest.java new file mode 100644 index 00000000000..c5cc8f2bb31 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbCopyTest.java @@ -0,0 +1,53 @@ +package org.tron.plugins; + +import java.io.IOException; +import java.util.UUID; +import org.junit.Assert; +import org.junit.Test; +import picocli.CommandLine; + +public class DbCopyTest extends DbTest { + + @Test + public void testRun() { + String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, + tmpDir + UUID.randomUUID()}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testHelp() { + String[] args = new String[] {"db", "cp", "-h"}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testNotExist() { + String[] args = new String[] {"db", "cp", UUID.randomUUID().toString(), + UUID.randomUUID().toString()}; + Assert.assertEquals(404, cli.execute(args)); + } + + @Test + public void testEmpty() throws IOException { + String[] args = new String[] {"db", "cp", temporaryFolder.newFolder().toString(), + tmpDir + UUID.randomUUID()}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testDestIsExist() throws IOException { + String[] args = new String[] {"db", "cp", temporaryFolder.newFile().toString(), + temporaryFolder.newFolder().toString()}; + Assert.assertEquals(402, cli.execute(args)); + } + + @Test + public void testSrcIsFile() throws IOException { + String[] args = new String[] {"db", "cp", temporaryFolder.newFile().toString(), + tmpDir + UUID.randomUUID()}; + Assert.assertEquals(403, cli.execute(args)); + } + +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java new file mode 100644 index 00000000000..792f93ad197 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java @@ -0,0 +1,13 @@ +package org.tron.plugins; + +import java.io.IOException; +import org.junit.Test; + + +public class DbLiteLevelDbTest extends DbLiteTest { + + @Test + public void testToolsWithLevelDB() throws InterruptedException, IOException { + testTools("LEVELDB", 1); + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java b/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java new file mode 100644 index 00000000000..ae48e1d66e9 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java @@ -0,0 +1,12 @@ +package org.tron.plugins; + +import java.io.IOException; +import org.junit.Test; + +public class DbLiteLevelDbV2Test extends DbLiteTest { + + @Test + public void testToolsWithLevelDBV2() throws InterruptedException, IOException { + testTools("LEVELDB", 2); + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java new file mode 100644 index 00000000000..e6910b1103a --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java @@ -0,0 +1,12 @@ +package org.tron.plugins; + +import java.io.IOException; +import org.junit.Test; + +public class DbLiteRocksDbTest extends DbLiteTest { + + @Test + public void testToolsWithRocksDB() throws InterruptedException, IOException { + testTools("ROCKSDB", 1); + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java new file mode 100644 index 00000000000..8baf48b0b48 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -0,0 +1,176 @@ +package org.tron.plugins; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.tron.api.WalletGrpc; +import org.tron.common.application.Application; +import org.tron.common.application.ApplicationFactory; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.config.DbBackupConfig; +import org.tron.common.crypto.ECKey; +import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PublicMethod; +import org.tron.common.utils.Utils; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.core.services.RpcApiService; +import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; +import picocli.CommandLine; + +@Slf4j +public class DbLiteTest { + + private TronApplicationContext context; + private WalletGrpc.WalletBlockingStub blockingStubFull = null; + private ManagedChannel channelFull; + private Application appTest; + private String databaseDir; + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + private String dbPath; + CommandLine cli = new CommandLine(new DbLite()); + + /** + * init logic. + */ + public void startApp() { + context = new TronApplicationContext(DefaultConfig.class); + appTest = ApplicationFactory.create(context); + appTest.addService(context.getBean(RpcApiService.class)); + appTest.addService(context.getBean(RpcApiServiceOnSolidity.class)); + appTest.startup(); + + String fullNode = String.format("%s:%d", "127.0.0.1", + Args.getInstance().getRpcPort()); + channelFull = ManagedChannelBuilder.forTarget(fullNode) + .usePlaintext() + .build(); + blockingStubFull = WalletGrpc.newBlockingStub(channelFull); + } + + /** + * shutdown the fullNode. + */ + public void shutdown() throws InterruptedException { + if (channelFull != null) { + channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } + context.close(); + } + + public void init() throws IOException { + dbPath = folder.newFolder().toString(); + Args.setParam(new String[]{"-d", dbPath, "-w", "--p2p-disable", "true"}, + "config-localtest.conf"); + // allow account root + Args.getInstance().setAllowAccountStateRoot(1); + Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); + databaseDir = Args.getInstance().getStorage().getDbDirectory(); + // init dbBackupConfig to avoid NPE + Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); + } + + @After + public void clear() { + Args.clearParam(); + } + + void testTools(String dbType, int checkpointVersion) + throws InterruptedException, IOException { + logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); + dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); + init(); + final String[] argsForSnapshot = + new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", + dbPath + File.separator + databaseDir, "--dataset-path", + dbPath}; + final String[] argsForHistory = + new String[]{"-o", "split", "-t", "history", "--fn-data-path", + dbPath + File.separator + databaseDir, "--dataset-path", + dbPath}; + final String[] argsForMerge = + new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, + "--dataset-path", dbPath + File.separator + "history"}; + Args.getInstance().getStorage().setDbEngine(dbType); + Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); + DbLite.setRecentBlks(3); + // start fullNode + startApp(); + // produce transactions for 18 seconds + generateSomeTransactions(18); + // stop the node + shutdown(); + // delete tran-cache + FileUtil.deleteDir(Paths.get(dbPath, databaseDir, "trans-cache").toFile()); + // generate snapshot + cli.execute(argsForSnapshot); + // start fullNode + startApp(); + // produce transactions for 6 seconds + generateSomeTransactions(6); + // stop the node + shutdown(); + // generate history + cli.execute(argsForHistory); + // backup original database to database_bak + File database = new File(Paths.get(dbPath, databaseDir).toString()); + if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { + throw new RuntimeException( + String.format("rename %s to %s failed", database.getPath(), + Paths.get(dbPath, databaseDir))); + } + // change snapshot to the new database + File snapshot = new File(Paths.get(dbPath, "snapshot").toString()); + if (!snapshot.renameTo(new File(Paths.get(dbPath, databaseDir).toString()))) { + throw new RuntimeException( + String.format("rename snapshot to %s failed", + Paths.get(dbPath, databaseDir))); + } + // start and validate the snapshot + startApp(); + generateSomeTransactions(6); + // stop the node + shutdown(); + // merge history + cli.execute(argsForMerge); + // start and validate + startApp(); + generateSomeTransactions(6); + shutdown(); + DbLite.reSetRecentBlks(); + } + + private void generateSomeTransactions(int during) { + during *= 1000; // ms + int runTime = 0; + int sleepOnce = 100; + while (true) { + ECKey ecKey2 = new ECKey(Utils.getRandom()); + byte[] address = ecKey2.getAddress(); + + String sunPri = "cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a"; + byte[] sunAddress = PublicMethod.getFinalAddress(sunPri); + PublicMethod.sendcoin(address, 1L, + sunAddress, sunPri, blockingStubFull); + try { + Thread.sleep(sleepOnce); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if ((runTime += sleepOnce) > during) { + return; + } + } + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbTest.java b/plugins/src/test/java/org/tron/plugins/DbTest.java new file mode 100644 index 00000000000..8a5f9de4a67 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbTest.java @@ -0,0 +1,89 @@ +package org.tron.plugins; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.UUID; +import org.iq80.leveldb.DB; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.tron.plugins.utils.ByteArray; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.MarketUtils; +import picocli.CommandLine; + +public class DbTest { + + String INPUT_DIRECTORY; + private static final String ACCOUNT = "account"; + private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; + CommandLine cli = new CommandLine(new Toolkit()); + String tmpDir = System.getProperty("java.io.tmpdir"); + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + + @Before + public void init() throws IOException { + INPUT_DIRECTORY = temporaryFolder.newFolder().toString(); + initDB(new File(INPUT_DIRECTORY, ACCOUNT)); + initDB(new File(INPUT_DIRECTORY, MARKET)); + initDB(new File(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2)); + } + + private static void initDB(File file) throws IOException { + if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(file.getName())) { + File dbFile = new File(file, DBUtils.CHECKPOINT_DB_V2); + if (dbFile.mkdirs()) { + for (int i = 0; i < 3; i++) { + try (DB db = DBUtils.newLevelDb(Paths.get(dbFile.getPath(), + System.currentTimeMillis() + ""))) { + for (int j = 0; j < 100; j++) { + byte[] bytes = UUID.randomUUID().toString().getBytes(); + db.put(bytes, bytes); + } + } + } + } + return; + } + try (DB db = DBUtils.newLevelDb(file.toPath())) { + if (MARKET.equalsIgnoreCase(file.getName())) { + byte[] sellTokenID1 = ByteArray.fromString("100"); + byte[] buyTokenID1 = ByteArray.fromString("200"); + byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( + sellTokenID1, + buyTokenID1, + 1000L, + 2001L + ); + byte[] pairPriceKey2 = MarketUtils.createPairPriceKey( + sellTokenID1, + buyTokenID1, + 1000L, + 2002L + ); + byte[] pairPriceKey3 = MarketUtils.createPairPriceKey( + sellTokenID1, + buyTokenID1, + 1000L, + 2003L + ); + + + //Use out-of-order insertion，key in store should be 1,2,3 + db.put(pairPriceKey1, "1".getBytes(StandardCharsets.UTF_8)); + db.put(pairPriceKey2, "2".getBytes(StandardCharsets.UTF_8)); + db.put(pairPriceKey3, "3".getBytes(StandardCharsets.UTF_8)); + } else { + for (int i = 0; i < 100; i++) { + byte[] bytes = UUID.randomUUID().toString().getBytes(); + db.put(bytes, bytes); + } + } + } + } +}