From 2f655be25bf7c4d044a353ebbffdfd336dedccc6 Mon Sep 17 00:00:00 2001 From: Satish Duggana Date: Tue, 6 Apr 2021 23:00:11 +0530 Subject: [PATCH] MINOR Moved tiered storage API classes from clients module to a new storage-api wmodule. Created storage and storage-api modules --- build.gradle | 128 +++++++++++++++++- settings.gradle | 2 + .../log/remote/storage/LogSegmentData.java | 0 .../storage/RemoteLogMetadataManager.java | 0 .../remote/storage/RemoteLogSegmentId.java | 0 .../storage/RemoteLogSegmentMetadata.java | 0 .../RemoteLogSegmentMetadataUpdate.java | 0 .../remote/storage/RemoteLogSegmentState.java | 0 .../RemotePartitionDeleteMetadata.java | 0 .../storage/RemotePartitionDeleteState.java | 0 .../RemoteResourceNotFoundException.java | 0 .../storage/RemoteStorageException.java | 0 .../remote/storage/RemoteStorageManager.java | 0 13 files changed, 129 insertions(+), 1 deletion(-) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java (100%) rename {clients => storage/api}/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java (100%) diff --git a/build.gradle b/build.gradle index 98ab70d8ca825..ac8f9e3d47154 100644 --- a/build.gradle +++ b/build.gradle @@ -775,6 +775,7 @@ project(':core') { implementation project(':metadata') implementation project(':raft') + implementation project(':storage') implementation libs.argparse4j implementation libs.jacksonDatabind @@ -992,6 +993,10 @@ project(':core') { from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror-client').jar) { into("libs/") } from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } + from(project(':storage').jar) { into("libs/") } + from(project(':storage').configurations.runtimeClasspath) { into("libs/") } + from(project(':storage:api').jar) { into("libs/") } + from(project(':storage:api').configurations.runtimeClasspath) { into("libs/") } from(project(':streams').jar) { into("libs/") } from(project(':streams').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:streams-scala').jar) { into("libs/") } @@ -1245,7 +1250,6 @@ project(':clients') { include "**/org/apache/kafka/server/authorizer/*" include "**/org/apache/kafka/server/policy/*" include "**/org/apache/kafka/server/quota/*" - include "**/org/apache/kafka/server/log/remote/storage/*" } } @@ -1331,6 +1335,128 @@ project(':raft') { } } +project(':storage:api') { + archivesBaseName = "kafka-storage-api" + + dependencies { + implementation project(':clients') + implementation libs.slf4jApi + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + + testRuntime libs.slf4jlog4j + } + + task createVersionFile(dependsOn: determineCommitId) { + ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName") + outputs.file receiptFile + outputs.upToDateWhen { false } + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + jar { + dependsOn createVersionFile + from("$buildDir") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "$buildDir/kafka/" + } + + javadoc { + include "**/org/apache/kafka/server/log/remote/storage/*" + } +} + +project(':storage') { + archivesBaseName = "kafka-storage" + + dependencies { + implementation project(':storage:api') + implementation project(':clients') + implementation libs.slf4jApi + implementation libs.jacksonDatabind + + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + + testRuntime libs.slf4jlog4j + } + + task createVersionFile(dependsOn: determineCommitId) { + ext.receiptFile = file("$buildDir/kafka/$buildVersionFileName") + outputs.file receiptFile + outputs.upToDateWhen { false } + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + sourceSets { + main { + java { + srcDirs = ["src/main/java"] + } + } + test { + java { + srcDirs = ["src/test/java"] + } + } + } + + jar { + dependsOn createVersionFile + from("$buildDir") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "$buildDir/kafka/" + } + + javadoc { + enabled = false + } +} + project(':tools') { archivesBaseName = "kafka-tools" diff --git a/settings.gradle b/settings.gradle index fedfa9a650cc4..91c71f52eb8d3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,6 +30,8 @@ include 'clients', 'metadata', 'raft', 'shell', + 'storage', + 'storage:api', 'streams', 'streams:examples', 'streams:streams-scala', diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java b/storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java similarity index 100% rename from clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java rename to storage/api/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java