From 7a32f2909532d5f004c1330f6d768adca55e8d10 Mon Sep 17 00:00:00 2001 From: rmoff Date: Thu, 22 Jan 2026 11:26:24 +0000 Subject: [PATCH 1/4] Add script to package up Kafka Connect connector for submission to Confluent Marketplace (hub.confluent.io) --- .../confluent-hub-package.sh | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 kafka-connect/kafka-connect-runtime/confluent-hub-package.sh diff --git a/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh b/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh new file mode 100755 index 000000000000..96d26133dff9 --- /dev/null +++ b/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Builds and repackages the connector for Confluent Hub submission. +# Usage: ./confluent-hub-package.sh [TAG] +# TAG: git tag to build (default: latest tag) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_DIR}/../.." + +# Determine tag +git fetch --tags 2>/dev/null || true +if [[ -n "${1:-}" ]]; then + TAG="$1" +else + TAG=$(git describe --tags "$(git rev-list --tags --max-count=1)") + echo "Using latest tag: $TAG" +fi + +git checkout "$TAG" + +# Extract version from tag (apache-iceberg-X.Y.Z -> X.Y.Z) +VERSION="${TAG#apache-iceberg-}" + +# Create version.txt to override gradle's SNAPSHOT versioning +echo "$VERSION" > "${PROJECT_ROOT}/version.txt" +trap "rm -f '${PROJECT_ROOT}/version.txt'" EXIT + +# Build (clean first to remove old artifacts) +"${PROJECT_ROOT}/gradlew" clean :iceberg-kafka-connect:iceberg-kafka-connect-runtime:distZip \ + -x test -x integrationTest --quiet + +# Extract +DIST_DIR="${SCRIPT_DIR}/build/distributions" +MAIN_ZIP="${DIST_DIR}/iceberg-kafka-connect-runtime-${VERSION}.zip" +[[ -f "${MAIN_ZIP}" ]] || { echo "Distribution not found: ${MAIN_ZIP}"; exit 1; } +PACKAGE_NAME="apache-iceberg-kafka-connect-${VERSION}" +WORK_DIR="${SCRIPT_DIR}/build/confluent-hub-work" + +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}" +unzip -q "${MAIN_ZIP}" -d "${WORK_DIR}" +mv "${WORK_DIR}/iceberg-kafka-connect-runtime-${VERSION}" "${WORK_DIR}/${PACKAGE_NAME}" +PKG="${WORK_DIR}/${PACKAGE_NAME}" + +# Update manifest owner +sed -i.bak 's/"username": "iceberg"/"username": "apache"/' "${PKG}/manifest.json" +rm -f "${PKG}/manifest.json.bak" + +# Add README +cat > "${PKG}/doc/README.md" << EOF +# Apache Iceberg Sink Connector + +A Kafka Connect sink connector for writing data from Apache Kafka into Apache Iceberg tables. + +Documentation: https://iceberg.apache.org/docs/${VERSION}/kafka-connect/ + +License: Apache License 2.0 +EOF + +# Add example config +mkdir -p "${PKG}/etc" +cat > "${PKG}/etc/iceberg-sink.properties.example" << 'EOF' +# Iceberg Sink Connector - Example Configuration +# https://iceberg.apache.org/docs/latest/kafka-connect/ +name=iceberg-sink +connector.class=org.apache.iceberg.connect.IcebergSinkConnector +tasks.max=1 +topics=events +iceberg.tables=db.table_name +iceberg.catalog.type=rest +iceberg.catalog.uri=https://your-catalog-uri +iceberg.catalog.warehouse=your-warehouse +EOF + +# Add logo +mkdir -p "${PKG}/assets" +cp "${SCRIPT_DIR}/src/main/resources/iceberg.png" "${PKG}/assets/" + +# Create archive +OUTPUT="${DIST_DIR}/${PACKAGE_NAME}.zip" +(cd "${WORK_DIR}" && zip -rq "${OUTPUT}" "${PACKAGE_NAME}") +rm -rf "${WORK_DIR}" + +echo "Created: ${OUTPUT}" From d9d549ed7056c35356f29590492f106a089495fc Mon Sep 17 00:00:00 2001 From: rmoff Date: Mon, 26 Jan 2026 11:18:55 +0000 Subject: [PATCH 2/4] Add trivy vuln scanning --- kafka-connect/kafka-connect-runtime/confluent-hub-package.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh b/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh index 96d26133dff9..a6b7b8acbd66 100755 --- a/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh +++ b/kafka-connect/kafka-connect-runtime/confluent-hub-package.sh @@ -95,6 +95,9 @@ EOF mkdir -p "${PKG}/assets" cp "${SCRIPT_DIR}/src/main/resources/iceberg.png" "${PKG}/assets/" +# Security scan +docker run --rm -v "${PKG}:/scan:ro" aquasec/trivy rootfs --severity HIGH,CRITICAL /scan + # Create archive OUTPUT="${DIST_DIR}/${PACKAGE_NAME}.zip" (cd "${WORK_DIR}" && zip -rq "${OUTPUT}" "${PACKAGE_NAME}") From 35483dfe80068344a8e09963f48f95d0b7ccc45f Mon Sep 17 00:00:00 2001 From: rmoff Date: Thu, 29 Jan 2026 16:52:13 +0000 Subject: [PATCH 3/4] Tidyup and move script --- dev/confluent-hub-package.sh | 111 +++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 dev/confluent-hub-package.sh diff --git a/dev/confluent-hub-package.sh b/dev/confluent-hub-package.sh new file mode 100755 index 000000000000..11c72be96a98 --- /dev/null +++ b/dev/confluent-hub-package.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Builds and repackages the connector for Confluent Hub submission. +# Usage: dev/confluent-hub-package.sh [TAG] +# TAG: git tag to build (default: latest tag) + +set -euo pipefail + +ORIGINAL_DIR="$(pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +CONNECT_RUNTIME_DIR="${PROJECT_ROOT}/kafka-connect/kafka-connect-runtime" + +# Return to original directory on exit +trap "cd '${ORIGINAL_DIR}'" EXIT + +# Determine tag +if [[ -n "${1:-}" ]]; then + TAG="$1" +else + git fetch --tags 2>/dev/null || true + TAG=$(git describe --tags "$(git rev-list --tags --max-count=1)") + echo "Using latest tag: $TAG" +fi + +git checkout "$TAG" + +# Extract version from tag (apache-iceberg-X.Y.Z -> X.Y.Z) +VERSION="${TAG#apache-iceberg-}" + +# Create version.txt to override gradle's SNAPSHOT versioning +echo "$VERSION" > "${PROJECT_ROOT}/version.txt" +trap "rm -f '${PROJECT_ROOT}/version.txt'; cd '${ORIGINAL_DIR}'" EXIT + +# Build (clean first to remove old artifacts) +"${PROJECT_ROOT}/gradlew" clean :iceberg-kafka-connect:iceberg-kafka-connect-runtime:distZip \ + -x test -x integrationTest --quiet + +# Extract +DIST_DIR="${CONNECT_RUNTIME_DIR}/build/distributions" +MAIN_ZIP="${DIST_DIR}/iceberg-kafka-connect-runtime-${VERSION}.zip" +[[ -f "${MAIN_ZIP}" ]] || { echo "Distribution not found: ${MAIN_ZIP}"; exit 1; } +PACKAGE_NAME="apache-iceberg-kafka-connect-${VERSION}" +WORK_DIR="${CONNECT_RUNTIME_DIR}/build/confluent-hub-work" + +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}" +unzip -q "${MAIN_ZIP}" -d "${WORK_DIR}" +mv "${WORK_DIR}/iceberg-kafka-connect-runtime-${VERSION}" "${WORK_DIR}/${PACKAGE_NAME}" +PKG="${WORK_DIR}/${PACKAGE_NAME}" + +# Update manifest owner +sed -i.bak 's/"username": "iceberg"/"username": "apache"/' "${PKG}/manifest.json" +rm -f "${PKG}/manifest.json.bak" + +# Add README +cat > "${PKG}/doc/README.md" << EOF +# Apache Iceberg Sink Connector + +A Kafka Connect sink connector for writing data from Apache Kafka into Apache Iceberg tables. + +Documentation: https://iceberg.apache.org/docs/${VERSION}/kafka-connect/ + +License: Apache License 2.0 +EOF + +# Add example config +mkdir -p "${PKG}/etc" +cat > "${PKG}/etc/iceberg-sink.properties.example" << 'EOF' +# Iceberg Sink Connector - Example Configuration +# https://iceberg.apache.org/docs/latest/kafka-connect/ +name=iceberg-sink +connector.class=org.apache.iceberg.connect.IcebergSinkConnector +tasks.max=1 +topics=events +iceberg.tables=db.table_name +iceberg.catalog.type=rest +iceberg.catalog.uri=https://your-catalog-uri +iceberg.catalog.warehouse=your-warehouse +EOF + +# Add logo +mkdir -p "${PKG}/assets" +cp "${CONNECT_RUNTIME_DIR}/src/main/resources/iceberg.png" "${PKG}/assets/" + +# Security scan +docker run --rm -v "${PKG}:/scan:ro" aquasec/trivy rootfs --severity HIGH,CRITICAL /scan + +# Create archive +OUTPUT="${DIST_DIR}/${PACKAGE_NAME}.zip" +(cd "${WORK_DIR}" && zip -rq "${OUTPUT}" "${PACKAGE_NAME}") +rm -rf "${WORK_DIR}" + +echo "Created: ${OUTPUT}" From db1c6a6ed5764bb47aeb98744c8fe107aba3684c Mon Sep 17 00:00:00 2001 From: rmoff Date: Fri, 30 Jan 2026 12:10:08 +0000 Subject: [PATCH 4/4] Add licenses, fix docs URL --- dev/confluent-hub-package.sh | 38 +++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/dev/confluent-hub-package.sh b/dev/confluent-hub-package.sh index 11c72be96a98..6252b10413a7 100755 --- a/dev/confluent-hub-package.sh +++ b/dev/confluent-hub-package.sh @@ -72,11 +72,30 @@ rm -f "${PKG}/manifest.json.bak" # Add README cat > "${PKG}/doc/README.md" << EOF + + # Apache Iceberg Sink Connector A Kafka Connect sink connector for writing data from Apache Kafka into Apache Iceberg tables. -Documentation: https://iceberg.apache.org/docs/${VERSION}/kafka-connect/ +Documentation: https://iceberg.apache.org/docs/latest/kafka-connect/ License: Apache License 2.0 EOF @@ -84,6 +103,23 @@ EOF # Add example config mkdir -p "${PKG}/etc" cat > "${PKG}/etc/iceberg-sink.properties.example" << 'EOF' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Iceberg Sink Connector - Example Configuration # https://iceberg.apache.org/docs/latest/kafka-connect/ name=iceberg-sink