From 78c6f948f3bf2f371ab6449cfd02fb4a4efb7c99 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 19 Apr 2022 19:14:44 +0300 Subject: [PATCH 01/11] Upgrade Netty to 4.1.76.Final, Netty Tcnative, grpc and protobuf (#15212) * Upgrade Netty to 4.1.76.Final and Netty Tcnative to 2.0.51.Final Fixes #14015 - release notes https://netty.io/news/2022/04/12/4-1-76-Final.html - contains fix for https://github.com/netty/netty/issues/11695 * Upgrade grpc to 1.45.1 and protobuf to 3.19.2 - grpc < 1.45.1 is not compatible with Netty > 4.1.74.Final - https://github.com/grpc/grpc-java/pull/9004 (cherry picked from commit 332a3c74c03184bb8d1298450ff21e5dc93be375) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 80 +++++++++++-------- pom.xml | 35 ++++++-- pulsar-sql/presto-distribution/LICENSE | 55 +++++++++---- 4 files changed, 116 insertions(+), 56 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index ed2b17d21bd1c..4cf0a3034b4f0 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -59,7 +59,7 @@ io.netty netty-common - 4.1.68.Final + 4.1.76.Final test diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 936528ad8b7e2..cf7c1bb961537 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -357,24 +357,31 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.6.jar * Netty - - io.netty-netty-buffer-4.1.68.Final.jar - - io.netty-netty-codec-4.1.68.Final.jar - - io.netty-netty-codec-dns-4.1.68.Final.jar - - io.netty-netty-codec-http-4.1.68.Final.jar - - io.netty-netty-codec-http2-4.1.68.Final.jar - - io.netty-netty-codec-socks-4.1.68.Final.jar - - io.netty-netty-codec-haproxy-4.1.68.Final.jar - - io.netty-netty-common-4.1.68.Final.jar - - io.netty-netty-handler-4.1.68.Final.jar - - io.netty-netty-handler-proxy-4.1.68.Final.jar - - io.netty-netty-resolver-4.1.68.Final.jar - - io.netty-netty-resolver-dns-4.1.68.Final.jar - - io.netty-netty-transport-4.1.68.Final.jar - - io.netty-netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.68.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.68.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.68.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.42.Final.jar + - io.netty-netty-buffer-4.1.76.Final.jar + - io.netty-netty-codec-4.1.76.Final.jar + - io.netty-netty-codec-dns-4.1.76.Final.jar + - io.netty-netty-codec-http-4.1.76.Final.jar + - io.netty-netty-codec-http2-4.1.76.Final.jar + - io.netty-netty-codec-socks-4.1.76.Final.jar + - io.netty-netty-codec-haproxy-4.1.76.Final.jar + - io.netty-netty-common-4.1.76.Final.jar + - io.netty-netty-handler-4.1.76.Final.jar + - io.netty-netty-handler-proxy-4.1.76.Final.jar + - io.netty-netty-resolver-4.1.76.Final.jar + - io.netty-netty-resolver-dns-4.1.76.Final.jar + - io.netty-netty-transport-4.1.76.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.76.Final.jar + - io.netty-netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.76.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.76.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.51.Final.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar @@ -456,20 +463,24 @@ The Apache Software License, Version 2.0 * Okio - com.squareup.okio-okio-1.13.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * gRPC - - io.grpc-grpc-all-1.18.0.jar - - io.grpc-grpc-auth-1.18.0.jar - - io.grpc-grpc-context-1.18.0.jar - - io.grpc-grpc-core-1.18.0.jar - - io.grpc-grpc-netty-1.18.0.jar - - io.grpc-grpc-okhttp-1.18.0.jar - - io.grpc-grpc-protobuf-1.18.0.jar - - io.grpc-grpc-protobuf-lite-1.18.0.jar - - io.grpc-grpc-stub-1.18.0.jar - - io.grpc-grpc-protobuf-nano-1.18.0.jar - - io.grpc-grpc-testing-1.18.0.jar + - io.grpc-grpc-all-1.45.1.jar + - io.grpc-grpc-auth-1.45.1.jar + - io.grpc-grpc-context-1.45.1.jar + - io.grpc-grpc-core-1.45.1.jar + - io.grpc-grpc-netty-1.45.1.jar + - io.grpc-grpc-protobuf-1.45.1.jar + - io.grpc-grpc-protobuf-lite-1.45.1.jar + - io.grpc-grpc-stub-1.45.1.jar + - io.grpc-grpc-alts-1.45.1.jar + - io.grpc-grpc-api-1.45.1.jar + - io.grpc-grpc-grpclb-1.45.1.jar + - io.grpc-grpc-netty-shaded-1.45.1.jar + - io.grpc-grpc-services-1.45.1.jar + - io.grpc-grpc-xds-1.45.1.jar + - io.grpc-grpc-rls-1.45.1.jar * OpenCensus - - io.opencensus-opencensus-api-0.18.0.jar - - io.opencensus-opencensus-contrib-grpc-metrics-0.18.0.jar + - io.opencensus-opencensus-api-0.28.0.jar + - io.opencensus-opencensus-contrib-http-util-0.28.0.jar * Jodah - net.jodah-typetools-0.5.0.jar * Apache Avro @@ -517,7 +528,8 @@ The Apache Software License, Version 2.0 BSD 3-clause "New" or "Revised" License * Google auth library - - com.google.auth-google-auth-library-credentials-0.9.0.jar -- licenses/LICENSE-google-auth-library.txt + - com.google.auth-google-auth-library-credentials-1.4.0.jar -- licenses/LICENSE-google-auth-library.txt + - com.google.auth-google-auth-library-oauth2-http-1.4.0.jar -- licenses/LICENSE-google-auth-library.txt * LevelDB -- (included in org.rocksdb.*.jar) -- licenses/LICENSE-LevelDB.txt * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- licenses/LICENSE-JSR305.txt * JavaHamcrest -- org.hamcrest-hamcrest-core-1.3.jar -- licenses/LICENSE-Hamcrest.txt @@ -539,8 +551,8 @@ MIT License Protocol Buffers License * Protocol Buffers - - com.google.protobuf-protobuf-java-3.16.1.jar -- licenses/LICENSE-protobuf.txt - - com.google.protobuf-protobuf-java-util-3.16.1.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-3.19.2.jar -- licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-util-3.19.2.jar -- licenses/LICENSE-protobuf.txt - com.google.protobuf.nano-protobuf-javanano-3.0.0-alpha-5.jar -- licenses/LICENSE-protobuf.txt CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt diff --git a/pom.xml b/pom.xml index e793d8998b7bf..1ff708b6863b8 100644 --- a/pom.xml +++ b/pom.xml @@ -100,8 +100,8 @@ flexible messaging model and an intuitive client API. 4.12.1 3.5.9 - 4.1.68.Final - 2.0.42.Final + 4.1.76.Final + 2.0.51.Final 9.4.43.v20210629 2.31 1.10.9 @@ -122,9 +122,10 @@ flexible messaging model and an intuitive client API. 1.4.13 0.5.0 2.4.1 - 3.16.1 + 3.19.2 ${protobuf3.version} - 1.18.0 + 1.45.1 + 1.41.0 ${grpc.version} 2.8.9 0.8.3 @@ -150,7 +151,7 @@ flexible messaging model and an intuitive client API. 2.11.12 1.0.0.Final 0.11.1 - 0.18.0 + 0.28.0 2.3.0 30.1-jre 1.0 @@ -805,6 +806,24 @@ flexible messaging model and an intuitive client API. ${grpc.version} + + com.google.http-client + google-http-client + ${google-http-client.version} + + + + com.google.http-client + google-http-client-jackson2 + ${google-http-client.version} + + + + com.google.http-client + google-http-client-gson + ${google-http-client.version} + + io.grpc grpc-stub @@ -918,6 +937,12 @@ flexible messaging model and an intuitive client API. ${opencensus.version} + + io.opencensus + opencensus-contrib-http-util + ${opencensus.version} + + io.opencensus opencensus-contrib-grpc-metrics diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 906cad724359e..238c246e93228 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -233,21 +233,44 @@ The Apache Software License, Version 2.0 - commons-lang3-3.6.jar * Netty - netty-3.10.6.Final.jar - - netty-buffer-4.1.68.Final.jar - - netty-codec-4.1.68.Final.jar - - netty-codec-dns-4.1.68.Final.jar - - netty-codec-http-4.1.68.Final.jar - - netty-codec-haproxy-4.1.68.Final.jar - - netty-common-4.1.68.Final.jar - - netty-handler-4.1.68.Final.jar + - netty-buffer-4.1.76.Final.jar + - netty-codec-4.1.76.Final.jar + - netty-codec-dns-4.1.76.Final.jar + - netty-codec-http-4.1.76.Final.jar + - netty-codec-haproxy-4.1.76.Final.jar + - netty-codec-socks-4.1.76.Final.jar + - netty-handler-proxy-4.1.76.Final.jar + - netty-common-4.1.76.Final.jar + - netty-handler-4.1.76.Final.jar - netty-reactive-streams-2.0.4.jar - - netty-resolver-4.1.68.Final.jar - - netty-resolver-dns-4.1.68.Final.jar - - netty-tcnative-boringssl-static-2.0.42.Final.jar - - netty-transport-4.1.68.Final.jar - - netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.68.Final.jar - - netty-transport-native-unix-common-4.1.68.Final-linux-x86_64.jar + - netty-resolver-4.1.76.Final.jar + - netty-resolver-dns-4.1.76.Final.jar + - netty-tcnative-boringssl-static-2.0.51.Final.jar + - netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.51.Final.jar + - netty-transport-4.1.76.Final.jar + - netty-transport-classes-epoll-4.1.76.Final.jar + - netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.76.Final.jar + - netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar + - netty-codec-http2-4.1.76.Final.jar + * GRPC + - grpc-api-1.42.1.jar + - grpc-context-1.42.1.jar + - grpc-core-1.42.1.jar + - grpc-grpclb-1.42.1.jar + - grpc-netty-1.42.1.jar + - grpc-protobuf-1.42.1.jar + - grpc-protobuf-lite-1.42.1.jar + - grpc-stub-1.42.1.jar + * JEtcd + - jetcd-common-0.5.11.jar + - jetcd-core-0.5.11.jar + * Joda Time - joda-time-2.10.5.jar - joda-time-2.10.1.jar @@ -465,8 +488,8 @@ The Apache Software License, Version 2.0 Protocol Buffers License * Protocol Buffers - protobuf-shaded-2.1.0-incubating.jar - - protobuf-java-3.16.1.jar - - protobuf-java-util-3.16.1.jar + - protobuf-java-3.19.2.jar + - protobuf-java-util-3.19.2.jar BSD 3-clause "New" or "Revised" License * RE2J TD -- re2j-td-1.4.jar From 6930e9d43725f5051a6a4993ead7dc8a014dd5ad Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 18 May 2022 15:06:06 +0300 Subject: [PATCH 02/11] Upgrade Netty to 4.1.77.Final and netty-tcnative to 2.0.52.Final (#15646) - release notes https://netty.io/news/2022/05/06/2-1-77-Final.html - improves Alpine / musl compatibility - could help issues such as #14534 #11415 #11224 #10798 - improves shading compatibility - fixes a bug related to the native epoll transport and epoll_pwait2 (cherry picked from commit a8045fc4d6c5a61ad5d288c9d68e91b028ee3879) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 50 +++++++++---------- pom.xml | 4 +- pulsar-sql/presto-distribution/LICENSE | 48 +++++++++--------- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 4cf0a3034b4f0..4236cd27a21ab 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -59,7 +59,7 @@ io.netty netty-common - 4.1.76.Final + 4.1.77.Final test diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index cf7c1bb961537..0bc1e779ed6ff 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -357,31 +357,31 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.6.jar * Netty - - io.netty-netty-buffer-4.1.76.Final.jar - - io.netty-netty-codec-4.1.76.Final.jar - - io.netty-netty-codec-dns-4.1.76.Final.jar - - io.netty-netty-codec-http-4.1.76.Final.jar - - io.netty-netty-codec-http2-4.1.76.Final.jar - - io.netty-netty-codec-socks-4.1.76.Final.jar - - io.netty-netty-codec-haproxy-4.1.76.Final.jar - - io.netty-netty-common-4.1.76.Final.jar - - io.netty-netty-handler-4.1.76.Final.jar - - io.netty-netty-handler-proxy-4.1.76.Final.jar - - io.netty-netty-resolver-4.1.76.Final.jar - - io.netty-netty-resolver-dns-4.1.76.Final.jar - - io.netty-netty-transport-4.1.76.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.76.Final.jar - - io.netty-netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.76.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.76.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.51.Final.jar + - io.netty-netty-buffer-4.1.77.Final.jar + - io.netty-netty-codec-4.1.77.Final.jar + - io.netty-netty-codec-dns-4.1.77.Final.jar + - io.netty-netty-codec-http-4.1.77.Final.jar + - io.netty-netty-codec-http2-4.1.77.Final.jar + - io.netty-netty-codec-socks-4.1.77.Final.jar + - io.netty-netty-codec-haproxy-4.1.77.Final.jar + - io.netty-netty-common-4.1.77.Final.jar + - io.netty-netty-handler-4.1.77.Final.jar + - io.netty-netty-handler-proxy-4.1.77.Final.jar + - io.netty-netty-resolver-4.1.77.Final.jar + - io.netty-netty-resolver-dns-4.1.77.Final.jar + - io.netty-netty-transport-4.1.77.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.77.Final.jar + - io.netty-netty-transport-native-epoll-4.1.77.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.77.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.77.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.77.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.52.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.52.Final.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar diff --git a/pom.xml b/pom.xml index 1ff708b6863b8..d8e004b437354 100644 --- a/pom.xml +++ b/pom.xml @@ -100,8 +100,8 @@ flexible messaging model and an intuitive client API. 4.12.1 3.5.9 - 4.1.76.Final - 2.0.51.Final + 4.1.77.Final + 2.0.52.Final 9.4.43.v20210629 2.31 1.10.9 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 238c246e93228..280e6d8295e4f 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -233,31 +233,31 @@ The Apache Software License, Version 2.0 - commons-lang3-3.6.jar * Netty - netty-3.10.6.Final.jar - - netty-buffer-4.1.76.Final.jar - - netty-codec-4.1.76.Final.jar - - netty-codec-dns-4.1.76.Final.jar - - netty-codec-http-4.1.76.Final.jar - - netty-codec-haproxy-4.1.76.Final.jar - - netty-codec-socks-4.1.76.Final.jar - - netty-handler-proxy-4.1.76.Final.jar - - netty-common-4.1.76.Final.jar - - netty-handler-4.1.76.Final.jar + - netty-buffer-4.1.77.Final.jar + - netty-codec-4.1.77.Final.jar + - netty-codec-dns-4.1.77.Final.jar + - netty-codec-http-4.1.77.Final.jar + - netty-codec-haproxy-4.1.77.Final.jar + - netty-codec-socks-4.1.77.Final.jar + - netty-handler-proxy-4.1.77.Final.jar + - netty-common-4.1.77.Final.jar + - netty-handler-4.1.77.Final.jar - netty-reactive-streams-2.0.4.jar - - netty-resolver-4.1.76.Final.jar - - netty-resolver-dns-4.1.76.Final.jar - - netty-tcnative-boringssl-static-2.0.51.Final.jar - - netty-tcnative-boringssl-static-2.0.51.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.51.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.51.Final.jar - - netty-transport-4.1.76.Final.jar - - netty-transport-classes-epoll-4.1.76.Final.jar - - netty-transport-native-epoll-4.1.76.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.76.Final.jar - - netty-transport-native-unix-common-4.1.76.Final-linux-x86_64.jar - - netty-codec-http2-4.1.76.Final.jar + - netty-resolver-4.1.77.Final.jar + - netty-resolver-dns-4.1.77.Final.jar + - netty-tcnative-boringssl-static-2.0.52.Final.jar + - netty-tcnative-boringssl-static-2.0.52.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.52.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.52.Final.jar + - netty-transport-4.1.77.Final.jar + - netty-transport-classes-epoll-4.1.77.Final.jar + - netty-transport-native-epoll-4.1.77.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.77.Final.jar + - netty-transport-native-unix-common-4.1.77.Final-linux-x86_64.jar + - netty-codec-http2-4.1.77.Final.jar * GRPC - grpc-api-1.42.1.jar - grpc-context-1.42.1.jar From 2bd9674b7f37268816adf8637dc2b2d6195e489d Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Mon, 1 Aug 2022 15:01:30 +0800 Subject: [PATCH 03/11] Fix License --- .../server/src/assemble/LICENSE.bin.txt | 19 +++++++++++++++---- pulsar-sql/presto-distribution/LICENSE | 16 ---------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 0bc1e779ed6ff..cffbf4e5632b3 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -321,7 +321,8 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.11.1.jar - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.11.1.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.6.2.jar - * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.12.0.jar + * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.1.jar + * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar * Joda -- org.joda-joda-convert-2.2.1.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.7.2.jar * Gson @@ -459,8 +460,8 @@ The Apache Software License, Version 2.0 * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.3.4.jar * Apache Thrifth - org.apache.thrift-libthrift-0.12.0.jar * OkHttp - - com.squareup.okhttp-okhttp-2.5.0.jar - * Okio - com.squareup.okio-okio-1.13.0.jar + - com.squareup.okhttp-okhttp-2.7.4.jar + * Okio - com.squareup.okio-okio-1.17.5.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * gRPC - io.grpc-grpc-all-1.45.1.jar @@ -478,9 +479,14 @@ The Apache Software License, Version 2.0 - io.grpc-grpc-services-1.45.1.jar - io.grpc-grpc-xds-1.45.1.jar - io.grpc-grpc-rls-1.45.1.jar + - io.grpc-grpc-okhttp-1.45.1.jar + - io.grpc-grpc-protobuf-nano-1.18.0.jar + - io.grpc-grpc-testing-1.18.0.jar * OpenCensus - io.opencensus-opencensus-api-0.28.0.jar - io.opencensus-opencensus-contrib-http-util-0.28.0.jar + - io.opencensus-opencensus-contrib-grpc-metrics-0.28.0.jar + - io.opencensus-opencensus-proto-0.2.0.jar * Jodah - net.jodah-typetools-0.5.0.jar * Apache Avro @@ -525,6 +531,11 @@ The Apache Software License, Version 2.0 - org.apache.zookeeper-zookeeper-jute-3.5.9.jar * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar + * Google HTTP Client + - com.google.auto.value-auto-value-annotations-1.9.jar + - com.google.http-client-google-http-client-1.41.0.jar + - com.google.http-client-google-http-client-gson-1.41.0.jar + - com.google.re2j-re2j-1.5.jar BSD 3-clause "New" or "Revised" License * Google auth library @@ -544,7 +555,7 @@ MIT License - org.slf4j-slf4j-api-1.7.25.jar - org.slf4j-jcl-over-slf4j-1.7.25.jar * Animal Sniffer Annotations - - org.codehaus.mojo-animal-sniffer-annotations-1.17.jar + - org.codehaus.mojo-animal-sniffer-annotations-1.19.jar * The Checker Framework - org.checkerframework-checker-compat-qual-2.5.2.jar - org.checkerframework-checker-qual-3.5.0.jar diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 280e6d8295e4f..38a47ceb3da3b 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -238,8 +238,6 @@ The Apache Software License, Version 2.0 - netty-codec-dns-4.1.77.Final.jar - netty-codec-http-4.1.77.Final.jar - netty-codec-haproxy-4.1.77.Final.jar - - netty-codec-socks-4.1.77.Final.jar - - netty-handler-proxy-4.1.77.Final.jar - netty-common-4.1.77.Final.jar - netty-handler-4.1.77.Final.jar - netty-reactive-streams-2.0.4.jar @@ -257,20 +255,6 @@ The Apache Software License, Version 2.0 - netty-transport-native-epoll-4.1.77.Final-linux-x86_64.jar - netty-transport-native-unix-common-4.1.77.Final.jar - netty-transport-native-unix-common-4.1.77.Final-linux-x86_64.jar - - netty-codec-http2-4.1.77.Final.jar - * GRPC - - grpc-api-1.42.1.jar - - grpc-context-1.42.1.jar - - grpc-core-1.42.1.jar - - grpc-grpclb-1.42.1.jar - - grpc-netty-1.42.1.jar - - grpc-protobuf-1.42.1.jar - - grpc-protobuf-lite-1.42.1.jar - - grpc-stub-1.42.1.jar - * JEtcd - - jetcd-common-0.5.11.jar - - jetcd-core-0.5.11.jar - * Joda Time - joda-time-2.10.5.jar - joda-time-2.10.1.jar From 6572c368f97db7a0e1617d42de9b9006d2aee7a1 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Mon, 1 Aug 2022 20:00:32 +0800 Subject: [PATCH 04/11] fix testTxnCommitMarker --- .../apache/pulsar/common/protocol/MarkersTest.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/protocol/MarkersTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/protocol/MarkersTest.java index 620cb35e56122..c1e857487bf13 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/protocol/MarkersTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/protocol/MarkersTest.java @@ -19,18 +19,13 @@ package org.apache.pulsar.common.protocol; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - import io.netty.buffer.ByteBuf; - import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; - import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata; import org.apache.pulsar.common.api.proto.PulsarMarkers; @@ -139,14 +134,6 @@ public void testTxnCommitMarker() throws IOException { } ByteBuf buf = Markers.newTxnCommitMarker(sequenceId, mostBits, leastBits, messageIdDataList); - for (MessageIdData messageIdData : messageIdDataList) { - try { - messageIdData.recycle(); - fail("message id data should be recycled after create the marker bytebuf."); - } catch (Exception e) { - assertTrue(e instanceof java.lang.IllegalStateException); - } - } MessageMetadata msgMetadata = Commands.parseMessageMetadata(buf); assertEquals(msgMetadata.getMarkerType(), PulsarMarkers.MarkerType.TXN_COMMIT_VALUE); From 852b82defabd0ea818e324184b7dfbae78144958 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 3 Aug 2022 17:39:56 +0800 Subject: [PATCH 05/11] fix ProcessRuntimeTest#testJavaConstructor failure --- pom.xml | 6 ++++++ pulsar-functions/instance/pom.xml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index d8e004b437354..56e5794099abc 100644 --- a/pom.xml +++ b/pom.xml @@ -834,6 +834,12 @@ flexible messaging model and an intuitive client API. io.grpc grpc-protobuf-lite ${grpc.version} + + + + + + diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 7052a7402b4fa..760b02be7d824 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -91,6 +91,10 @@ io.grpc grpc-all + + io.grpc + grpc-protobuf-lite + com.google.protobuf protobuf-java From 9129f8f41b0d98a8e4a3336c0044406cc59f6e1e Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Wed, 3 Aug 2022 20:52:40 +0800 Subject: [PATCH 06/11] bump bookkeeper.version to 4.14.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 56e5794099abc..8f6a2e2931447 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.12.1 + 4.14.5 3.5.9 4.1.77.Final 2.0.52.Final From 5e0219e66d5b143e3e166a6b12e568d331c1af54 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Thu, 4 Aug 2022 10:22:05 +0800 Subject: [PATCH 07/11] fix compile --- .../org/apache/bookkeeper/client/BookKeeperTestClient.java | 3 ++- .../org/apache/bookkeeper/client/PulsarMockBookKeeper.java | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java index 184fc7f4c025d..910c829761be4 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.zk.ZKMetadataClientDriver; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; @@ -32,7 +33,7 @@ public BookKeeperTestClient(ClientConfiguration conf) throws IOException, Interr } public ZooKeeper getZkHandle() { - return super.getZkHandle(); + return ((ZKMetadataClientDriver) metadataDriver).getZk(); } public ClientConfiguration getConf() { diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index 37d1cd2662a4f..4a2ca8759dd4e 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -29,7 +29,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicLong; - import org.apache.bookkeeper.client.AsyncCallback.CreateCallback; import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback; import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; @@ -39,6 +38,7 @@ import org.apache.bookkeeper.client.impl.OpenBuilderBase; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.zk.ZKMetadataClientDriver; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,9 +53,8 @@ public class PulsarMockBookKeeper extends BookKeeper { final ExecutorService executor; final ZooKeeper zkc; - @Override public ZooKeeper getZkHandle() { - return zkc; + return ((ZKMetadataClientDriver) metadataDriver).getZk(); } @Override From 8dc05e4450a544e1d6e1d35a7ab685109ded3998 Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Thu, 4 Aug 2022 11:17:53 +0800 Subject: [PATCH 08/11] [Build] Use grpc-bom to align grpc library versions --- pom.xml | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index 8f6a2e2931447..1ec4ab384927a 100644 --- a/pom.xml +++ b/pom.xml @@ -788,6 +788,14 @@ flexible messaging model and an intuitive client API. ${typetools.version} + + io.grpc + grpc-bom + ${grpc.version} + pom + import + + io.grpc grpc-all @@ -800,12 +808,6 @@ flexible messaging model and an intuitive client API. - - io.grpc - grpc-core - ${grpc.version} - - com.google.http-client google-http-client @@ -824,24 +826,6 @@ flexible messaging model and an intuitive client API. ${google-http-client.version} - - io.grpc - grpc-stub - ${grpc.version} - - - - io.grpc - grpc-protobuf-lite - ${grpc.version} - - - - - - - - com.google.protobuf protobuf-bom From 9769bae9eb0cce16ff8e18525cb77550889196b3 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Tue, 1 Jun 2021 14:58:28 +0800 Subject: [PATCH 09/11] Upgrade BookKeeper to 4.14.1 (#10686) * upgrade BookKeeper to 4.14.0 * format code * upgrade bookkeeper to 4.14.1 to fix prometheus metric bug * update license Co-authored-by: chenhang --- .../metrics/DataSketchesOpStatsLogger.java | 2 +- .../metrics/DataSketchesSummaryLogger.java | 2 +- .../prometheus/metrics/LongAdderCounter.java | 61 +++++++++++++++++++ .../prometheus/metrics/ObserverGauge.java | 2 + .../metrics/PrometheusMetricsProvider.java | 7 +-- .../metrics/PrometheusStatsLogger.java | 3 +- .../metrics/PrometheusTextFormatUtil.java | 3 +- .../stats/prometheus/metrics/SimpleGauge.java | 2 +- 8 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesOpStatsLogger.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesOpStatsLogger.java index 3ef453ddc654e..2dcfb10247100 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesOpStatsLogger.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesOpStatsLogger.java @@ -55,7 +55,7 @@ public class DataSketchesOpStatsLogger implements OpStatsLogger { private final LongAdder successSumAdder = new LongAdder(); private final LongAdder failSumAdder = new LongAdder(); - DataSketchesOpStatsLogger() { + public DataSketchesOpStatsLogger() { this.current = new ThreadLocalAccessor(); this.replacement = new ThreadLocalAccessor(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java index 96e7bd2fec59f..0ef11d8e37483 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java @@ -46,7 +46,7 @@ public class DataSketchesSummaryLogger { private final LongAdder countAdder = new LongAdder(); private final LongAdder sumAdder = new LongAdder(); - DataSketchesSummaryLogger() { + public DataSketchesSummaryLogger() { this.current = new ThreadLocalAccessor(); this.replacement = new ThreadLocalAccessor(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java new file mode 100644 index 0000000000000..39be12b35509b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus.metrics; + +import java.util.concurrent.atomic.LongAdder; +import org.apache.bookkeeper.stats.Counter; + +/** + * {@link Counter} implementation based on {@link LongAdder}. + * + *

LongAdder keeps a counter per-thread and then aggregates to get the result, in order to avoid contention between + * multiple threads. + */ +public class LongAdderCounter implements Counter { + private final LongAdder counter = new LongAdder(); + + public LongAdderCounter() { + + } + + @Override + public void clear() { + counter.reset(); + } + + @Override + public void inc() { + counter.increment(); + } + + @Override + public void dec() { + counter.decrement(); + } + + @Override + public void add(long delta) { + counter.add(delta); + } + + @Override + public Long get() { + return counter.sum(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/ObserverGauge.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/ObserverGauge.java index 8cc7ff3c04963..93f39c802ce9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/ObserverGauge.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/ObserverGauge.java @@ -33,6 +33,8 @@ public class ObserverGauge extends SimpleCollector implemen public static class Builder extends SimpleCollector.Builder { private Supplier supplier; + public Builder() {} + public Builder supplier(Supplier supplier) { this.supplier = supplier; return this; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java index e05c6c43326cc..1007f41c47dad 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusMetricsProvider.java @@ -24,7 +24,6 @@ import org.apache.bookkeeper.stats.CachingStatsProvider; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.StatsProvider; -import org.apache.bookkeeper.stats.prometheus.LongAdderCounter; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; @@ -53,9 +52,9 @@ public class PrometheusMetricsProvider implements StatsProvider { /** * These acts a registry of the metrics defined in this provider */ - final ConcurrentMap counters = new ConcurrentSkipListMap<>(); - final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); - final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); + public final ConcurrentMap counters = new ConcurrentSkipListMap<>(); + public final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); + public final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); public PrometheusMetricsProvider() { this.cachingStatsProvider = new CachingStatsProvider(new StatsProvider() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusStatsLogger.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusStatsLogger.java index ad3c62f78d731..43efcc9b0b812 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusStatsLogger.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusStatsLogger.java @@ -24,7 +24,6 @@ import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.OpStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.bookkeeper.stats.prometheus.LongAdderCounter; /** * A {@code Prometheus} based {@link StatsLogger} implementation. @@ -34,7 +33,7 @@ public class PrometheusStatsLogger implements StatsLogger { private final PrometheusMetricsProvider provider; private final String scope; - PrometheusStatsLogger(PrometheusMetricsProvider provider, String scope) { + public PrometheusStatsLogger(PrometheusMetricsProvider provider, String scope) { this.provider = provider; this.scope = scope; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java index abe0b560aa746..0f9a4c86924fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java @@ -128,7 +128,8 @@ private static void writeSum(Writer w, DataSketchesOpStatsLogger opStat, String .append(Double.toString(opStat.getSum(success))).append('\n'); } - static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) throws IOException { + public static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) + throws IOException { Enumeration metricFamilySamples = registry.metricFamilySamples(); while (metricFamilySamples.hasMoreElements()) { MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/SimpleGauge.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/SimpleGauge.java index a93a26c1bd1e6..612168a2d9143 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/SimpleGauge.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/SimpleGauge.java @@ -31,7 +31,7 @@ public SimpleGauge(final Gauge gauge) { this.gauge = gauge; } - Number getSample() { + public Number getSample() { return gauge.getSample(); } } From 80a1c30b907ca8db3ca39ff424f2de7a6c07591e Mon Sep 17 00:00:00 2001 From: JiangHaiting Date: Thu, 4 Aug 2022 13:07:23 +0800 Subject: [PATCH 10/11] fix compile --- .../server/src/assemble/LICENSE.bin.txt | 77 +++++++++---------- .../server/src/assemble/NOTICE.bin.txt | 1 - pulsar-sql/presto-distribution/LICENSE | 33 ++++---- .../client/PulsarMockBookKeeper.java | 3 +- 4 files changed, 54 insertions(+), 60 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index cffbf4e5632b3..e0e007d5c826b 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -403,35 +403,35 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.12.1.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.12.1.jar - - org.apache.bookkeeper-bookkeeper-proto-4.12.1.jar - - org.apache.bookkeeper-bookkeeper-server-4.12.1.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.12.1.jar - - org.apache.bookkeeper-circe-checksum-4.12.1.jar - - org.apache.bookkeeper-cpu-affinity-4.12.1.jar - - org.apache.bookkeeper-statelib-4.12.1.jar - - org.apache.bookkeeper-stream-storage-api-4.12.1.jar - - org.apache.bookkeeper-stream-storage-common-4.12.1.jar - - org.apache.bookkeeper-stream-storage-java-client-4.12.1.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.12.1.jar - - org.apache.bookkeeper-stream-storage-proto-4.12.1.jar - - org.apache.bookkeeper-stream-storage-server-4.12.1.jar - - org.apache.bookkeeper-stream-storage-service-api-4.12.1.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.12.1.jar - - org.apache.bookkeeper.http-http-server-4.12.1.jar - - org.apache.bookkeeper.http-vertx-http-server-4.12.1.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.12.1.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.12.1.jar - - org.apache.bookkeeper.tests-stream-storage-tests-common-4.12.1.jar - - org.apache.distributedlog-distributedlog-common-4.12.1.jar - - org.apache.distributedlog-distributedlog-core-4.12.1-tests.jar - - org.apache.distributedlog-distributedlog-core-4.12.1.jar - - org.apache.distributedlog-distributedlog-protocol-4.12.1.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.12.1.jar + - org.apache.bookkeeper-bookkeeper-common-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-proto-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-server-4.14.5.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.14.5.jar + - org.apache.bookkeeper-circe-checksum-4.14.5.jar + - org.apache.bookkeeper-cpu-affinity-4.14.5.jar + - org.apache.bookkeeper-statelib-4.14.5.jar + - org.apache.bookkeeper-stream-storage-api-4.14.5.jar + - org.apache.bookkeeper-stream-storage-common-4.14.5.jar + - org.apache.bookkeeper-stream-storage-java-client-4.14.5.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.14.5.jar + - org.apache.bookkeeper-stream-storage-proto-4.14.5.jar + - org.apache.bookkeeper-stream-storage-server-4.14.5.jar + - org.apache.bookkeeper-stream-storage-service-api-4.14.5.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.14.5.jar + - org.apache.bookkeeper.http-http-server-4.14.5.jar + - org.apache.bookkeeper.http-vertx-http-server-4.14.5.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.5.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.5.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.5.jar + - org.apache.bookkeeper.tests-stream-storage-tests-common-4.14.5.jar + - org.apache.distributedlog-distributedlog-common-4.14.5.jar + - org.apache.distributedlog-distributedlog-core-4.14.5-tests.jar + - org.apache.distributedlog-distributedlog-core-4.14.5.jar + - org.apache.distributedlog-distributedlog-protocol-4.14.5.jar * Apache HTTP Client - - org.apache.httpcomponents-httpclient-4.5.5.jar - - org.apache.httpcomponents-httpcore-4.4.9.jar + - org.apache.httpcomponents-httpclient-4.5.13.jar + - org.apache.httpcomponents-httpcore-4.4.13.jar * AirCompressor - io.airlift-aircompressor-0.20.jar * AsyncHttpClient @@ -458,7 +458,7 @@ The Apache Software License, Version 2.0 * SnakeYaml -- org.yaml-snakeyaml-1.26.jar * RocksDB - org.rocksdb-rocksdbjni-6.10.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.3.4.jar - * Apache Thrifth - org.apache.thrift-libthrift-0.12.0.jar + * Apache Thrifth - org.apache.thrift-libthrift-0.14.2.jar * OkHttp - com.squareup.okhttp-okhttp-2.7.4.jar * Okio - com.squareup.okio-okio-1.17.5.jar @@ -480,12 +480,10 @@ The Apache Software License, Version 2.0 - io.grpc-grpc-xds-1.45.1.jar - io.grpc-grpc-rls-1.45.1.jar - io.grpc-grpc-okhttp-1.45.1.jar - - io.grpc-grpc-protobuf-nano-1.18.0.jar - - io.grpc-grpc-testing-1.18.0.jar + - io.grpc-grpc-testing-1.45.1.jar * OpenCensus - io.opencensus-opencensus-api-0.28.0.jar - io.opencensus-opencensus-contrib-http-util-0.28.0.jar - - io.opencensus-opencensus-contrib-grpc-metrics-0.28.0.jar - io.opencensus-opencensus-proto-0.2.0.jar * Jodah - net.jodah-typetools-0.5.0.jar @@ -499,7 +497,7 @@ The Apache Software License, Version 2.0 * Apache Yetus - org.apache.yetus-audience-annotations-0.5.0.jar * @FreeBuilder - - org.inferred-freebuilder-1.14.9.jar + - org.inferred-freebuilder-2.7.0.jar * Squareup - com.squareup.okhttp3-logging-interceptor-3.14.3.jar - com.squareup.okhttp3-okhttp-3.14.3.jar @@ -511,9 +509,9 @@ The Apache Software License, Version 2.0 - joda-time-2.10.1.jar - joda-time-joda-time-2.10.1.jar * Dropwizard - - io.dropwizard.metrics-metrics-core-3.1.0.jar - - io.dropwizard.metrics-metrics-graphite-3.1.0.jar - - io.dropwizard.metrics-metrics-jvm-3.1.0.jar + - io.dropwizard.metrics-metrics-core-3.2.5.jar + - io.dropwizard.metrics-metrics-graphite-3.2.5.jar + - io.dropwizard.metrics-metrics-jvm-3.2.5.jar * Prometheus - io.prometheus-simpleclient_httpserver-0.5.0.jar * Java JSON WebTokens @@ -531,10 +529,13 @@ The Apache Software License, Version 2.0 - org.apache.zookeeper-zookeeper-jute-3.5.9.jar * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar + * Snappy Java + - org.xerial.snappy-snappy-java-1.1.7.jar * Google HTTP Client - com.google.auto.value-auto-value-annotations-1.9.jar - com.google.http-client-google-http-client-1.41.0.jar - com.google.http-client-google-http-client-gson-1.41.0.jar + - com.google.http-client-google-http-client-jackson2-1.41.0.jar - com.google.re2j-re2j-1.5.jar BSD 3-clause "New" or "Revised" License @@ -554,17 +555,13 @@ MIT License - org.slf4j-jul-to-slf4j-1.7.25.jar - org.slf4j-slf4j-api-1.7.25.jar - org.slf4j-jcl-over-slf4j-1.7.25.jar - * Animal Sniffer Annotations - - org.codehaus.mojo-animal-sniffer-annotations-1.19.jar * The Checker Framework - - org.checkerframework-checker-compat-qual-2.5.2.jar - org.checkerframework-checker-qual-3.5.0.jar Protocol Buffers License * Protocol Buffers - com.google.protobuf-protobuf-java-3.19.2.jar -- licenses/LICENSE-protobuf.txt - com.google.protobuf-protobuf-java-util-3.19.2.jar -- licenses/LICENSE-protobuf.txt - - com.google.protobuf.nano-protobuf-javanano-3.0.0-alpha-5.jar -- licenses/LICENSE-protobuf.txt CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt * Java Annotations API diff --git a/distribution/server/src/assemble/NOTICE.bin.txt b/distribution/server/src/assemble/NOTICE.bin.txt index 119cb6f6ec8ba..fbab905a6e042 100644 --- a/distribution/server/src/assemble/NOTICE.bin.txt +++ b/distribution/server/src/assemble/NOTICE.bin.txt @@ -201,7 +201,6 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html * Performance and memory usage improvement ------------------------------------------------------------------------------------ -- org.checkerframework-checker-compat-qual-2.5.2.jar A few parts of the Checker Framework have more permissive licenses. diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 38a47ceb3da3b..82ae02543f1d9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -311,8 +311,8 @@ The Apache Software License, Version 2.0 - trace-token-0.195.jar - units-1.6.jar * Apache HTTP Client - - httpclient-4.5.5.jar - - httpcore-4.4.9.jar + - httpclient-4.5.13.jar + - httpcore-4.4.13.jar * Error Prone Annotations - error_prone_annotations-2.3.4.jar * Esri Geometry API For Java @@ -403,9 +403,9 @@ The Apache Software License, Version 2.0 * Objectsize - objectsize-0.0.12.jar * Dropwizard Metrics - - metrics-core-3.1.0.jar - - metrics-graphite-3.1.0.jar - - metrics-jvm-3.1.0.jar + - metrics-core-3.2.5.jar + - metrics-graphite-3.2.5.jar + - metrics-jvm-3.2.5.jar * Prometheus - simpleclient-0.5.0.jar - simpleclient_common-0.5.0.jar @@ -418,18 +418,17 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.12.1.jar - - bookkeeper-common-allocator-4.12.1.jar - - bookkeeper-proto-4.12.1.jar - - bookkeeper-server-4.12.1.jar - - bookkeeper-stats-api-4.12.1.jar - - bookkeeper-tools-framework-4.12.1.jar - - circe-checksum-4.12.1.jar - - codahale-metrics-provider-4.12.1.jar - - cpu-affinity-4.12.1.jar - - http-server-4.12.1.jar - - prometheus-metrics-provider-4.12.1.jar - - codahale-metrics-provider-4.12.1.jar + - bookkeeper-common-4.14.5.jar + - bookkeeper-common-allocator-4.14.5.jar + - bookkeeper-proto-4.14.5.jar + - bookkeeper-server-4.14.5.jar + - bookkeeper-stats-api-4.14.5.jar + - bookkeeper-tools-framework-4.14.5.jar + - circe-checksum-4.14.5.jar + - codahale-metrics-provider-4.14.5.jar + - cpu-affinity-4.14.5.jar + - prometheus-metrics-provider-4.14.5.jar + - http-server-4.14.5.jar * Apache Commons - commons-cli-1.2.jar - commons-codec-1.10.jar diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index 4a2ca8759dd4e..8503947005cd8 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -38,7 +38,6 @@ import org.apache.bookkeeper.client.impl.OpenBuilderBase; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.conf.ClientConfiguration; -import org.apache.bookkeeper.meta.zk.ZKMetadataClientDriver; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +53,7 @@ public class PulsarMockBookKeeper extends BookKeeper { final ZooKeeper zkc; public ZooKeeper getZkHandle() { - return ((ZKMetadataClientDriver) metadataDriver).getZk(); + return zkc; } @Override From 3b3b5a1eac6bdf998c1a6bbf1182c0c2f95b388f Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 4 Mar 2021 16:33:12 -0800 Subject: [PATCH 11/11] [Issue 9728] Upgrade GRPC and bookkeeper (#9729) * Upgraded bookkeeper to 4.13.0; grpc to 1.33 * updated LICENSE * fixed the rest of the license check * no idea why license check results on CI are different from local :/ * one more license fix * Integration tests failed to start container/service there because of java.lang.NoClassDefFoundError: io/perfmark/PerfMark * another try to get the permark.jar in * another take on perfmark.jar and integration tests * add perfmark.jar to LICENSE --- distribution/server/pom.xml | 6 ++++++ distribution/server/src/assemble/LICENSE.bin.txt | 5 ++++- docker/pulsar-all/pom.xml | 5 +++++ pom.xml | 14 ++++++++++++++ pulsar-broker-shaded/pom.xml | 1 + pulsar-client-admin-shaded/pom.xml | 1 + pulsar-client-all/pom.xml | 1 + pulsar-client-shaded/pom.xml | 1 + pulsar-functions/instance/pom.xml | 6 ++++++ pulsar-functions/localrun-shaded/pom.xml | 4 ++++ pulsar-functions/localrun/pom.xml | 7 +++++++ pulsar-functions/proto/pom.xml | 6 ++++++ pulsar-zookeeper-utils/pom.xml | 6 ++++++ 13 files changed, 62 insertions(+), 1 deletion(-) diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 6f77e696ee3e5..9eb49d755a35a 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -182,6 +182,12 @@ grpc-all + + io.perfmark + perfmark-api + compile + + org.apache.bookkeeper.http diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index e0e007d5c826b..6eff25d78cad9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -481,6 +481,8 @@ The Apache Software License, Version 2.0 - io.grpc-grpc-rls-1.45.1.jar - io.grpc-grpc-okhttp-1.45.1.jar - io.grpc-grpc-testing-1.45.1.jar + * Perfmark + - io.perfmark-perfmark-api-0.19.0.jar * OpenCensus - io.opencensus-opencensus-api-0.28.0.jar - io.opencensus-opencensus-contrib-http-util-0.28.0.jar @@ -591,7 +593,8 @@ CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt * Mimepull -- org.jvnet.mimepull-mimepull-1.9.13.jar Eclipse Distribution License 1.0 -- licenses/LICENSE-EDL-1.0.txt - * Jakarta Activation -- jakarta.activation-jakarta.activation-api-1.2.1.jar + * Jakarta Activation + - jakarta.activation-jakarta.activation-api-1.2.1.jar * Jakarta XML Binding -- jakarta.xml.bind-jakarta.xml.bind-api-2.3.2.jar Eclipse Public License 1.0 -- licenses/LICENSE-AspectJ.txt diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 0d3e40097aa59..c93db63093db0 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -38,6 +38,11 @@ pom provided + + io.perfmark + perfmark-api + runtime + ${project.groupId} pulsar-offloader-distribution diff --git a/pom.xml b/pom.xml index 1ec4ab384927a..ded0749c74c69 100644 --- a/pom.xml +++ b/pom.xml @@ -125,6 +125,7 @@ flexible messaging model and an intuitive client API. 3.19.2 ${protobuf3.version} 1.45.1 + 0.19.0 1.41.0 ${grpc.version} 2.8.9 @@ -814,6 +815,19 @@ flexible messaging model and an intuitive client API. ${google-http-client.version} + + io.perfmark + perfmark-api + ${perfmark.version} + runtime + + + error_prone_annotations + com.google.errorprone + + + + com.google.http-client google-http-client-jackson2 diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 035e2e14f746b..47ce4da802d98 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -82,6 +82,7 @@ net.java.dev.jna:* com.carrotsearch:* io.prometheus:* + io.perfmark:* com.github.ben-manes.caffeine:* org.glassfish.jersey.*:* org.rocksdb:* diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index de67904b64495..1102587b0a106 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -87,6 +87,7 @@ org.glassfish.hk2*:* com.fasterxml.jackson.*:* io.grpc:* + io.perfmark:* com.yahoo.datasketches:* com.squareup.*:* com.google.*:* diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 490360e2dcbe7..a71e66ed000c3 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -134,6 +134,7 @@ org.glassfish.hk2*:* com.fasterxml.jackson.*:* io.grpc:* + io.perfmark:* com.yahoo.datasketches:* io.netty:* com.squareup.*:* diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index f8d3e1e8d4395..206f574767c32 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -123,6 +123,7 @@ com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.dataformat io.netty:* + io.perfmark:* org.eclipse.jetty:* com.yahoo.datasketches:* commons-*:* diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 760b02be7d824..e541a540d9ce7 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -113,6 +113,12 @@ grpc-all + + io.perfmark + perfmark-api + runtime + + org.apache.bookkeeper bookkeeper-common diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 5de0b06049f4a..69c334b76f238 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -132,6 +132,10 @@ io.grpc org.apache.pulsar.functions.runtime.shaded.io.grpc + + io.perfmark + org.apache.pulsar.functions.runtime.shaded.io.perfmark + org.joda org.apache.pulsar.functions.runtime.shaded.org.joda diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index b474296eb20e6..e591ef32620e8 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -58,5 +58,12 @@ io.grpc grpc-all + + + io.perfmark + perfmark-api + runtime + + diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index f85b21daa4d2c..be6bcb8b6e724 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -61,6 +61,12 @@ + + io.perfmark + perfmark-api + runtime + + diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index fa4768d9b62db..7eee71298e9ea 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -61,6 +61,12 @@ grpc-all + + io.perfmark + perfmark-api + runtime + + org.apache.bookkeeper.stats prometheus-metrics-provider