From 967fd97d4ea3f68542dcd4b548bfccf92a207560 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Mon, 11 Nov 2019 11:20:54 +0800 Subject: [PATCH 1/5] fix prometheusMetrics export contains " bug --- .../stats/prometheus/PrometheusMetricsGenerator.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index e346d27c2913c..d3213fd2697bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -95,10 +95,17 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String stream.write(sample.name); stream.write("{cluster=\"").write(cluster).write('"'); for (int j = 0; j < sample.labelNames.size(); j++) { + String labelValue = sample.labelValues.get(j); + if(labelValue != null && + (labelValue.startsWith("\"") || + labelValue.endsWith("\""))){ + labelValue = labelValue.replace("\"", "\\\""); + } + stream.write(","); stream.write(sample.labelNames.get(j)); stream.write("=\""); - stream.write(sample.labelValues.get(j)); + stream.write(labelValue); stream.write('"'); } From e6db8545c68ab284114e4769706c1479c7c3ace8 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Mon, 11 Nov 2019 16:50:12 +0800 Subject: [PATCH 2/5] simplify code for PrometheusMetrics " bug fix --- .../broker/stats/prometheus/PrometheusMetricsGenerator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index d3213fd2697bd..1af8a7bfb4155 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -96,9 +96,7 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String stream.write("{cluster=\"").write(cluster).write('"'); for (int j = 0; j < sample.labelNames.size(); j++) { String labelValue = sample.labelValues.get(j); - if(labelValue != null && - (labelValue.startsWith("\"") || - labelValue.endsWith("\""))){ + if(labelValue != null){ labelValue = labelValue.replace("\"", "\\\""); } From 3620e085a12dd9750a1c331632d7f7ceed98b8b5 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Tue, 12 Nov 2019 10:00:03 +0800 Subject: [PATCH 3/5] Update pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java format code style Co-Authored-By: Sijie Guo --- .../broker/stats/prometheus/PrometheusMetricsGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index 1af8a7bfb4155..b2259583bbeae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -96,7 +96,7 @@ private static void generateSystemMetrics(SimpleTextOutputStream stream, String stream.write("{cluster=\"").write(cluster).write('"'); for (int j = 0; j < sample.labelNames.size(); j++) { String labelValue = sample.labelValues.get(j); - if(labelValue != null){ + if (labelValue != null) { labelValue = labelValue.replace("\"", "\\\""); } From 665ece0ed87f138b24eac06d95296ed4b35c6c49 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sat, 16 Nov 2019 15:47:33 +0800 Subject: [PATCH 4/5] fix bkenv has no effect bug when start bookie using bin/pulsar-daemon --- bin/pulsar | 8 +++++++- bin/pulsar-daemon | 6 ++++++ bin/pulsar-perf | 6 ++++++ conf/pulsar_env.sh | 4 ---- src/pulsar-io-gen | 6 ++++++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/bin/pulsar b/bin/pulsar index ce79cfbfa4459..31d29324e9486 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -44,12 +44,18 @@ FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS SQL_HOME=$PULSAR_HOME/pulsar-sql PRESTO_HOME=${PULSAR_HOME}/lib/presto - +# Check pulsar env and load pulser_env.sh if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] then . "$PULSAR_HOME/conf/pulsar_env.sh" fi +# Check bookkeeper env and load bkenv.sh +if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] +then + . "$PULSAR_HOME/conf/bkenv.sh" +fi + # Check for the java to use if [[ -z $JAVA_HOME ]]; then JAVA=$(which java) diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon index 89c66a9cc6b51..925945918466d 100755 --- a/bin/pulsar-daemon +++ b/bin/pulsar-daemon @@ -45,6 +45,12 @@ then . "$PULSAR_HOME/conf/pulsar_env.sh" fi +# Check bookkeeper env and load bkenv.sh +if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] +then + . "$PULSAR_HOME/conf/bkenv.sh" +fi + PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RollingFile"} PULSAR_STOP_TIMEOUT=${PULSAR_STOP_TIMEOUT:-30} diff --git a/bin/pulsar-perf b/bin/pulsar-perf index af2d010f6f055..42e8c1e3778cd 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -29,6 +29,12 @@ then . "$PULSAR_HOME/conf/pulsar_env.sh" fi +# Check bookkeeper env and load bkenv.sh +if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] +then + . "$PULSAR_HOME/conf/bkenv.sh" +fi + # Check for the java to use if [[ -z $JAVA_HOME ]]; then JAVA=$(which java) diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh index 7428d1d468398..98155f28560a9 100755 --- a/conf/pulsar_env.sh +++ b/conf/pulsar_env.sh @@ -59,7 +59,3 @@ PULSAR_EXTRA_OPTS=${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -Di #Wait time before forcefully kill the pulser server instance, if the stop is not successful #PULSAR_STOP_TIMEOUT= -# Set BOOKIE_EXTRA_OPTS option here to ensure that all pulsar scripts can work seamless with bookkeeper - -# Extra options to be passed to the jvm -BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" \ No newline at end of file diff --git a/src/pulsar-io-gen b/src/pulsar-io-gen index 05ace66b748ee..8f9515f0b1060 100755 --- a/src/pulsar-io-gen +++ b/src/pulsar-io-gen @@ -28,6 +28,12 @@ then . "$PULSAR_HOME/conf/pulsar_env.sh" fi +# Check bookkeeper env and load bkenv.sh +if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] +then + . "$PULSAR_HOME/conf/bkenv.sh" +fi + # Check for the java to use if [[ -z $JAVA_HOME ]]; then JAVA=$(which java) From b43cc430704f65719e3b4f436048cc708e4a6e18 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Tue, 19 Nov 2019 10:35:56 +0800 Subject: [PATCH 5/5] movve load bkenv.conf before pulsar_env.conf for backward compatibility consideration, So if people already define environments in pulsar_env, those env vars will not be overridden by bkenv.sh --- bin/pulsar | 12 ++++++------ bin/pulsar-daemon | 10 +++++----- bin/pulsar-perf | 10 +++++----- src/pulsar-io-gen | 10 +++++----- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bin/pulsar b/bin/pulsar index 31d29324e9486..17f2cd825cf74 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -44,18 +44,18 @@ FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS SQL_HOME=$PULSAR_HOME/pulsar-sql PRESTO_HOME=${PULSAR_HOME}/lib/presto -# Check pulsar env and load pulser_env.sh -if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_env.sh" -fi - # Check bookkeeper env and load bkenv.sh if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] then . "$PULSAR_HOME/conf/bkenv.sh" fi +# Check pulsar env and load pulser_env.sh +if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] +then + . "$PULSAR_HOME/conf/pulsar_env.sh" +fi + # Check for the java to use if [[ -z $JAVA_HOME ]]; then JAVA=$(which java) diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon index 925945918466d..9848c26d33158 100755 --- a/bin/pulsar-daemon +++ b/bin/pulsar-daemon @@ -40,17 +40,17 @@ EOF BINDIR=$(dirname "$0") PULSAR_HOME=$(cd $BINDIR/..;pwd) -if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_env.sh" -fi - # Check bookkeeper env and load bkenv.sh if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] then . "$PULSAR_HOME/conf/bkenv.sh" fi +if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] +then + . "$PULSAR_HOME/conf/pulsar_env.sh" +fi + PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RollingFile"} PULSAR_STOP_TIMEOUT=${PULSAR_STOP_TIMEOUT:-30} diff --git a/bin/pulsar-perf b/bin/pulsar-perf index 42e8c1e3778cd..83814e9fcaf19 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -24,17 +24,17 @@ PULSAR_HOME=`cd $BINDIR/..;pwd` DEFAULT_CLIENT_CONF=${PULSAR_CLIENT_CONF:-"$PULSAR_HOME/conf/client.conf"} DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml -if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_env.sh" -fi - # Check bookkeeper env and load bkenv.sh if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] then . "$PULSAR_HOME/conf/bkenv.sh" fi +if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] +then + . "$PULSAR_HOME/conf/pulsar_env.sh" +fi + # Check for the java to use if [[ -z $JAVA_HOME ]]; then JAVA=$(which java) diff --git a/src/pulsar-io-gen b/src/pulsar-io-gen index 8f9515f0b1060..a3083e8f2b549 100755 --- a/src/pulsar-io-gen +++ b/src/pulsar-io-gen @@ -23,17 +23,17 @@ PULSAR_HOME=`cd $BINDIR/..;pwd` DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml -if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_env.sh" -fi - # Check bookkeeper env and load bkenv.sh if [ -f "$PULSAR_HOME/conf/bkenv.sh" ] then . "$PULSAR_HOME/conf/bkenv.sh" fi +if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] +then + . "$PULSAR_HOME/conf/pulsar_env.sh" +fi + # Check for the java to use if [[ -z $JAVA_HOME ]]; then JAVA=$(which java)