From fbb68dda56fbd8ba5d7558f024b1f6352e97abce Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 16:19:52 +0800 Subject: [PATCH 01/42] Support protobuf 3.22 or upper Signed-off-by: owent --- .github/workflows/ci.yml | 23 +++++++++++++++++++++++ CMakeLists.txt | 13 +++++++++++++ api/CMakeLists.txt | 2 -- ci/do_ci.sh | 4 ++++ ci/install_abseil.sh | 4 ++-- ci/install_protobuf.sh | 12 +++++++++--- exporters/otlp/src/otlp_http_client.cc | 12 +++++++++++- 7 files changed, 62 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ac8999459..b04179fdbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -387,6 +387,29 @@ jobs: sudo ./ci/setup_grpc.sh ./ci/do_ci.sh cmake.exporter.otprotocol.test + cmake_modern_protobuf_test: + name: CMake test (with modern protobuf and otlp-exporter) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: setup + env: + PROTOBUF_VERSION: '23.2' + ABSEIL_CPP_VERSION: '20230125.3' + run: | + sudo ./ci/setup_cmake.sh + sudo ./ci/setup_ci_environment.sh + sudo ./ci/install_abseil.sh + sudo ./ci/install_protobuf.sh + - name: run otlp exporter tests + env: + WITH_ABSEIL: 'ON' + run: | + sudo ./ci/setup_grpc.sh + ./ci/do_ci.sh cmake.exporter.otprotocol.test + cmake_do_not_install_test: name: CMake do not install test (with otlp-exporter) runs-on: ubuntu-20.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index 96941f0d91..b371831ace 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,9 +355,22 @@ if(WITH_PROMETHEUS) endif() endif() +if(WITH_ABSEIL) + find_package(absl CONFIG REQUIRED) +endif() + if(WITH_OTLP) set(protobuf_MODULE_COMPATIBLE ON) find_package(Protobuf) + if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") + if(WITH_ABSEIL) + message( + FATAL_ERROR + "Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125.0 or upper)" + ) + endif() + endif() + if(WITH_OTLP_GRPC OR (NOT DEFINED WITH_OTLP_GRPC AND NOT DEFINED CACHE{WITH_OTLP_GRPC})) find_package(gRPC) diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index 31863b25e3..da9f69db7e 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -48,8 +48,6 @@ endif() if(WITH_ABSEIL) - find_package(absl CONFIG REQUIRED) - target_compile_definitions(opentelemetry_api INTERFACE HAVE_ABSEIL) target_link_libraries( opentelemetry_api INTERFACE absl::bad_variant_access absl::any absl::base diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 04fdbc483c..26fc5804cb 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -258,9 +258,13 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then cd "${BUILD_DIR}" rm -rf * + if [[ -z "$WITH_ABSEIL" ]]; then + WITH_ABSEIL="OFF" + fi cmake -DCMAKE_BUILD_TYPE=Debug \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ + -DWITH_ABSEIL=$WITH_ABSEIL \ "${SRC_DIR}" grpc_cpp_plugin=`which grpc_cpp_plugin` proto_make_file="CMakeFiles/opentelemetry_proto.dir/build.make" diff --git a/ci/install_abseil.sh b/ci/install_abseil.sh index 7764b4d815..1184a1ddbc 100755 --- a/ci/install_abseil.sh +++ b/ci/install_abseil.sh @@ -5,12 +5,12 @@ set -e export DEBIAN_FRONTEND=noninteractive +[ -z "${ABSEIL_CPP_VERSION}" ] && export ABSEIL_CPP_VERSION="20220623.1" BUILD_DIR=/tmp/ INSTALL_DIR=/usr/local/ -TAG=20220623.1 pushd $BUILD_DIR -git clone --depth=1 -b ${TAG} https://github.com/abseil/abseil-cpp.git +git clone --depth=1 -b ${ABSEIL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git cd abseil-cpp mkdir build && pushd build cmake -DBUILD_TESTING=OFF -DCMAKE_CXX_STANDARD=11 \ diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index dde94111e1..bfc8b1f092 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -31,11 +31,17 @@ set -e # when calling this script # -export CPP_PROTOBUF_VERSION="3.${PROTOBUF_VERSION}" +if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then + export CPP_PROTOBUF_VERSION="${PROTOBUF_VERSION}" + CPP_PROTOBUF_PACKAGE_NAME="protobuf-${CPP_PROTOBUF_VERSION}" +else + export CPP_PROTOBUF_VERSION="3.${PROTOBUF_VERSION}" + CPP_PROTOBUF_PACKAGE_NAME="protobuf-cpp-${CPP_PROTOBUF_VERSION}" +fi cd / -wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${CPP_PROTOBUF_VERSION}.tar.gz -tar zxf protobuf-cpp-${CPP_PROTOBUF_VERSION}.tar.gz --no-same-owner +wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz +tar zxf ${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz --no-same-owner cd protobuf-${CPP_PROTOBUF_VERSION} ./configure make -j $(nproc) && make install diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index efb60ff982..da36ac9e5d 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -20,7 +20,9 @@ #include "google/protobuf/stubs/stringpiece.h" #include "nlohmann/json.hpp" -#if defined(GOOGLE_PROTOBUF_VERSION) && GOOGLE_PROTOBUF_VERSION >= 3007000 +#if defined(HAVE_ABSEIL) +# include "absl/strings/escaping.h" +#elif defined(GOOGLE_PROTOBUF_VERSION) && GOOGLE_PROTOBUF_VERSION >= 3007000 # include "google/protobuf/stubs/strutil.h" #else # include "google/protobuf/stubs/port.h" @@ -414,14 +416,22 @@ static std::string BytesMapping(const std::string &bytes, else { std::string base64_value; +#if defined(HAVE_ABSEIL) + absl::Base64Escape(bytes, &base64_value); +#else google::protobuf::Base64Escape(bytes, &base64_value); +#endif return base64_value; } } case JsonBytesMappingKind::kBase64: { // Base64 is the default bytes mapping of protobuf std::string base64_value; +#if defined(HAVE_ABSEIL) + absl::Base64Escape(bytes, &base64_value); +#else google::protobuf::Base64Escape(bytes, &base64_value); +#endif return base64_value; } case JsonBytesMappingKind::kHex: From 5c013c0b76f15a1356e2732f6beb337ba7bd43f2 Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 16:52:25 +0800 Subject: [PATCH 02/42] Test modern gRPC Signed-off-by: owent --- .github/workflows/ci.yml | 6 +++--- CHANGELOG.md | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b04179fdbc..c618f1688c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -387,8 +387,8 @@ jobs: sudo ./ci/setup_grpc.sh ./ci/do_ci.sh cmake.exporter.otprotocol.test - cmake_modern_protobuf_test: - name: CMake test (with modern protobuf and otlp-exporter) + cmake_modern_protobuf_grpc_test: + name: CMake test (with modern protobuf and grpc) runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -407,7 +407,7 @@ jobs: env: WITH_ABSEIL: 'ON' run: | - sudo ./ci/setup_grpc.sh + sudo ./ci/setup_grpc.sh -r v1.54.2 ./ci/do_ci.sh cmake.exporter.otprotocol.test cmake_do_not_install_test: diff --git a/CHANGELOG.md b/CHANGELOG.md index a58e5ad3ca..3241a62c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ Increment the: ## [Unreleased] +* [EXPORTER] Support protobuf 3.22 or upper + [#2163](https://github.com/open-telemetry/opentelemetry-cpp/pull/2163) + ## [1.9.1] 2023-05-26 * [DEPRECATION] Drop C++11 support @@ -24,7 +27,7 @@ Increment the: [#2118](https://github.com/open-telemetry/opentelemetry-cpp/pull/2118) * [CI] Upgrade Google Benchmark version from 1.6.0 to 1.7.1 [#2116](https://github.com/open-telemetry/opentelemetry-cpp/pull/2116) -* [CI] Upgrade Nlohmann JSON library version from 3.10.5 to 3.11.2 +* [CI] Upgrade Nlohmann JSON library version from 3.10.5 to 3.11. [#2115](https://github.com/open-telemetry/opentelemetry-cpp/pull/2115) * [BUILD] Missed include From 7b5f9556929951847b1d7a76c6b25fb1ee1d38aa Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 17:02:46 +0800 Subject: [PATCH 03/42] Fix dependency Signed-off-by: owent --- exporters/otlp/BUILD | 1 + exporters/otlp/CMakeLists.txt | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/exporters/otlp/BUILD b/exporters/otlp/BUILD index 01dc83f00b..a5d86df774 100644 --- a/exporters/otlp/BUILD +++ b/exporters/otlp/BUILD @@ -132,6 +132,7 @@ cc_library( "//sdk:headers", "@com_github_opentelemetry_proto//:common_proto_cc", "@github_nlohmann_json//:json", + "@com_google_absl//absl/strings", ], ) diff --git a/exporters/otlp/CMakeLists.txt b/exporters/otlp/CMakeLists.txt index e05b0e24f4..9e4b49fb35 100644 --- a/exporters/otlp/CMakeLists.txt +++ b/exporters/otlp/CMakeLists.txt @@ -116,6 +116,10 @@ if(WITH_OTLP_HTTP) PUBLIC opentelemetry_sdk opentelemetry_ext PRIVATE opentelemetry_proto opentelemetry_http_client_curl nlohmann_json::nlohmann_json) + if(TARGET absl::strings) + target_link_libraries(opentelemetry_exporter_otlp_http_client + PUBLIC absl::strings) + endif() if(nlohmann_json_clone) add_dependencies(opentelemetry_exporter_otlp_http_client nlohmann_json::nlohmann_json) From f95fd57ea7ddf2d1759f56cded7dd29e9f33084b Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 17:27:10 +0800 Subject: [PATCH 04/42] Using installed protobuf to build gRPC when using modern protobuf and gRPC Signed-off-by: owent --- .github/workflows/ci.yml | 2 +- ci/setup_grpc.sh | 41 +++++++++++++++++++++++++++------------- exporters/otlp/BUILD | 2 +- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c618f1688c..f193802258 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -407,7 +407,7 @@ jobs: env: WITH_ABSEIL: 'ON' run: | - sudo ./ci/setup_grpc.sh -r v1.54.2 + sudo ./ci/setup_grpc.sh -r v1.54.2 -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test cmake_do_not_install_test: diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index e1002b6616..2cedba285e 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -12,9 +12,11 @@ std_version='14' install_grpc_version=${new_grpc_version} install_dir='/usr/local/' build_shared_libs='' +build_internal_abseil_cpp=1 +GRPC_BUILD_OPTIONS=() usage() { echo "Usage: $0 [-v ] [-i "] 1>&2; exit 1;} -while getopts ":v:i:r:s:TH" o; do +while getopts ":v:i:p:r:s:TH" o; do case "${o}" in v) gcc_version=${OPTARG} @@ -22,6 +24,14 @@ while getopts ":v:i:r:s:TH" o; do i) install_dir=${OPTARG} ;; + p) + if [[ "${OPTARG}" == "protobuf" ]]; then + GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DgRPC_PROTOBUF_PROVIDER=package") + elif [[ "${OPTARG}" == "abseil-cpp" ]]; then + GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DgRPC_ABSL_PROVIDER=package") + build_internal_abseil_cpp=0 + fi + ;; r) install_grpc_version=${OPTARG} ;; @@ -39,6 +49,7 @@ while getopts ":v:i:r:s:TH" o; do ;; esac done + if [ -z "${gcc_version}" ]; then gcc_version=`gcc --version | awk '/gcc/ {print $NF}'` fi @@ -59,22 +70,26 @@ git clone --depth=1 -b ${install_grpc_version} https://github.com/grpc/grpc pushd grpc git submodule init git submodule update --depth 1 -mkdir -p "third_party/abseil-cpp/build" && pushd "third_party/abseil-cpp/build" -set -x -ABSEIL_CPP_BUILD_OPTIONS=( - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -) -if [ ! -z "$build_shared_libs" ]; then - ABSEIL_CPP_BUILD_OPTIONS=(${ABSEIL_CPP_BUILD_OPTIONS[@]} "-DBUILD_SHARED_LIBS=$build_shared_libs") +if [[ $build_internal_abseil_cpp -ne 0 ]]; then + mkdir -p "third_party/abseil-cpp/build" && pushd "third_party/abseil-cpp/build" + set -x + + ABSEIL_CPP_BUILD_OPTIONS=( + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR + ) + if [ ! -z "$build_shared_libs" ]; then + ABSEIL_CPP_BUILD_OPTIONS=(${ABSEIL_CPP_BUILD_OPTIONS[@]} "-DBUILD_SHARED_LIBS=$build_shared_libs") + fi + cmake ${ABSEIL_CPP_BUILD_OPTIONS[@]} .. + cmake --build . -j${nproc} --target install && popd fi -cmake ${ABSEIL_CPP_BUILD_OPTIONS[@]} .. -cmake --build . -j${nproc} --target install && popd mkdir -p build && pushd build GRPC_BUILD_OPTIONS=( + ${GRPC_BUILD_OPTIONS[@]} -DgRPC_INSTALL=ON -DCMAKE_CXX_STANDARD=${std_version} -DgRPC_BUILD_TESTS=OFF @@ -85,7 +100,7 @@ if [ ! -z "$build_shared_libs" ]; then GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DBUILD_SHARED_LIBS=$build_shared_libs") fi -cmake ${GRPC_BUILD_OPTIONS[@]} .. +cmake ${GRPC_BUILD_OPTIONS[@]} "$@" .. cmake --build . -j$(nproc) cmake --install . popd diff --git a/exporters/otlp/BUILD b/exporters/otlp/BUILD index a5d86df774..7da4a18c49 100644 --- a/exporters/otlp/BUILD +++ b/exporters/otlp/BUILD @@ -131,8 +131,8 @@ cc_library( "//ext/src/http/client/curl:http_client_curl", "//sdk:headers", "@com_github_opentelemetry_proto//:common_proto_cc", - "@github_nlohmann_json//:json", "@com_google_absl//absl/strings", + "@github_nlohmann_json//:json", ], ) From f016a7559722b4da05650a3fdc259176d1f0a783 Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 19:58:49 +0800 Subject: [PATCH 05/42] Make install_abseil.sh has the same options like it in setup_grpc.sh Signed-off-by: owent --- ci/install_abseil.sh | 3 ++- ci/setup_grpc.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/install_abseil.sh b/ci/install_abseil.sh index 1184a1ddbc..b9858ebf84 100755 --- a/ci/install_abseil.sh +++ b/ci/install_abseil.sh @@ -13,7 +13,8 @@ pushd $BUILD_DIR git clone --depth=1 -b ${ABSEIL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git cd abseil-cpp mkdir build && pushd build -cmake -DBUILD_TESTING=OFF -DCMAKE_CXX_STANDARD=11 \ +cmake -DBUILD_TESTING=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ .. make -j $(nproc) diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index 2cedba285e..77937b3479 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -25,9 +25,9 @@ while getopts ":v:i:p:r:s:TH" o; do install_dir=${OPTARG} ;; p) - if [[ "${OPTARG}" == "protobuf" ]]; then + if [ "${OPTARG}" == "protobuf" ]; then GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DgRPC_PROTOBUF_PROVIDER=package") - elif [[ "${OPTARG}" == "abseil-cpp" ]]; then + elif [ "${OPTARG}" == "abseil-cpp" ]; then GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DgRPC_ABSL_PROVIDER=package") build_internal_abseil_cpp=0 fi @@ -100,7 +100,7 @@ if [ ! -z "$build_shared_libs" ]; then GRPC_BUILD_OPTIONS=(${GRPC_BUILD_OPTIONS[@]} "-DBUILD_SHARED_LIBS=$build_shared_libs") fi -cmake ${GRPC_BUILD_OPTIONS[@]} "$@" .. +cmake ${GRPC_BUILD_OPTIONS[@]} .. cmake --build . -j$(nproc) cmake --install . popd From c94e488a388ad65f06a903573473cdbf4da14381 Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 20:18:24 +0800 Subject: [PATCH 06/42] Fix abseil-cpp dependency of proto when using protobuf 3.22 or upper. Signed-off-by: owent --- cmake/opentelemetry-proto.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 34b33d3cff..2f6cf23656 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -251,6 +251,10 @@ add_library( ${LOGS_SERVICE_PB_CPP_FILE} ${METRICS_SERVICE_PB_CPP_FILE}) +if(WITH_ABSEIL AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") + target_link_libraries(opentelemetry_proto PUBLIC absl::bad_variant_access) +endif() + if(WITH_OTLP_GRPC) add_library( opentelemetry_proto_grpc From 0dc757553edbe9423b5cbdb09fe71d3592d9b657 Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 21:35:12 +0800 Subject: [PATCH 07/42] Fix protobuf installer with package absl. Signed-off-by: owent --- ci/install_protobuf.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index bfc8b1f092..768b4b9462 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -31,9 +31,21 @@ set -e # when calling this script # +CPP_PROTOBUF_BUILD_OPTIONS=( + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + "-Dprotobuf_BUILD_TESTS=OFF" + "-Dprotobuf_BUILD_EXAMPLES=OFF" + "-Dprotobuf_MODULE_COMPATIBLE=ON" +) + +if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then + CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-Dprotobuf_MODULE_COMPATIBLE=ON") +fi + if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then export CPP_PROTOBUF_VERSION="${PROTOBUF_VERSION}" CPP_PROTOBUF_PACKAGE_NAME="protobuf-${CPP_PROTOBUF_VERSION}" + CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-Dprotobuf_ABSL_PROVIDER=package") else export CPP_PROTOBUF_VERSION="3.${PROTOBUF_VERSION}" CPP_PROTOBUF_PACKAGE_NAME="protobuf-cpp-${CPP_PROTOBUF_VERSION}" @@ -42,7 +54,14 @@ fi cd / wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz tar zxf ${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz --no-same-owner -cd protobuf-${CPP_PROTOBUF_VERSION} -./configure -make -j $(nproc) && make install + +mkdir protobuf-${CPP_PROTOBUF_VERSION}/build && pushd protobuf-${CPP_PROTOBUF_VERSION}/build +if [ -e "../CMakeLists.txt" ]; then + cmake .. ${CPP_PROTOBUF_BUILD_OPTIONS[@]} +else + cmake ../cmake ${CPP_PROTOBUF_BUILD_OPTIONS[@]} +fi +cmake --build . -j $(nproc) +cmake --install . +popd ldconfig From 8db4a61e07e0888eab454408b45b760715b2c8fd Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 22:08:20 +0800 Subject: [PATCH 08/42] Fix CI job for custom protobuf and abseil version Signed-off-by: owent --- .github/workflows/ci.yml | 6 +++--- ci/install_abseil.sh | 2 +- ci/install_protobuf.sh | 2 +- cmake/opentelemetry-proto.cmake | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f193802258..d17ce9c234 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -401,13 +401,13 @@ jobs: run: | sudo ./ci/setup_cmake.sh sudo ./ci/setup_ci_environment.sh - sudo ./ci/install_abseil.sh - sudo ./ci/install_protobuf.sh + sudo -E ./ci/install_abseil.sh + sudo -E ./ci/install_protobuf.sh - name: run otlp exporter tests env: WITH_ABSEIL: 'ON' run: | - sudo ./ci/setup_grpc.sh -r v1.54.2 -p protobuf -p abseil-cpp + sudo -E ./ci/setup_grpc.sh -r v1.54.2 -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test cmake_do_not_install_test: diff --git a/ci/install_abseil.sh b/ci/install_abseil.sh index b9858ebf84..864c6fd6ce 100755 --- a/ci/install_abseil.sh +++ b/ci/install_abseil.sh @@ -3,7 +3,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -set -e +set -ex export DEBIAN_FRONTEND=noninteractive [ -z "${ABSEIL_CPP_VERSION}" ] && export ABSEIL_CPP_VERSION="20220623.1" diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index 768b4b9462..47701a4013 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -51,7 +51,7 @@ else CPP_PROTOBUF_PACKAGE_NAME="protobuf-cpp-${CPP_PROTOBUF_VERSION}" fi -cd / +cd /tmp wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz tar zxf ${CPP_PROTOBUF_PACKAGE_NAME}.tar.gz --no-same-owner diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 2f6cf23656..8333a1ad77 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -251,7 +251,7 @@ add_library( ${LOGS_SERVICE_PB_CPP_FILE} ${METRICS_SERVICE_PB_CPP_FILE}) -if(WITH_ABSEIL AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") +if(WITH_ABSEIL) target_link_libraries(opentelemetry_proto PUBLIC absl::bad_variant_access) endif() From 0f0085a1a0ad04953a25bce3a7d5e2032a9cf9e7 Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 22:33:54 +0800 Subject: [PATCH 09/42] Allow to set `CXX_STANDARD` by environment variable in CI job. Signed-off-by: owent --- .github/workflows/ci.yml | 2 ++ ci/do_ci.sh | 47 ++++++++++++++++++++++------------------ ci/install_abseil.sh | 15 +++++++++---- ci/install_protobuf.sh | 4 ++++ ci/setup_grpc.sh | 3 +++ 5 files changed, 46 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d17ce9c234..15436b34e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -398,6 +398,7 @@ jobs: env: PROTOBUF_VERSION: '23.2' ABSEIL_CPP_VERSION: '20230125.3' + CXX_STANDARD: '17' run: | sudo ./ci/setup_cmake.sh sudo ./ci/setup_ci_environment.sh @@ -406,6 +407,7 @@ jobs: - name: run otlp exporter tests env: WITH_ABSEIL: 'ON' + CXX_STANDARD: '17' run: | sudo -E ./ci/setup_grpc.sh -r v1.54.2 -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 26fc5804cb..6f82384cf3 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -73,12 +73,17 @@ BAZEL_MACOS_TEST_OPTIONS="$BAZEL_MACOS_OPTIONS --test_output=errors" BAZEL_STARTUP_OPTIONS="--output_user_root=$HOME/.cache/bazel" +CMAKE_OPTIONS=(-DCMAKE_BUILD_TYPE=Debug) +if [ ! -z "${CXX_STANDARD}" ]; then + CMAKE_OPTIONS=(${CMAKE_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") +fi + export CTEST_OUTPUT_ON_FAILURE=1 if [[ "$1" == "cmake.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_PROMETHEUS=ON \ -DWITH_ZIPKIN=ON \ -DWITH_JAEGER=ON \ @@ -93,7 +98,7 @@ if [[ "$1" == "cmake.test" ]]; then elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_HTTP_SSL_PREVIEW=ON \ @@ -117,7 +122,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then elif [[ "$1" == "cmake.maintainer.async.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_HTTP_SSL_PREVIEW=ON \ @@ -141,7 +146,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_STANDARD=11 \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ @@ -166,7 +171,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then elif [[ "$1" == "cmake.with_async_export.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_PROMETHEUS=ON \ -DWITH_ZIPKIN=ON \ -DWITH_JAEGER=ON \ @@ -182,7 +187,7 @@ elif [[ "$1" == "cmake.with_async_export.test" ]]; then elif [[ "$1" == "cmake.abseil.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_METRICS_EXEMPLAR_PREVIEW=ON \ -DWITH_LOGS_PREVIEW=ON \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ @@ -195,7 +200,7 @@ elif [[ "$1" == "cmake.abseil.test" ]]; then elif [[ "$1" == "cmake.opentracing_shim.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror -Wno-error=redundant-move $CXXFLAGS" \ -DWITH_OPENTRACING=ON \ "${SRC_DIR}" @@ -205,7 +210,7 @@ elif [[ "$1" == "cmake.opentracing_shim.test" ]]; then elif [[ "$1" == "cmake.c++20.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ -DCMAKE_CXX_STANDARD=20 \ @@ -216,7 +221,7 @@ elif [[ "$1" == "cmake.c++20.test" ]]; then elif [[ "$1" == "cmake.c++20.stl.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_METRICS_EXEMPLAR_PREVIEW=ON \ -DWITH_LOGS_PREVIEW=ON \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ @@ -231,7 +236,7 @@ elif [[ "$1" == "cmake.legacy.test" ]]; then rm -rf * export BUILD_ROOT="${BUILD_DIR}" ${SRC_DIR}/tools/build-benchmark.sh - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DCMAKE_CXX_STANDARD=11 \ "${SRC_DIR}" @@ -243,7 +248,7 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then rm -rf * export BUILD_ROOT="${BUILD_DIR}" ${SRC_DIR}/tools/build-benchmark.sh - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_STANDARD=11 \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ @@ -258,13 +263,13 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then cd "${BUILD_DIR}" rm -rf * - if [[ -z "$WITH_ABSEIL" ]]; then - WITH_ABSEIL="OFF" + CMAKE_OPTIONS=() + if [[ ! -z "${WITH_ABSEIL}" ]]; then + CMAKE_OPTIONS=(${CMAKE_OPTIONS[@]} "-DWITH_ABSEIL=${WITH_ABSEIL}") fi - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ - -DWITH_ABSEIL=$WITH_ABSEIL \ "${SRC_DIR}" grpc_cpp_plugin=`which grpc_cpp_plugin` proto_make_file="CMakeFiles/opentelemetry_proto.dir/build.make" @@ -275,7 +280,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then elif [[ "$1" == "cmake.exporter.otprotocol.shared_libs.with_static_grpc.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ -DBUILD_SHARED_LIBS=ON \ @@ -289,7 +294,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.shared_libs.with_static_grpc.test" ]] elif [[ "$1" == "cmake.exporter.otprotocol.with_async_export.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ @@ -303,7 +308,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.with_async_export.test" ]]; then elif [[ "$1" == "cmake.do_not_install.test" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DWITH_OTLP=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ @@ -336,7 +341,7 @@ EOF -static-libgcc \ -Wl,--version-script=${PWD}/export.map \ " - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ -DCMAKE_EXE_LINKER_FLAGS="$LINKER_FLAGS" \ -DCMAKE_SHARED_LINKER_FLAGS="$LINKER_FLAGS" \ @@ -347,7 +352,7 @@ EOF # Verify we can load the plugin cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS" \ "${SRC_DIR}" make load_plugin_example @@ -429,7 +434,7 @@ elif [[ "$1" == "format" ]]; then elif [[ "$1" == "code.coverage" ]]; then cd "${BUILD_DIR}" rm -rf * - cmake -DCMAKE_BUILD_TYPE=Debug \ + cmake ${CMAKE_OPTIONS[@]} \ -DCMAKE_CXX_FLAGS="-Werror --coverage $CXXFLAGS" \ "${SRC_DIR}" make diff --git a/ci/install_abseil.sh b/ci/install_abseil.sh index 864c6fd6ce..3bc69df89d 100755 --- a/ci/install_abseil.sh +++ b/ci/install_abseil.sh @@ -12,11 +12,18 @@ INSTALL_DIR=/usr/local/ pushd $BUILD_DIR git clone --depth=1 -b ${ABSEIL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git cd abseil-cpp +ABSEIL_CPP_BUILD_OPTIONS=( + "-DBUILD_TESTING=OFF" + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + "-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR" +) + +if [ ! -z "${CXX_STANDARD}" ]; then + ABSEIL_CPP_BUILD_OPTIONS=(${ABSEIL_CPP_BUILD_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") +fi + mkdir build && pushd build -cmake -DBUILD_TESTING=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ - .. +cmake ${ABSEIL_CPP_BUILD_OPTIONS[@]} .. make -j $(nproc) make install popd diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index 47701a4013..cb5b830bb8 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -38,6 +38,10 @@ CPP_PROTOBUF_BUILD_OPTIONS=( "-Dprotobuf_MODULE_COMPATIBLE=ON" ) +if [ ! -z "${CXX_STANDARD}" ]; then + CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") +fi + if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-Dprotobuf_MODULE_COMPATIBLE=ON") fi diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index 77937b3479..0740fa54a4 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -9,6 +9,9 @@ old_grpc_version='v1.33.2' new_grpc_version='v1.49.2' gcc_version_for_new_grpc='5.1' std_version='14' +if [ ! -z "${CXX_STANDARD}" ]; then + std_version="${CXX_STANDARD}" +fi install_grpc_version=${new_grpc_version} install_dir='/usr/local/' build_shared_libs='' From 1dcceaaf4295b732aa7d7324f5fbb9fc7c61f8f1 Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 23:02:28 +0800 Subject: [PATCH 10/42] Update modern gRPC version Signed-off-by: owent --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15436b34e1..ddae285162 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,7 +409,7 @@ jobs: WITH_ABSEIL: 'ON' CXX_STANDARD: '17' run: | - sudo -E ./ci/setup_grpc.sh -r v1.54.2 -p protobuf -p abseil-cpp + sudo -E ./ci/setup_grpc.sh -r v1.55.0 -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test cmake_do_not_install_test: From 6fa4e952fd7465036a850109dce08a0b199efa6e Mon Sep 17 00:00:00 2001 From: owent Date: Sat, 27 May 2023 23:23:12 +0800 Subject: [PATCH 11/42] Fix gcc version detection Signed-off-by: owent --- ci/setup_grpc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index 0740fa54a4..cd89ac3e5b 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -56,8 +56,8 @@ done if [ -z "${gcc_version}" ]; then gcc_version=`gcc --version | awk '/gcc/ {print $NF}'` fi -if [[ "${gcc_version}" < "${gcc_version_for_new_grpc}" ]]; then - echo "less" +if [[ "${gcc_version}" < "${gcc_version_for_new_grpc}" ]] && [[ "${gcc_version:1:1}" == "." ]]; then + echo "${gcc_version} less than ${gcc_version_for_new_grpc}" std_version='11' install_grpc_version=${old_grpc_version} fi From 93d347cc4cbd698afab36ebc8d2be73af7f32137 Mon Sep 17 00:00:00 2001 From: owent Date: Sun, 28 May 2023 00:11:02 +0800 Subject: [PATCH 12/42] Fix abseil-cpp checking. Signed-off-by: owent --- CMakeLists.txt | 4 ++-- ci/do_ci.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b371831ace..528c04d9ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -363,10 +363,10 @@ if(WITH_OTLP) set(protobuf_MODULE_COMPATIBLE ON) find_package(Protobuf) if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") - if(WITH_ABSEIL) + if(NOT WITH_ABSEIL) message( FATAL_ERROR - "Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125.0 or upper)" + "Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125.3 or upper)" ) endif() endif() diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 6f82384cf3..6a3394b30f 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -263,7 +263,6 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then cd "${BUILD_DIR}" rm -rf * - CMAKE_OPTIONS=() if [[ ! -z "${WITH_ABSEIL}" ]]; then CMAKE_OPTIONS=(${CMAKE_OPTIONS[@]} "-DWITH_ABSEIL=${WITH_ABSEIL}") fi From 234ab401322a360efa615d3d458a6c4c6febb5b1 Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 29 May 2023 14:52:52 +0800 Subject: [PATCH 13/42] Fix stringpiece including. Signed-off-by: owent --- exporters/otlp/src/otlp_http_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index da36ac9e5d..2077afeb28 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -17,7 +17,6 @@ #include "google/protobuf/message.h" #include "google/protobuf/reflection.h" #include "google/protobuf/stubs/common.h" -#include "google/protobuf/stubs/stringpiece.h" #include "nlohmann/json.hpp" #if defined(HAVE_ABSEIL) @@ -26,6 +25,7 @@ # include "google/protobuf/stubs/strutil.h" #else # include "google/protobuf/stubs/port.h" +# include "google/protobuf/stubs/stringpiece.h" namespace google { namespace protobuf From 89561934b7fb953dc381ed47d6fb31e58f9b4934 Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 29 May 2023 18:46:18 +0800 Subject: [PATCH 14/42] Fix link order of grpc example Signed-off-by: owent --- examples/grpc/CMakeLists.txt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/grpc/CMakeLists.txt b/examples/grpc/CMakeLists.txt index 1ea1c9bee5..6d08b8255e 100644 --- a/examples/grpc/CMakeLists.txt +++ b/examples/grpc/CMakeLists.txt @@ -33,16 +33,20 @@ include_directories( include_directories(${CMAKE_CURRENT_BINARY_DIR}) if(TARGET protobuf::libprotobuf) - target_link_libraries(example_grpc_proto gRPC::grpc++ protobuf::libprotobuf) + target_link_libraries(example_grpc_proto PUBLIC gRPC::grpc++ + protobuf::libprotobuf) else() - target_include_directories(example_grpc_proto ${Protobuf_INCLUDE_DIRS}) - target_link_libraries(example_grpc_proto ${Protobuf_LIBRARIES}) + target_include_directories(example_grpc_proto PUBLIC ${Protobuf_INCLUDE_DIRS}) + target_link_libraries(example_grpc_proto PUBLIC gRPC::grpc++ + ${Protobuf_LIBRARIES}) +endif() +if(WITH_ABSEIL) + target_link_libraries(example_grpc_proto PUBLIC absl::bad_variant_access) endif() foreach(_target client server) add_executable(${_target} "${_target}.cc") - target_link_libraries( - ${_target} example_grpc_proto protobuf::libprotobuf gRPC::grpc++ - opentelemetry_trace opentelemetry_exporter_ostream_span) + target_link_libraries(${_target} example_grpc_proto opentelemetry_trace + opentelemetry_exporter_ostream_span) patch_protobuf_targets(${_target}) endforeach() From 5f3e6dbe18586ed4f4a03e1043eeb7fe434851aa Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 29 May 2023 23:04:47 +0800 Subject: [PATCH 15/42] Patch dependencies. Signed-off-by: owent --- .github/workflows/ci.yml | 2 +- CMakeLists.txt | 3 ++ ci/setup_grpc.sh | 6 ++- cmake/opentelemetry-proto.cmake | 70 +++++++++++++++++++++++++++++++++ cmake/proto-options-patch.cmake | 68 ++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddae285162..972fc1a65e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,7 +409,7 @@ jobs: WITH_ABSEIL: 'ON' CXX_STANDARD: '17' run: | - sudo -E ./ci/setup_grpc.sh -r v1.55.0 -p protobuf -p abseil-cpp + sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test cmake_do_not_install_test: diff --git a/CMakeLists.txt b/CMakeLists.txt index 528c04d9ff..326fc43ff3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,9 @@ project(opentelemetry-cpp) # Mark variables as used so cmake doesn't complain about them mark_as_advanced(CMAKE_TOOLCHAIN_FILE) +# Prefer cmake CONFIG to auto resolve dependencies. +set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) + # Don't use customized cmake modules if vcpkg is used to resolve dependence. if(NOT DEFINED CMAKE_TOOLCHAIN_FILE) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/") diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index cd89ac3e5b..14682fd2b7 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -7,6 +7,7 @@ set -e export DEBIAN_FRONTEND=noninteractive old_grpc_version='v1.33.2' new_grpc_version='v1.49.2' +moderd_grpc_version='v1.55.0' gcc_version_for_new_grpc='5.1' std_version='14' if [ ! -z "${CXX_STANDARD}" ]; then @@ -19,7 +20,7 @@ build_internal_abseil_cpp=1 GRPC_BUILD_OPTIONS=() usage() { echo "Usage: $0 [-v ] [-i "] 1>&2; exit 1;} -while getopts ":v:i:p:r:s:TH" o; do +while getopts ":v:i:mp:r:s:TH" o; do case "${o}" in v) gcc_version=${OPTARG} @@ -38,6 +39,9 @@ while getopts ":v:i:p:r:s:TH" o; do r) install_grpc_version=${OPTARG} ;; + m) + install_grpc_version=${moderd_grpc_version} + ;; s) std_version=${OPTARG} ;; diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 8333a1ad77..41d38473dc 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -320,3 +320,73 @@ if(BUILD_SHARED_LIBS) set_property(TARGET ${proto_target} PROPERTY POSITION_INDEPENDENT_CODE ON) endforeach() endif() + +if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") + project_build_tools_patch_default_imported_config( + absl::bad_variant_access + absl::raw_logging_internal + absl::log_severity + protobuf::libprotobuf + ZLIB::ZLIB + absl::log_internal_check_op + absl::log_internal_nullguard + absl::strings + absl::strings_internal + absl::base + absl::spinlock_wait + absl::int128 + absl::throw_delegate + absl::log_internal_message + absl::examine_stack + absl::stacktrace + absl::debugging_internal + absl::symbolize + absl::demangle_internal + absl::malloc_internal + absl::log_internal_format + absl::log_internal_globals + absl::time + absl::civil_time + absl::time_zone + absl::str_format_internal + absl::log_internal_proto + absl::log_internal_log_sink_set + absl::log_globals + absl::hash + absl::city + absl::bad_optional_access + absl::low_level_hash + absl::log_entry + absl::log_sink + absl::synchronization + absl::graphcycles_internal + absl::strerror + absl::log_internal_conditions + absl::cord + absl::cord_internal + absl::crc_cord_state + absl::crc32c + absl::crc_cpu_detect + absl::crc_internal + absl::cordz_functions + absl::exponential_biased + absl::cordz_info + absl::cordz_handle + absl::leak_check + absl::die_if_null + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_program_name + absl::flags_internal + absl::flags_marshalling + absl::flags_reflection + absl::flags_private_handle_accessor + absl::raw_hash_set + absl::hashtablez_sampler + absl::log_initialize + absl::status + absl::statusor + utf8_range::utf8_validity) +endif() diff --git a/cmake/proto-options-patch.cmake b/cmake/proto-options-patch.cmake index 97872050c9..dee9a94dd3 100644 --- a/cmake/proto-options-patch.cmake +++ b/cmake/proto-options-patch.cmake @@ -117,3 +117,71 @@ function(project_build_tools_get_imported_location OUTPUT_VAR_NAME TARGET_NAME) PARENT_SCOPE) endif() endfunction() + +function(project_build_tools_patch_default_imported_config) + set(PATCH_VARS + IMPORTED_IMPLIB + IMPORTED_LIBNAME + IMPORTED_LINK_DEPENDENT_LIBRARIES + IMPORTED_LINK_INTERFACE_LANGUAGES + IMPORTED_LINK_INTERFACE_LIBRARIES + IMPORTED_LINK_INTERFACE_MULTIPLICITY + IMPORTED_LOCATION + IMPORTED_NO_SONAME + IMPORTED_OBJECTS + IMPORTED_SONAME) + foreach(TARGET_NAME ${ARGN}) + if(TARGET ${TARGET_NAME}) + get_target_property(IS_IMPORTED_TARGET ${TARGET_NAME} IMPORTED) + if(NOT IS_IMPORTED_TARGET) + continue() + endif() + + if(CMAKE_VERSION VERSION_LESS "3.19.0") + get_target_property(TARGET_TYPE_NAME ${TARGET_NAME} TYPE) + if(TARGET_TYPE_NAME STREQUAL "INTERFACE_LIBRARY") + continue() + endif() + endif() + + get_target_property(DO_NOT_OVERWRITE ${TARGET_NAME} IMPORTED_LOCATION) + if(DO_NOT_OVERWRITE) + continue() + endif() + + # MSVC's STL and debug level must match the target, so we can only move + # out IMPORTED_LOCATION_NOCONFIG + if(MSVC) + set(PATCH_IMPORTED_CONFIGURATION "NOCONFIG") + else() + get_target_property(PATCH_IMPORTED_CONFIGURATION ${TARGET_NAME} + IMPORTED_CONFIGURATIONS) + endif() + + if(NOT PATCH_IMPORTED_CONFIGURATION) + continue() + endif() + + get_target_property(PATCH_TARGET_LOCATION ${TARGET_NAME} + "IMPORTED_LOCATION_${PATCH_IMPORTED_CONFIGURATION}") + if(NOT PATCH_TARGET_LOCATION) + continue() + endif() + + foreach(PATCH_IMPORTED_KEY IN LISTS PATCH_VARS) + get_target_property( + PATCH_IMPORTED_VALUE ${TARGET_NAME} + "${PATCH_IMPORTED_KEY}_${PATCH_IMPORTED_CONFIGURATION}") + if(PATCH_IMPORTED_VALUE) + set_target_properties( + ${TARGET_NAME} PROPERTIES "${PATCH_IMPORTED_KEY}" + "${PATCH_IMPORTED_VALUE}") + message( + STATUS + "Patch: ${TARGET_NAME} ${PATCH_IMPORTED_KEY} will use ${PATCH_IMPORTED_KEY}_${PATCH_IMPORTED_CONFIGURATION}(\"${PATCH_IMPORTED_VALUE}\") by default." + ) + endif() + endforeach() + endif() + endforeach() +endfunction() From 8a363788ba172b9547be6019b5dbcea0e2d4bb39 Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 29 May 2023 23:45:25 +0800 Subject: [PATCH 16/42] Allow to ignore more libraries's CONFIGs. Signed-off-by: owent --- CMakeLists.txt | 11 ++++++++++ cmake/opentelemetry-proto.cmake | 22 +++++++++++++++++-- ...{proto-options-patch.cmake => tools.cmake} | 10 +++++---- 3 files changed, 37 insertions(+), 6 deletions(-) rename cmake/{proto-options-patch.cmake => tools.cmake} (95%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 326fc43ff3..c540665347 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,8 @@ endif() option(OPENTELEMETRY_INSTALL "Whether to install opentelemetry targets" ${OPENTELEMETRY_INSTALL_default}) +include("${PROJECT_SOURCE_DIR}/cmake/tools.cmake") + if(NOT DEFINED CMAKE_CXX_STANDARD) if(WITH_STL) # Require at least C++17. C++20 is needed to avoid gsl::span @@ -587,6 +589,15 @@ if(BUILD_TESTING) # Benchmark respects the CMAKE_PREFIX_PATH find_package(benchmark CONFIG REQUIRED) endif() + project_build_tools_patch_default_imported_config( + GTest::gtest + GTest::gtest_main + GTest::gmock + GTest::gmock_main + GTest::GTest + GTest::Main + benchmark::benchmark + benchmark::benchmark_main) endif() include(CMakePackageConfigHelpers) diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 41d38473dc..47f3ee806b 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -65,8 +65,6 @@ else() endif() endif() -include(${PROJECT_SOURCE_DIR}/cmake/proto-options-patch.cmake) - set(COMMON_PROTO "${PROTO_PATH}/opentelemetry/proto/common/v1/common.proto") set(RESOURCE_PROTO "${PROTO_PATH}/opentelemetry/proto/resource/v1/resource.proto") @@ -167,6 +165,26 @@ if(WITH_OTLP_GRPC) endif() endif() message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}") + + project_build_tools_patch_default_imported_config( + gRPC::cares + gRPC::re2 + gRPC::ssl + gRPC::crypto + gRPC::zlibstatic + gRPC::address_sorting + gRPC::gpr + gRPC::grpc + gRPC::grpc_unsecure + gRPC::grpc++ + gRPC::grpc++_alts + gRPC::grpc++_error_details + gRPC::grpc++_reflection + gRPC::grpc++_unsecure + gRPC::grpc_authorization_provider + gRPC::grpc_plugin_support + gRPC::grpcpp_channelz + gRPC::upb) endif() if(WITH_OTLP_GRPC) diff --git a/cmake/proto-options-patch.cmake b/cmake/tools.cmake similarity index 95% rename from cmake/proto-options-patch.cmake rename to cmake/tools.cmake index dee9a94dd3..6b91ca07b6 100644 --- a/cmake/proto-options-patch.cmake +++ b/cmake/tools.cmake @@ -176,10 +176,12 @@ function(project_build_tools_patch_default_imported_config) set_target_properties( ${TARGET_NAME} PROPERTIES "${PATCH_IMPORTED_KEY}" "${PATCH_IMPORTED_VALUE}") - message( - STATUS - "Patch: ${TARGET_NAME} ${PATCH_IMPORTED_KEY} will use ${PATCH_IMPORTED_KEY}_${PATCH_IMPORTED_CONFIGURATION}(\"${PATCH_IMPORTED_VALUE}\") by default." - ) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + message( + STATUS + "Patch: ${TARGET_NAME} ${PATCH_IMPORTED_KEY} will use ${PATCH_IMPORTED_KEY}_${PATCH_IMPORTED_CONFIGURATION}(\"${PATCH_IMPORTED_VALUE}\") by default." + ) + endif() endif() endforeach() endif() From 2ecaf755a92f5ec33acb99fa8986e2cf101cb253 Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 31 May 2023 15:26:15 +0800 Subject: [PATCH 17/42] Patch several third party libraries to allow they are built in a different cmake CONFIG Signed-off-by: owent --- CMakeLists.txt | 11 +-- cmake/opentelemetry-proto.cmake | 90 -------------------- cmake/patch-imported-config.cmake | 134 ++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 99 deletions(-) create mode 100644 cmake/patch-imported-config.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c540665347..04682eccde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -589,15 +589,6 @@ if(BUILD_TESTING) # Benchmark respects the CMAKE_PREFIX_PATH find_package(benchmark CONFIG REQUIRED) endif() - project_build_tools_patch_default_imported_config( - GTest::gtest - GTest::gtest_main - GTest::gmock - GTest::gmock_main - GTest::GTest - GTest::Main - benchmark::benchmark - benchmark::benchmark_main) endif() include(CMakePackageConfigHelpers) @@ -661,6 +652,8 @@ if(NOT WITH_API_ONLY) endif() endif() +include("${PROJECT_SOURCE_DIR}/cmake/patch-imported-config.cmake") + if(OPENTELEMETRY_INSTALL) # Export cmake config and support find_packages(opentelemetry-cpp CONFIG) # Write config file for find_packages(opentelemetry-cpp CONFIG) diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 47f3ee806b..26849f858d 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -165,26 +165,6 @@ if(WITH_OTLP_GRPC) endif() endif() message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}") - - project_build_tools_patch_default_imported_config( - gRPC::cares - gRPC::re2 - gRPC::ssl - gRPC::crypto - gRPC::zlibstatic - gRPC::address_sorting - gRPC::gpr - gRPC::grpc - gRPC::grpc_unsecure - gRPC::grpc++ - gRPC::grpc++_alts - gRPC::grpc++_error_details - gRPC::grpc++_reflection - gRPC::grpc++_unsecure - gRPC::grpc_authorization_provider - gRPC::grpc_plugin_support - gRPC::grpcpp_channelz - gRPC::upb) endif() if(WITH_OTLP_GRPC) @@ -338,73 +318,3 @@ if(BUILD_SHARED_LIBS) set_property(TARGET ${proto_target} PROPERTY POSITION_INDEPENDENT_CODE ON) endforeach() endif() - -if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") - project_build_tools_patch_default_imported_config( - absl::bad_variant_access - absl::raw_logging_internal - absl::log_severity - protobuf::libprotobuf - ZLIB::ZLIB - absl::log_internal_check_op - absl::log_internal_nullguard - absl::strings - absl::strings_internal - absl::base - absl::spinlock_wait - absl::int128 - absl::throw_delegate - absl::log_internal_message - absl::examine_stack - absl::stacktrace - absl::debugging_internal - absl::symbolize - absl::demangle_internal - absl::malloc_internal - absl::log_internal_format - absl::log_internal_globals - absl::time - absl::civil_time - absl::time_zone - absl::str_format_internal - absl::log_internal_proto - absl::log_internal_log_sink_set - absl::log_globals - absl::hash - absl::city - absl::bad_optional_access - absl::low_level_hash - absl::log_entry - absl::log_sink - absl::synchronization - absl::graphcycles_internal - absl::strerror - absl::log_internal_conditions - absl::cord - absl::cord_internal - absl::crc_cord_state - absl::crc32c - absl::crc_cpu_detect - absl::crc_internal - absl::cordz_functions - absl::exponential_biased - absl::cordz_info - absl::cordz_handle - absl::leak_check - absl::die_if_null - absl::flags - absl::flags_commandlineflag - absl::flags_commandlineflag_internal - absl::flags_config - absl::flags_program_name - absl::flags_internal - absl::flags_marshalling - absl::flags_reflection - absl::flags_private_handle_accessor - absl::raw_hash_set - absl::hashtablez_sampler - absl::log_initialize - absl::status - absl::statusor - utf8_range::utf8_validity) -endif() diff --git a/cmake/patch-imported-config.cmake b/cmake/patch-imported-config.cmake new file mode 100644 index 0000000000..131e99f89b --- /dev/null +++ b/cmake/patch-imported-config.cmake @@ -0,0 +1,134 @@ +# Common dependencies +project_build_tools_patch_default_imported_config(ZLIB::ZLIB) + +# protobuf targets +if(Protobuf_FOUND OR PROTOBUF_FOUND) + project_build_tools_patch_default_imported_config( + utf8_range::utf8_range utf8_range::utf8_validity protobuf::libprotobuf-lite + protobuf::libprotobuf protobuf::libprotoc) +endif() + +# cares targets +if(TARGET c-ares::cares) + project_build_tools_patch_default_imported_config(c-ares::cares) +endif() + +# curl targets +if(TARGET CURL::libcurl) + project_build_tools_patch_default_imported_config(CURL::libcurl) +endif() + +# abseil targets +if(WITH_ABSEIL) + project_build_tools_patch_default_imported_config( + absl::bad_variant_access + absl::raw_logging_internal + absl::log_severity + absl::log_internal_check_op + absl::log_internal_nullguard + absl::strings + absl::strings_internal + absl::base + absl::spinlock_wait + absl::int128 + absl::throw_delegate + absl::log_internal_message + absl::examine_stack + absl::stacktrace + absl::debugging_internal + absl::symbolize + absl::demangle_internal + absl::malloc_internal + absl::log_internal_format + absl::log_internal_globals + absl::time + absl::civil_time + absl::time_zone + absl::str_format_internal + absl::log_internal_proto + absl::log_internal_log_sink_set + absl::log_globals + absl::hash + absl::city + absl::bad_optional_access + absl::low_level_hash + absl::log_entry + absl::log_sink + absl::synchronization + absl::graphcycles_internal + absl::strerror + absl::log_internal_conditions + absl::cord + absl::cord_internal + absl::crc_cord_state + absl::crc32c + absl::crc_cpu_detect + absl::crc_internal + absl::cordz_functions + absl::exponential_biased + absl::cordz_info + absl::cordz_handle + absl::leak_check + absl::die_if_null + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_program_name + absl::flags_internal + absl::flags_marshalling + absl::flags_reflection + absl::flags_private_handle_accessor + absl::raw_hash_set + absl::hashtablez_sampler + absl::log_initialize + absl::status + absl::statusor) +endif() + +# gRPC targets +if(TARGET gRPC::grpc++) + project_build_tools_patch_default_imported_config( + gRPC::cares + gRPC::re2 + gRPC::ssl + gRPC::crypto + gRPC::zlibstatic + gRPC::address_sorting + gRPC::gpr + gRPC::grpc + gRPC::grpc_unsecure + gRPC::grpc++ + gRPC::grpc++_alts + gRPC::grpc++_error_details + gRPC::grpc++_reflection + gRPC::grpc++_unsecure + gRPC::grpc_authorization_provider + gRPC::grpc_plugin_support + gRPC::grpcpp_channelz + gRPC::upb) +endif() + +# prometheus targets +if(TARGET prometheus-cpp::core) + project_build_tools_patch_default_imported_config( + prometheus-cpp::core prometheus-cpp::pull prometheus-cpp::push) +endif() + +# civetweb targets +if(TARGET civetweb::civetweb) + project_build_tools_patch_default_imported_config( + civetweb::civetweb civetweb::server civetweb::civetweb-cpp) +endif() + +if(BUILD_TESTING) + project_build_tools_patch_default_imported_config( + GTest::gtest + GTest::gtest_main + GTest::gmock + GTest::gmock_main + GTest::GTest + GTest::Main + benchmark::benchmark + benchmark::benchmark_main) +endif() From ace2e842c8ffa55e525ecda2a6b64a81d2d10fa4 Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 31 May 2023 16:33:12 +0800 Subject: [PATCH 18/42] Support GTest cmake package Signed-off-by: owent --- CMakeLists.txt | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04682eccde..6c684acf15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -565,10 +565,14 @@ if(BUILD_TESTING) set(GTEST_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googletest/include ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest/googlemock/include) - set(GTEST_BOTH_LIBRARIES - ${CMAKE_BINARY_DIR}/lib/libgtest.a - ${CMAKE_BINARY_DIR}/lib/libgtest_main.a - ${CMAKE_BINARY_DIR}/lib/libgmock.a) + if(TARGET gtest) + set(GTEST_BOTH_LIBRARIES gtest gtest_main gmock) + else() + set(GTEST_BOTH_LIBRARIES + ${CMAKE_BINARY_DIR}/lib/libgtest.a + ${CMAKE_BINARY_DIR}/lib/libgtest_main.a + ${CMAKE_BINARY_DIR}/lib/libgmock.a) + endif() elseif(WIN32) # Make sure we are always bootsrapped with vcpkg on Windows find_package(GTest) @@ -580,8 +584,18 @@ if(BUILD_TESTING) else() # Prefer GTest installed by OS distro, brew or vcpkg package manager find_package(GTest REQUIRED) + if(NOT GTEST_BOTH_LIBRARIES) + # New GTest package names + if(TARGET GTest::gtest) + set(GTEST_BOTH_LIBRARIES GTest::gtest GTest::gtest_main GTest::gmock) + elseif(TARGET GTest::GTest) + set(GTEST_BOTH_LIBRARIES GTest::GTest GTest::Main) + endif() + endif() + endif() + if(GTEST_INCLUDE_DIRS) + include_directories(SYSTEM ${GTEST_INCLUDE_DIRS}) endif() - include_directories(SYSTEM ${GTEST_INCLUDE_DIRS}) message("GTEST_INCLUDE_DIRS = ${GTEST_INCLUDE_DIRS}") message("GTEST_BOTH_LIBRARIES = ${GTEST_BOTH_LIBRARIES}") enable_testing() From 2e1af98d8be50cd6c38423480c835e34108e9a45 Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 31 May 2023 17:17:43 +0800 Subject: [PATCH 19/42] Optimize protoc execute. Signed-off-by: owent --- cmake/opentelemetry-proto.cmake | 120 ++++++++++++++++-------------- cmake/patch-imported-config.cmake | 3 + 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 26849f858d..59dc1b9b6e 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -167,64 +167,74 @@ if(WITH_OTLP_GRPC) message(STATUS "gRPC_CPP_PLUGIN_EXECUTABLE=${gRPC_CPP_PLUGIN_EXECUTABLE}") endif() +set(PROTOBUF_COMMON_FLAGS "--proto_path=${PROTO_PATH}" + "--cpp_out=${GENERATED_PROTOBUF_PATH}") +if(Protobuf_VERSION AND Protobuf_VERSION VERSION_LESS "3.16") + list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") +elseif(PROTOBUF_VERSION AND PROTOBUF_VERSION VERSION_LESS "3.16") + list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") +endif() + +set(PROTOBUF_GENERATED_FILES + ${COMMON_PB_H_FILE} + ${COMMON_PB_CPP_FILE} + ${RESOURCE_PB_H_FILE} + ${RESOURCE_PB_CPP_FILE} + ${TRACE_PB_H_FILE} + ${TRACE_PB_CPP_FILE} + ${LOGS_PB_H_FILE} + ${LOGS_PB_CPP_FILE} + ${METRICS_PB_H_FILE} + ${METRICS_PB_CPP_FILE} + ${TRACE_SERVICE_PB_H_FILE} + ${TRACE_SERVICE_PB_CPP_FILE} + ${LOGS_SERVICE_PB_H_FILE} + ${LOGS_SERVICE_PB_CPP_FILE} + ${METRICS_SERVICE_PB_H_FILE} + ${METRICS_SERVICE_PB_CPP_FILE}) + if(WITH_OTLP_GRPC) - add_custom_command( - OUTPUT ${COMMON_PB_H_FILE} - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_H_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_H_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_H_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_H_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_H_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${TRACE_SERVICE_GRPC_PB_H_FILE} - ${TRACE_SERVICE_GRPC_PB_CPP_FILE} - ${LOGS_SERVICE_PB_H_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_GRPC_PB_H_FILE} - ${LOGS_SERVICE_GRPC_PB_CPP_FILE} - ${METRICS_SERVICE_PB_H_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_GRPC_PB_H_FILE} - ${METRICS_SERVICE_GRPC_PB_CPP_FILE} - COMMAND - ${PROTOBUF_PROTOC_EXECUTABLE} ARGS "--experimental_allow_proto3_optional" - "--proto_path=${PROTO_PATH}" ${PROTOBUF_INCLUDE_FLAGS} - "--cpp_out=${GENERATED_PROTOBUF_PATH}" - "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}" - --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}" ${COMMON_PROTO} - ${RESOURCE_PROTO} ${TRACE_PROTO} ${LOGS_PROTO} ${METRICS_PROTO} - ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO} ${METRICS_SERVICE_PROTO}) -else() - add_custom_command( - OUTPUT ${COMMON_PB_H_FILE} - ${COMMON_PB_CPP_FILE} - ${RESOURCE_PB_H_FILE} - ${RESOURCE_PB_CPP_FILE} - ${TRACE_PB_H_FILE} - ${TRACE_PB_CPP_FILE} - ${LOGS_PB_H_FILE} - ${LOGS_PB_CPP_FILE} - ${METRICS_PB_H_FILE} - ${METRICS_PB_CPP_FILE} - ${TRACE_SERVICE_PB_H_FILE} - ${TRACE_SERVICE_PB_CPP_FILE} - ${LOGS_SERVICE_PB_H_FILE} - ${LOGS_SERVICE_PB_CPP_FILE} - ${METRICS_SERVICE_PB_H_FILE} - ${METRICS_SERVICE_PB_CPP_FILE} - COMMAND - ${PROTOBUF_PROTOC_EXECUTABLE} ARGS "--experimental_allow_proto3_optional" - "--proto_path=${PROTO_PATH}" ${PROTOBUF_INCLUDE_FLAGS} - "--cpp_out=${GENERATED_PROTOBUF_PATH}" ${COMMON_PROTO} ${RESOURCE_PROTO} - ${TRACE_PROTO} ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} - ${LOGS_SERVICE_PROTO} ${METRICS_SERVICE_PROTO}) + list(APPEND PROTOBUF_COMMON_FLAGS + "--grpc_out=generate_mock_code=true:${GENERATED_PROTOBUF_PATH}" + --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}") + + list( + APPEND + PROTOBUF_GENERATED_FILES + ${TRACE_SERVICE_GRPC_PB_H_FILE} + ${TRACE_SERVICE_GRPC_PB_CPP_FILE} + ${LOGS_SERVICE_GRPC_PB_H_FILE} + ${LOGS_SERVICE_GRPC_PB_CPP_FILE} + ${METRICS_SERVICE_GRPC_PB_H_FILE} + ${METRICS_SERVICE_GRPC_PB_CPP_FILE}) endif() +set(PROTOBUF_RUN_PROTOC_COMMAND "\"${PROTOBUF_PROTOC_EXECUTABLE}\"") +foreach( + PROTOBUF_RUN_PROTOC_ARG + ${PROTOBUF_COMMON_FLAGS} + ${PROTOBUF_INCLUDE_FLAGS} + ${COMMON_PROTO} + ${RESOURCE_PROTO} + ${TRACE_PROTO} + ${LOGS_PROTO} + ${METRICS_PROTO} + ${TRACE_SERVICE_PROTO} + ${LOGS_SERVICE_PROTO} + ${METRICS_SERVICE_PROTO}) + set(PROTOBUF_RUN_PROTOC_COMMAND + "${PROTOBUF_RUN_PROTOC_COMMAND} \"${PROTOBUF_RUN_PROTOC_ARG}\"") +endforeach() + +add_custom_command( + OUTPUT ${PROTOBUF_GENERATED_FILES} + COMMAND + ${PROTOBUF_PROTOC_EXECUTABLE} ${PROTOBUF_COMMON_FLAGS} + ${PROTOBUF_INCLUDE_FLAGS} ${COMMON_PROTO} ${RESOURCE_PROTO} ${TRACE_PROTO} + ${LOGS_PROTO} ${METRICS_PROTO} ${TRACE_SERVICE_PROTO} ${LOGS_SERVICE_PROTO} + ${METRICS_SERVICE_PROTO} + COMMENT "[Run]: ${PROTOBUF_RUN_PROTOC_COMMAND}") + include_directories("${GENERATED_PROTOBUF_PATH}") unset(OTELCPP_PROTO_TARGET_OPTIONS) diff --git a/cmake/patch-imported-config.cmake b/cmake/patch-imported-config.cmake index 131e99f89b..d4d14b6853 100644 --- a/cmake/patch-imported-config.cmake +++ b/cmake/patch-imported-config.cmake @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + # Common dependencies project_build_tools_patch_default_imported_config(ZLIB::ZLIB) From c5624244bb36ef9ade733229bb536a72f054cbca Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 31 May 2023 19:03:55 +0800 Subject: [PATCH 20/42] Rename `proto-options-patch.cmake` to `tools.cmake` Signed-off-by: owent --- examples/grpc/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/grpc/CMakeLists.txt b/examples/grpc/CMakeLists.txt index 6d08b8255e..938a009e0a 100644 --- a/examples/grpc/CMakeLists.txt +++ b/examples/grpc/CMakeLists.txt @@ -23,7 +23,6 @@ add_custom_command( add_library(example_grpc_proto ${example_grpc_srcs} ${example_grpc_hdrs} ${example_proto_srcs} ${example_proto_hdrs}) -include(${PROJECT_SOURCE_DIR}/cmake/proto-options-patch.cmake) patch_protobuf_targets(example_grpc_proto) include_directories( From 77f4ec1e83ce7a06ef5cde827f30bc21ee769718 Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 31 May 2023 19:25:30 +0800 Subject: [PATCH 21/42] Support set PROTOBUF_PROTOC_EXECUTABLE from protobuf::protoc Signed-off-by: owent --- CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c684acf15..eb07dd49ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -408,10 +408,13 @@ if(WITH_OTLP) ) endif() endif() - # Latest Protobuf uses mixed case instead of uppercase - if(Protobuf_PROTOC_EXECUTABLE) - set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) - endif() + endif() + # Latest Protobuf uses mixed case instead of uppercase + if(TARGET protobuf::protoc) + project_build_tools_get_imported_location(PROTOBUF_PROTOC_EXECUTABLE + protobuf::protoc) + elseif(Protobuf_PROTOC_EXECUTABLE) + set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) endif() include(CMakeDependentOption) From 1fa86311ed8cb16f6135e03b8f5a07076ef9d1b3 Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 31 May 2023 20:26:32 +0800 Subject: [PATCH 22/42] Fix GTest package on Windows Signed-off-by: owent --- CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb07dd49ac..dc6a50b580 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -587,13 +587,13 @@ if(BUILD_TESTING) else() # Prefer GTest installed by OS distro, brew or vcpkg package manager find_package(GTest REQUIRED) - if(NOT GTEST_BOTH_LIBRARIES) - # New GTest package names - if(TARGET GTest::gtest) - set(GTEST_BOTH_LIBRARIES GTest::gtest GTest::gtest_main GTest::gmock) - elseif(TARGET GTest::GTest) - set(GTEST_BOTH_LIBRARIES GTest::GTest GTest::Main) - endif() + endif() + if(NOT GTEST_BOTH_LIBRARIES) + # New GTest package names + if(TARGET GTest::gtest) + set(GTEST_BOTH_LIBRARIES GTest::gtest GTest::gtest_main GTest::gmock) + elseif(TARGET GTest::GTest) + set(GTEST_BOTH_LIBRARIES GTest::GTest GTest::Main) endif() endif() if(GTEST_INCLUDE_DIRS) From 68aa11cdb553f5cdf290bae51fa25e9b19f23f79 Mon Sep 17 00:00:00 2001 From: owent Date: Thu, 1 Jun 2023 11:23:21 +0800 Subject: [PATCH 23/42] Restore `WITH_OTLP_GRPC` in `cmake.maintainer.sync.test` Signed-off-by: owent --- ci/do_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 133f363048..fd690d803e 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -99,7 +99,6 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then cd "${BUILD_DIR}" rm -rf * cmake ${CMAKE_OPTIONS[@]} \ - -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ -DWITH_OTLP_HTTP_SSL_PREVIEW=ON \ -DWITH_OTLP_HTTP_SSL_TLS_PREVIEW=ON \ From f63e1457c91eaac2a262a9bfb1086225b575a771 Mon Sep 17 00:00:00 2001 From: owent Date: Thu, 1 Jun 2023 12:28:45 +0800 Subject: [PATCH 24/42] Fix sepss problem. Signed-off-by: owent --- ci/setup_grpc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index 14682fd2b7..31220f8727 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -7,7 +7,7 @@ set -e export DEBIAN_FRONTEND=noninteractive old_grpc_version='v1.33.2' new_grpc_version='v1.49.2' -moderd_grpc_version='v1.55.0' +modern_grpc_version='v1.55.0' gcc_version_for_new_grpc='5.1' std_version='14' if [ ! -z "${CXX_STANDARD}" ]; then @@ -40,7 +40,7 @@ while getopts ":v:i:mp:r:s:TH" o; do install_grpc_version=${OPTARG} ;; m) - install_grpc_version=${moderd_grpc_version} + install_grpc_version=${modern_grpc_version} ;; s) std_version=${OPTARG} From f2e30099383efe93f7a233051fa1956662f7dd20 Mon Sep 17 00:00:00 2001 From: owent Date: Thu, 1 Jun 2023 22:21:51 +0800 Subject: [PATCH 25/42] Fix undefined uint64_t in metrics headers. Signed-off-by: owent --- sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h b/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h index 1608a3a011..26f272f837 100644 --- a/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h +++ b/sdk/include/opentelemetry/sdk/metrics/data/circular_buffer.h @@ -5,6 +5,7 @@ #include "opentelemetry/nostd/variant.h" +#include #include #include From 85bea458d46ec3c83d62a018c17fc281152d2157 Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 6 Jun 2023 10:52:19 +0800 Subject: [PATCH 26/42] Remove the legacy `protobuf_MODULE_COMPATIBLE` requirement. Signed-off-by: owent --- CMakeLists.txt | 4 ++-- ci/install_protobuf.sh | 8 +++----- ci/setup_grpc.sh | 21 +++++++++++++++++++-- cmake/opentelemetry-proto.cmake | 2 ++ cmake/patch-imported-config.cmake | 4 ++++ 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d13b73e65a..1610f5ff47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,7 +366,6 @@ if(WITH_ABSEIL) endif() if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) - set(protobuf_MODULE_COMPATIBLE ON) find_package(Protobuf) if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") if(NOT WITH_ABSEIL) @@ -407,11 +406,12 @@ if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) endif() endif() endif() - # Latest Protobuf uses mixed case instead of uppercase + # Latest Protobuf imported targets and without legacy module support if(TARGET protobuf::protoc) project_build_tools_get_imported_location(PROTOBUF_PROTOC_EXECUTABLE protobuf::protoc) elseif(Protobuf_PROTOC_EXECUTABLE) + # Some versions of FindProtobuf.cmake uses mixed case instead of uppercase set(PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) endif() include(CMakeDependentOption) diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index cb5b830bb8..459ebaafe6 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -35,17 +35,15 @@ CPP_PROTOBUF_BUILD_OPTIONS=( "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" "-Dprotobuf_BUILD_TESTS=OFF" "-Dprotobuf_BUILD_EXAMPLES=OFF" - "-Dprotobuf_MODULE_COMPATIBLE=ON" ) if [ ! -z "${CXX_STANDARD}" ]; then CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-DCMAKE_CXX_STANDARD=${CXX_STANDARD}") fi -if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then - CPP_PROTOBUF_BUILD_OPTIONS=(${CPP_PROTOBUF_BUILD_OPTIONS[@]} "-Dprotobuf_MODULE_COMPATIBLE=ON") -fi - +# After protobuf 22/4.22, protobuf depends on absl and we can use +# "-Dprotobuf_ABSL_PROVIDER=package" to tell protobuf to find absl +# from the system. Otherwise, it will build absl from source. if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then export CPP_PROTOBUF_VERSION="${PROTOBUF_VERSION}" CPP_PROTOBUF_PACKAGE_NAME="protobuf-${CPP_PROTOBUF_VERSION}" diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index 31220f8727..ad2fe6e00b 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -18,13 +18,29 @@ install_dir='/usr/local/' build_shared_libs='' build_internal_abseil_cpp=1 GRPC_BUILD_OPTIONS=() -usage() { echo "Usage: $0 [-v ] [-i "] 1>&2; exit 1;} +usage() { + echo "Usage: $0 [options...]" 1>&2; + echo "Available options:" 1>&2; + echo " -v Set GCC version" 1>&2; + echo " -h Show help message and exit" 1>&2; + echo " -i Set installation prefix" 1>&2; + echo " -m Use the modern gRPC version" 1>&2; + echo " -p Let gRPC find protobuf or abseil-cpp by CONFIG package" 1>&2; + echo " -r Specify the version of gRPC" 1>&2; + echo " -s Specify std version(CMAKE_CXX_STANDARD)" 1>&2; + echo " -T Build static libraries" 1>&2; + echo " -H Build shared libraries" 1>&2; +} -while getopts ":v:i:mp:r:s:TH" o; do +while getopts ":v:hi:mp:r:s:TH" o; do case "${o}" in v) gcc_version=${OPTARG} ;; + h) + usage + exit 0; + ;; i) install_dir=${OPTARG} ;; @@ -53,6 +69,7 @@ while getopts ":v:i:mp:r:s:TH" o; do ;; *) usage + exit 1; ;; esac done diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 59dc1b9b6e..84b664b3e3 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -169,6 +169,8 @@ endif() set(PROTOBUF_COMMON_FLAGS "--proto_path=${PROTO_PATH}" "--cpp_out=${GENERATED_PROTOBUF_PATH}") +# --experimental_allow_proto3_optional is available from 3.13 and be stable and +# enabled by default from 3.16 if(Protobuf_VERSION AND Protobuf_VERSION VERSION_LESS "3.16") list(APPEND PROTOBUF_COMMON_FLAGS "--experimental_allow_proto3_optional") elseif(PROTOBUF_VERSION AND PROTOBUF_VERSION VERSION_LESS "3.16") diff --git a/cmake/patch-imported-config.cmake b/cmake/patch-imported-config.cmake index d4d14b6853..ec68d74099 100644 --- a/cmake/patch-imported-config.cmake +++ b/cmake/patch-imported-config.cmake @@ -1,6 +1,10 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +# Some prebuilt or installed targets may have different CONFIG settings than +# what we use to configure otel-cpp. This file applies patches to the imported +# targets in order to use compatible CONFIG settings for fallback. + # Common dependencies project_build_tools_patch_default_imported_config(ZLIB::ZLIB) From b903710cd9a8a9607f7c18865e4dbb79cf62e235 Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 6 Jun 2023 11:06:20 +0800 Subject: [PATCH 27/42] Fix format Signed-off-by: owent --- ci/install_protobuf.sh | 6 ++++-- ci/setup_grpc.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/install_protobuf.sh b/ci/install_protobuf.sh index 459ebaafe6..3fb18e7c85 100755 --- a/ci/install_protobuf.sh +++ b/ci/install_protobuf.sh @@ -42,8 +42,10 @@ if [ ! -z "${CXX_STANDARD}" ]; then fi # After protobuf 22/4.22, protobuf depends on absl and we can use -# "-Dprotobuf_ABSL_PROVIDER=package" to tell protobuf to find absl -# from the system. Otherwise, it will build absl from source. +# "-Dprotobuf_ABSL_PROVIDER=package" to tell protobuf to find absl from the +# system. Otherwise, it will build absl from source. +# 4.XX.YY and 3.XX.YY are alias of XX.YY, and source pacakges are moved into the +# tag of XX.YY and without -cpp suffix from protobuf v22. if [[ ${PROTOBUF_VERSION/.*/} -ge 22 ]]; then export CPP_PROTOBUF_VERSION="${PROTOBUF_VERSION}" CPP_PROTOBUF_PACKAGE_NAME="protobuf-${CPP_PROTOBUF_VERSION}" diff --git a/ci/setup_grpc.sh b/ci/setup_grpc.sh index ad2fe6e00b..9cab7e77fe 100755 --- a/ci/setup_grpc.sh +++ b/ci/setup_grpc.sh @@ -37,7 +37,7 @@ while getopts ":v:hi:mp:r:s:TH" o; do v) gcc_version=${OPTARG} ;; - h) + h) usage exit 0; ;; From afed7304bf50e88d0abeb746df82dad15ae02157 Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 4 Jul 2023 11:14:12 +0800 Subject: [PATCH 28/42] Try to use c++14 for modern protobuf and gRPC --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 972fc1a65e..45bcdc4782 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -396,9 +396,9 @@ jobs: submodules: 'recursive' - name: setup env: - PROTOBUF_VERSION: '23.2' + PROTOBUF_VERSION: '23.3' ABSEIL_CPP_VERSION: '20230125.3' - CXX_STANDARD: '17' + CXX_STANDARD: '14' run: | sudo ./ci/setup_cmake.sh sudo ./ci/setup_ci_environment.sh @@ -407,7 +407,7 @@ jobs: - name: run otlp exporter tests env: WITH_ABSEIL: 'ON' - CXX_STANDARD: '17' + CXX_STANDARD: '14' run: | sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test From 43f5996df390e7f52ce34d37d095124b2f6aab8e Mon Sep 17 00:00:00 2001 From: owent Date: Thu, 13 Jul 2023 14:38:23 +0800 Subject: [PATCH 29/42] Use environment to control c++ standardn in CI jobs. --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 399dd9f061..7fbc82ada0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,6 +284,7 @@ jobs: env: CC: /usr/bin/gcc-4.8 CXX: /usr/bin/g++-4.8 + CXX_STANDARD: '11' run: ./ci/do_ci.sh cmake.legacy.test cmake_gcc_48_otlp_exporter_test: @@ -329,8 +330,12 @@ jobs: sudo -E ./ci/setup_ci_environment.sh sudo -E ./ci/setup_cmake.sh - name: run tests + env: + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.test - name: run tests (enable stl) + env: + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.stl.test cmake_test_cxx20_clang: @@ -354,12 +359,14 @@ jobs: CC: /usr/bin/clang CXX: /usr/bin/clang++ CXXFLAGS: "-stdlib=libc++" + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.test - name: run tests (enable stl) env: CC: /usr/bin/clang CXX: /usr/bin/clang++ CXXFLAGS: "-stdlib=libc++" + CXX_STANDARD: '20' run: ./ci/do_ci.sh cmake.c++20.stl.test cmake_otprotocol_test: From b24f299d7dcdaeb4ebf2a78a41b7d193a922f4ba Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 26 Jul 2023 13:08:13 +0800 Subject: [PATCH 30/42] Restore the changelogs by mistake. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f4783dcc..0b27ac0db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,7 +180,7 @@ Breaking changes: [#2118](https://github.com/open-telemetry/opentelemetry-cpp/pull/2118) * [CI] Upgrade Google Benchmark version from 1.6.0 to 1.7.1 [#2116](https://github.com/open-telemetry/opentelemetry-cpp/pull/2116) -* [CI] Upgrade Nlohmann JSON library version from 3.10.5 to 3.11. +* [CI] Upgrade Nlohmann JSON library version from 3.10.5 to 3.11.2 [#2115](https://github.com/open-telemetry/opentelemetry-cpp/pull/2115) * [BUILD] Missed include From c73f9b802dae69e38ab349051a508afaf1b4da43 Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 1 Aug 2023 14:16:30 +0800 Subject: [PATCH 31/42] Ignore more warnings os MSVC for protobuf generated sources. --- cmake/tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 6b91ca07b6..2ba0b22bd9 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -14,7 +14,7 @@ endmacro() if(NOT PATCH_PROTOBUF_SOURCES_OPTIONS_SET) if(MSVC) unset(PATCH_PROTOBUF_SOURCES_OPTIONS CACHE) - set(PATCH_PROTOBUF_SOURCES_OPTIONS /wd4244 /wd4251 /wd4267 /wd4309) + set(PATCH_PROTOBUF_SOURCES_OPTIONS /wd4244 /wd4251 /wd4267 /wd4309 /wd4668 /wd4946 /wd6001 /wd6244 /wd6246) if(MSVC_VERSION GREATER_EQUAL 1922) # see From 7a2c75bbf14edaaa7f290d17e297b58be1fccb75 Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 1 Aug 2023 14:33:19 +0800 Subject: [PATCH 32/42] Remove different package CONFIG supporting. --- CMakeLists.txt | 2 - cmake/patch-imported-config.cmake | 141 ------------------------------ cmake/tools.cmake | 70 --------------- 3 files changed, 213 deletions(-) delete mode 100644 cmake/patch-imported-config.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 77745675a3..564aca99be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -643,8 +643,6 @@ if(NOT WITH_API_ONLY) endif() endif() -include("${PROJECT_SOURCE_DIR}/cmake/patch-imported-config.cmake") - include(cmake/opentelemetry-build-external-component.cmake) if(OPENTELEMETRY_INSTALL) diff --git a/cmake/patch-imported-config.cmake b/cmake/patch-imported-config.cmake deleted file mode 100644 index ec68d74099..0000000000 --- a/cmake/patch-imported-config.cmake +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# Some prebuilt or installed targets may have different CONFIG settings than -# what we use to configure otel-cpp. This file applies patches to the imported -# targets in order to use compatible CONFIG settings for fallback. - -# Common dependencies -project_build_tools_patch_default_imported_config(ZLIB::ZLIB) - -# protobuf targets -if(Protobuf_FOUND OR PROTOBUF_FOUND) - project_build_tools_patch_default_imported_config( - utf8_range::utf8_range utf8_range::utf8_validity protobuf::libprotobuf-lite - protobuf::libprotobuf protobuf::libprotoc) -endif() - -# cares targets -if(TARGET c-ares::cares) - project_build_tools_patch_default_imported_config(c-ares::cares) -endif() - -# curl targets -if(TARGET CURL::libcurl) - project_build_tools_patch_default_imported_config(CURL::libcurl) -endif() - -# abseil targets -if(WITH_ABSEIL) - project_build_tools_patch_default_imported_config( - absl::bad_variant_access - absl::raw_logging_internal - absl::log_severity - absl::log_internal_check_op - absl::log_internal_nullguard - absl::strings - absl::strings_internal - absl::base - absl::spinlock_wait - absl::int128 - absl::throw_delegate - absl::log_internal_message - absl::examine_stack - absl::stacktrace - absl::debugging_internal - absl::symbolize - absl::demangle_internal - absl::malloc_internal - absl::log_internal_format - absl::log_internal_globals - absl::time - absl::civil_time - absl::time_zone - absl::str_format_internal - absl::log_internal_proto - absl::log_internal_log_sink_set - absl::log_globals - absl::hash - absl::city - absl::bad_optional_access - absl::low_level_hash - absl::log_entry - absl::log_sink - absl::synchronization - absl::graphcycles_internal - absl::strerror - absl::log_internal_conditions - absl::cord - absl::cord_internal - absl::crc_cord_state - absl::crc32c - absl::crc_cpu_detect - absl::crc_internal - absl::cordz_functions - absl::exponential_biased - absl::cordz_info - absl::cordz_handle - absl::leak_check - absl::die_if_null - absl::flags - absl::flags_commandlineflag - absl::flags_commandlineflag_internal - absl::flags_config - absl::flags_program_name - absl::flags_internal - absl::flags_marshalling - absl::flags_reflection - absl::flags_private_handle_accessor - absl::raw_hash_set - absl::hashtablez_sampler - absl::log_initialize - absl::status - absl::statusor) -endif() - -# gRPC targets -if(TARGET gRPC::grpc++) - project_build_tools_patch_default_imported_config( - gRPC::cares - gRPC::re2 - gRPC::ssl - gRPC::crypto - gRPC::zlibstatic - gRPC::address_sorting - gRPC::gpr - gRPC::grpc - gRPC::grpc_unsecure - gRPC::grpc++ - gRPC::grpc++_alts - gRPC::grpc++_error_details - gRPC::grpc++_reflection - gRPC::grpc++_unsecure - gRPC::grpc_authorization_provider - gRPC::grpc_plugin_support - gRPC::grpcpp_channelz - gRPC::upb) -endif() - -# prometheus targets -if(TARGET prometheus-cpp::core) - project_build_tools_patch_default_imported_config( - prometheus-cpp::core prometheus-cpp::pull prometheus-cpp::push) -endif() - -# civetweb targets -if(TARGET civetweb::civetweb) - project_build_tools_patch_default_imported_config( - civetweb::civetweb civetweb::server civetweb::civetweb-cpp) -endif() - -if(BUILD_TESTING) - project_build_tools_patch_default_imported_config( - GTest::gtest - GTest::gtest_main - GTest::gmock - GTest::gmock_main - GTest::GTest - GTest::Main - benchmark::benchmark - benchmark::benchmark_main) -endif() diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 2ba0b22bd9..345fc88ff7 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -117,73 +117,3 @@ function(project_build_tools_get_imported_location OUTPUT_VAR_NAME TARGET_NAME) PARENT_SCOPE) endif() endfunction() - -function(project_build_tools_patch_default_imported_config) - set(PATCH_VARS - IMPORTED_IMPLIB - IMPORTED_LIBNAME - IMPORTED_LINK_DEPENDENT_LIBRARIES - IMPORTED_LINK_INTERFACE_LANGUAGES - IMPORTED_LINK_INTERFACE_LIBRARIES - IMPORTED_LINK_INTERFACE_MULTIPLICITY - IMPORTED_LOCATION - IMPORTED_NO_SONAME - IMPORTED_OBJECTS - IMPORTED_SONAME) - foreach(TARGET_NAME ${ARGN}) - if(TARGET ${TARGET_NAME}) - get_target_property(IS_IMPORTED_TARGET ${TARGET_NAME} IMPORTED) - if(NOT IS_IMPORTED_TARGET) - continue() - endif() - - if(CMAKE_VERSION VERSION_LESS "3.19.0") - get_target_property(TARGET_TYPE_NAME ${TARGET_NAME} TYPE) - if(TARGET_TYPE_NAME STREQUAL "INTERFACE_LIBRARY") - continue() - endif() - endif() - - get_target_property(DO_NOT_OVERWRITE ${TARGET_NAME} IMPORTED_LOCATION) - if(DO_NOT_OVERWRITE) - continue() - endif() - - # MSVC's STL and debug level must match the target, so we can only move - # out IMPORTED_LOCATION_NOCONFIG - if(MSVC) - set(PATCH_IMPORTED_CONFIGURATION "NOCONFIG") - else() - get_target_property(PATCH_IMPORTED_CONFIGURATION ${TARGET_NAME} - IMPORTED_CONFIGURATIONS) - endif() - - if(NOT PATCH_IMPORTED_CONFIGURATION) - continue() - endif() - - get_target_property(PATCH_TARGET_LOCATION ${TARGET_NAME} - "IMPORTED_LOCATION_${PATCH_IMPORTED_CONFIGURATION}") - if(NOT PATCH_TARGET_LOCATION) - continue() - endif() - - foreach(PATCH_IMPORTED_KEY IN LISTS PATCH_VARS) - get_target_property( - PATCH_IMPORTED_VALUE ${TARGET_NAME} - "${PATCH_IMPORTED_KEY}_${PATCH_IMPORTED_CONFIGURATION}") - if(PATCH_IMPORTED_VALUE) - set_target_properties( - ${TARGET_NAME} PROPERTIES "${PATCH_IMPORTED_KEY}" - "${PATCH_IMPORTED_VALUE}") - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - message( - STATUS - "Patch: ${TARGET_NAME} ${PATCH_IMPORTED_KEY} will use ${PATCH_IMPORTED_KEY}_${PATCH_IMPORTED_CONFIGURATION}(\"${PATCH_IMPORTED_VALUE}\") by default." - ) - endif() - endif() - endforeach() - endif() - endforeach() -endfunction() From ef1f0be5f93d6fca3a325cae3a0ef12a944584c5 Mon Sep 17 00:00:00 2001 From: owent Date: Thu, 3 Aug 2023 22:28:18 +0800 Subject: [PATCH 33/42] Add CI job without abseil --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fbc82ada0..a7c5f0ae2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -385,8 +385,8 @@ jobs: sudo ./ci/setup_grpc.sh ./ci/do_ci.sh cmake.exporter.otprotocol.test - cmake_modern_protobuf_grpc_test: - name: CMake test (with modern protobuf and grpc) + cmake_modern_protobuf_grpc_with_abseil_test: + name: CMake test (with modern protobuf,grpc and abseil) runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -410,6 +410,30 @@ jobs: sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test + cmake_modern_protobuf_grpc_without_abseil_test: + name: CMake test (with modern protobuf and grpc, without abseil) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: setup + env: + PROTOBUF_VERSION: '23.3' + ABSEIL_CPP_VERSION: '20230125.3' + CXX_STANDARD: '14' + run: | + sudo ./ci/setup_cmake.sh + sudo ./ci/setup_ci_environment.sh + sudo -E ./ci/install_abseil.sh + sudo -E ./ci/install_protobuf.sh + - name: run otlp exporter tests + env: + CXX_STANDARD: '14' + run: | + sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp + ./ci/do_ci.sh cmake.exporter.otprotocol.test + cmake_do_not_install_test: name: CMake do not install test (with otlp-exporter) runs-on: ubuntu-20.04 From edeed2673e2231920b9889bb537f73c1ac174c86 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 00:46:35 +0800 Subject: [PATCH 34/42] Remove dependency of absl for OTLP HTTP client. --- CMakeLists.txt | 9 -- exporters/otlp/src/otlp_http_client.cc | 169 +++++++++++++++++++++---- 2 files changed, 143 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dcf1034804..6f318f8e47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,15 +351,6 @@ endif() if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) find_package(Protobuf) - if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") - if(NOT WITH_ABSEIL) - message( - FATAL_ERROR - "Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125 or upper)" - ) - endif() - endif() - if(WITH_OTLP_GRPC) find_package(gRPC) endif() diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index 297f9c9cca..aa3b8d3c4f 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -19,25 +19,10 @@ #include "google/protobuf/stubs/common.h" #include "nlohmann/json.hpp" -#if defined(HAVE_ABSEIL) -# include "absl/strings/escaping.h" -#elif defined(GOOGLE_PROTOBUF_VERSION) && GOOGLE_PROTOBUF_VERSION >= 3007000 -# include "google/protobuf/stubs/strutil.h" -#else -# include "google/protobuf/stubs/port.h" -# include "google/protobuf/stubs/stringpiece.h" -namespace google -{ -namespace protobuf -{ -LIBPROTOBUF_EXPORT void Base64Escape(StringPiece src, std::string *dest); -} // namespace protobuf -} // namespace google -#endif - #include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" #include "opentelemetry/common/timestamp.h" +#include "opentelemetry/nostd/string_view.h" #include "opentelemetry/sdk/common/global_log_handler.h" #include "opentelemetry/sdk_config.h" @@ -65,6 +50,146 @@ namespace otlp namespace { +using Base64EscapeChars = const unsigned char[64]; +static constexpr Base64EscapeChars kBase64CharsBasic = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + +static int Base64EscapeInternal(unsigned char *dest, + size_t dlen, + size_t *olen, + const unsigned char *src, + size_t slen, + Base64EscapeChars &base64_enc_map, + unsigned char padding_char) +{ + size_t i, n, nopadding; + int C1, C2, C3; + unsigned char *p; + + if (slen == 0) + { + *olen = 0; + return (0); + } + + n = (slen + 2) / 3; + + if (n > (BASE64_SIZE_T_MAX - 1) / 4) + { + *olen = BASE64_SIZE_T_MAX; + return -1; + } + + n *= 4; + + // no padding + if (0 == padding_char) + { + nopadding = slen % 3; + if (0 != nopadding) + { + n -= 3 - nopadding; + } + } + + if ((dlen < n + 1) || (nullptr == dest)) + { + *olen = n + 1; + return -1; + } + + n = (slen / 3) * 3; + + for (i = 0, p = dest; i < n; i += 3) + { + C1 = *src++; + C2 = *src++; + C3 = *src++; + + *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; + *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; + *p++ = base64_enc_map[(((C2 & 15) << 2) + (C3 >> 6)) & 0x3F]; + *p++ = base64_enc_map[C3 & 0x3F]; + } + + if (i < slen) + { + C1 = *src++; + C2 = ((i + 1) < slen) ? *src++ : 0; + + *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; + *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; + + if ((i + 1) < slen) + { + *p++ = base64_enc_map[((C2 & 15) << 2) & 0x3F]; + } + else if (padding_char) + { + *p++ = padding_char; + } + + if (padding_char) + { + *p++ = padding_char; + } + } + + *olen = static_cast(p - dest); + *p = 0; + + return (0); +} + +static inline int Base64EscapeInternal(std::string &dest, + const unsigned char *src, + size_t slen, + Base64EscapeChars &base64_enc_map, + unsigned char padding_char) +{ + size_t olen = 0; + Base64EscapeInternal(nullptr, 0, &olen, src, slen, base64_enc_map, padding_char); + dest.resize(olen); + + if (nullptr == src || 0 == slen) + { + return 0; + } + + int ret = Base64EscapeInternal(reinterpret_cast(&dest[0]), dest.size(), &olen, + src, slen, base64_enc_map, padding_char); +#if defined(HAVE_GSL) + Expects(0 != ret || dest.size() == olen + 1); +#else + assert(0 != ret || dest.size() == olen + 1); +#endif + // pop back last zero + if (!dest.empty() && *dest.rbegin() == 0) + { + dest.resize(dest.size() - 1); + } + return ret; +} + +// Base64Escape() +// +// Encodes a `src` string into a base64-encoded 'dest' string with padding +// characters. This function conforms with RFC 4648 section 4 (base64) and RFC +// 2045. See also CalculateBase64EscapedLen(). +static void Base64Escape(nostd::string_view src, std::string *dest) +{ + if (nullptr == dest) + { + return; + } + + Base64EscapeInternal(*dest, reinterpret_cast(src.data()), src.size(), + kBase64CharsBasic, '='); +} + /** * This class handles the response message from the HTTP request */ @@ -414,22 +539,14 @@ static std::string BytesMapping(const std::string &bytes, else { std::string base64_value; -#if defined(HAVE_ABSEIL) - absl::Base64Escape(bytes, &base64_value); -#else - google::protobuf::Base64Escape(bytes, &base64_value); -#endif + Base64Escape(bytes, &base64_value); return base64_value; } } case JsonBytesMappingKind::kBase64: { // Base64 is the default bytes mapping of protobuf std::string base64_value; -#if defined(HAVE_ABSEIL) - absl::Base64Escape(bytes, &base64_value); -#else - google::protobuf::Base64Escape(bytes, &base64_value); -#endif + Base64Escape(bytes, &base64_value); return base64_value; } case JsonBytesMappingKind::kHex: From 9bfce35d1c21a321eeffc913aca253b4b6e012b8 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 00:58:58 +0800 Subject: [PATCH 35/42] Fix type --- exporters/otlp/src/otlp_http_client.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index aa3b8d3c4f..c7150cfece 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -58,14 +59,14 @@ static constexpr Base64EscapeChars kBase64CharsBasic = { 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; static int Base64EscapeInternal(unsigned char *dest, - size_t dlen, - size_t *olen, + std::size_t dlen, + std::size_t *olen, const unsigned char *src, - size_t slen, + std::size_t slen, Base64EscapeChars &base64_enc_map, unsigned char padding_char) { - size_t i, n, nopadding; + std::size_t i, n, nopadding; int C1, C2, C3; unsigned char *p; @@ -77,9 +78,9 @@ static int Base64EscapeInternal(unsigned char *dest, n = (slen + 2) / 3; - if (n > (BASE64_SIZE_T_MAX - 1) / 4) + if (n > (std::numeric_limits::max() - 1) / 4) { - *olen = BASE64_SIZE_T_MAX; + *olen = std::numeric_limits::max(); return -1; } @@ -138,7 +139,7 @@ static int Base64EscapeInternal(unsigned char *dest, } } - *olen = static_cast(p - dest); + *olen = static_cast(p - dest); *p = 0; return (0); @@ -146,11 +147,11 @@ static int Base64EscapeInternal(unsigned char *dest, static inline int Base64EscapeInternal(std::string &dest, const unsigned char *src, - size_t slen, + std::size_t slen, Base64EscapeChars &base64_enc_map, unsigned char padding_char) { - size_t olen = 0; + std::size_t olen = 0; Base64EscapeInternal(nullptr, 0, &olen, src, slen, base64_enc_map, padding_char); dest.resize(olen); From 0d3b76fdf05aeab1d68d8453db73733051b74e74 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 14:07:32 +0800 Subject: [PATCH 36/42] Move base64 encoder and decoder into sdk --- exporters/otlp/CMakeLists.txt | 14 +- exporters/otlp/src/otlp_http_client.cc | 149 +------- sdk/include/opentelemetry/sdk/common/base64.h | 35 ++ sdk/src/common/BUILD | 13 + sdk/src/common/CMakeLists.txt | 2 +- sdk/src/common/base64.cc | 349 ++++++++++++++++++ sdk/test/common/BUILD | 22 ++ sdk/test/common/CMakeLists.txt | 11 + sdk/test/common/base64_benchmark.cc | 49 +++ sdk/test/common/base64_test.cc | 100 +++++ 10 files changed, 592 insertions(+), 152 deletions(-) create mode 100644 sdk/include/opentelemetry/sdk/common/base64.h create mode 100644 sdk/src/common/base64.cc create mode 100644 sdk/test/common/base64_benchmark.cc create mode 100644 sdk/test/common/base64_test.cc diff --git a/exporters/otlp/CMakeLists.txt b/exporters/otlp/CMakeLists.txt index 9de34bb635..4e6d0766ce 100644 --- a/exporters/otlp/CMakeLists.txt +++ b/exporters/otlp/CMakeLists.txt @@ -225,7 +225,9 @@ if(BUILD_TESTING) TEST_PREFIX exporter.otlp. TEST_LIST otlp_metrics_serialization_test) - if(MSVC) + if(NOT GMOCK_LIB AND TARGET GTest::gmock) + set(GMOCK_LIB GTest::gmock) + elseif(MSVC) # Explicitly specify that we consume GTest from shared library. The rest of # code logic below determines whether we link Release or Debug flavor of the # library. These flavors have different prefix on Windows, gmock and gmockd @@ -238,10 +240,12 @@ if(BUILD_TESTING) unset(GMOCK_LIB CACHE) endif() endif() - if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") - find_library(GMOCK_LIB gmockd PATH_SUFFIXES lib) - else() - find_library(GMOCK_LIB gmock PATH_SUFFIXES lib) + if (NOT GMOCK_LIB) + if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") + find_library(GMOCK_LIB gmockd PATH_SUFFIXES lib) + else() + find_library(GMOCK_LIB gmock PATH_SUFFIXES lib) + endif() endif() if(WITH_OTLP_GRPC) diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index c7150cfece..c6bc1e1122 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -23,6 +23,7 @@ #include "opentelemetry/common/timestamp.h" #include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/sdk/common/base64.h" #include "opentelemetry/sdk/common/global_log_handler.h" #include "opentelemetry/sdk_config.h" @@ -51,146 +52,6 @@ namespace otlp namespace { -using Base64EscapeChars = const unsigned char[64]; -static constexpr Base64EscapeChars kBase64CharsBasic = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - -static int Base64EscapeInternal(unsigned char *dest, - std::size_t dlen, - std::size_t *olen, - const unsigned char *src, - std::size_t slen, - Base64EscapeChars &base64_enc_map, - unsigned char padding_char) -{ - std::size_t i, n, nopadding; - int C1, C2, C3; - unsigned char *p; - - if (slen == 0) - { - *olen = 0; - return (0); - } - - n = (slen + 2) / 3; - - if (n > (std::numeric_limits::max() - 1) / 4) - { - *olen = std::numeric_limits::max(); - return -1; - } - - n *= 4; - - // no padding - if (0 == padding_char) - { - nopadding = slen % 3; - if (0 != nopadding) - { - n -= 3 - nopadding; - } - } - - if ((dlen < n + 1) || (nullptr == dest)) - { - *olen = n + 1; - return -1; - } - - n = (slen / 3) * 3; - - for (i = 0, p = dest; i < n; i += 3) - { - C1 = *src++; - C2 = *src++; - C3 = *src++; - - *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; - *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; - *p++ = base64_enc_map[(((C2 & 15) << 2) + (C3 >> 6)) & 0x3F]; - *p++ = base64_enc_map[C3 & 0x3F]; - } - - if (i < slen) - { - C1 = *src++; - C2 = ((i + 1) < slen) ? *src++ : 0; - - *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; - *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; - - if ((i + 1) < slen) - { - *p++ = base64_enc_map[((C2 & 15) << 2) & 0x3F]; - } - else if (padding_char) - { - *p++ = padding_char; - } - - if (padding_char) - { - *p++ = padding_char; - } - } - - *olen = static_cast(p - dest); - *p = 0; - - return (0); -} - -static inline int Base64EscapeInternal(std::string &dest, - const unsigned char *src, - std::size_t slen, - Base64EscapeChars &base64_enc_map, - unsigned char padding_char) -{ - std::size_t olen = 0; - Base64EscapeInternal(nullptr, 0, &olen, src, slen, base64_enc_map, padding_char); - dest.resize(olen); - - if (nullptr == src || 0 == slen) - { - return 0; - } - - int ret = Base64EscapeInternal(reinterpret_cast(&dest[0]), dest.size(), &olen, - src, slen, base64_enc_map, padding_char); -#if defined(HAVE_GSL) - Expects(0 != ret || dest.size() == olen + 1); -#else - assert(0 != ret || dest.size() == olen + 1); -#endif - // pop back last zero - if (!dest.empty() && *dest.rbegin() == 0) - { - dest.resize(dest.size() - 1); - } - return ret; -} - -// Base64Escape() -// -// Encodes a `src` string into a base64-encoded 'dest' string with padding -// characters. This function conforms with RFC 4648 section 4 (base64) and RFC -// 2045. See also CalculateBase64EscapedLen(). -static void Base64Escape(nostd::string_view src, std::string *dest) -{ - if (nullptr == dest) - { - return; - } - - Base64EscapeInternal(*dest, reinterpret_cast(src.data()), src.size(), - kBase64CharsBasic, '='); -} - /** * This class handles the response message from the HTTP request */ @@ -539,16 +400,12 @@ static std::string BytesMapping(const std::string &bytes, } else { - std::string base64_value; - Base64Escape(bytes, &base64_value); - return base64_value; + return opentelemetry::sdk::common::Base64Escape(bytes); } } case JsonBytesMappingKind::kBase64: { // Base64 is the default bytes mapping of protobuf - std::string base64_value; - Base64Escape(bytes, &base64_value); - return base64_value; + return opentelemetry::sdk::common::Base64Escape(bytes); } case JsonBytesMappingKind::kHex: return HexEncode(bytes); diff --git a/sdk/include/opentelemetry/sdk/common/base64.h b/sdk/include/opentelemetry/sdk/common/base64.h new file mode 100644 index 0000000000..6d720a9649 --- /dev/null +++ b/sdk/include/opentelemetry/sdk/common/base64.h @@ -0,0 +1,35 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/version.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace common +{ +// Base64Escape() +// +// Encodes a `src` string into a base64-encoded 'dest' string with padding +// characters. This function conforms with RFC 4648 section 4 (base64) and RFC +// 2045. +OPENTELEMETRY_EXPORT void Base64Escape(opentelemetry::nostd::string_view src, std::string *dest); +OPENTELEMETRY_EXPORT std::string Base64Escape(opentelemetry::nostd::string_view src); + +// Base64Unescape() +// +// Converts a `src` string encoded in Base64 (RFC 4648 section 4) to its binary +// equivalent, writing it to a `dest` buffer, returning `true` on success. If +// `src` contains invalid characters, `dest` is cleared and returns `false`. +// If padding is included (note that `Base64Escape()` does produce it), it must +// be correct. In the padding, '=' are treated identically. +OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, std::string* dest); + +} // namespace common +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/src/common/BUILD b/sdk/src/common/BUILD index cdbd1955f3..fe75f08d7c 100644 --- a/sdk/src/common/BUILD +++ b/sdk/src/common/BUILD @@ -21,6 +21,19 @@ cc_library( ], ) +cc_library( + name = "base64", + srcs = [ + "base64.cc", + ], + include_prefix = "src/common", + deps = [ + "//api", + "//sdk:headers", + "//sdk/src/common/platform:fork", + ], +) + cc_library( name = "env_variables", srcs = [ diff --git a/sdk/src/common/CMakeLists.txt b/sdk/src/common/CMakeLists.txt index 7bb645f5b7..0dc8a3be72 100644 --- a/sdk/src/common/CMakeLists.txt +++ b/sdk/src/common/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc) +set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc base64.cc) if(WIN32) list(APPEND COMMON_SRCS platform/fork_windows.cc) else() diff --git a/sdk/src/common/base64.cc b/sdk/src/common/base64.cc new file mode 100644 index 0000000000..866e7abfd7 --- /dev/null +++ b/sdk/src/common/base64.cc @@ -0,0 +1,349 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/sdk/common/base64.h" + +#if defined(HAVE_GSL) +# include +#else +# include +#endif +#include + +#if defined(HAVE_ABSEIL) +# include "absl/strings/escaping.h" +#endif + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace common +{ +#if !defined(HAVE_ABSEIL) +namespace +{ +using Base64EscapeChars = const unsigned char[64]; +using Base64UnescapeChars = const unsigned char[128]; + +static constexpr Base64EscapeChars kBase64EscapeCharsBasic = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + +static constexpr Base64UnescapeChars kBase64UnescapeCharsBasic = { + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 62, 127, 127, 127, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 127, 127, 127, 127, 127, 127, 127, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 127, 127, 127, 127, + 127, 127, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 127, 127, 127, 127, 127}; + +static int Base64EscapeInternal(unsigned char *dest, + std::size_t dlen, + std::size_t *olen, + const unsigned char *src, + std::size_t slen, + Base64EscapeChars &base64_enc_map, + unsigned char padding_char) +{ + std::size_t i, n, nopadding; + int C1, C2, C3; + unsigned char *p; + + if (slen == 0) + { + *olen = 0; + return 0; + } + + n = (slen + 2) / 3; + + if (n > (std::numeric_limits::max() - 1) / 4) + { + *olen = std::numeric_limits::max(); + return -1; + } + + n *= 4; + + // no padding + if (0 == padding_char) + { + nopadding = slen % 3; + if (0 != nopadding) + { + n -= 3 - nopadding; + } + } + + if ((dlen < n + 1) || (nullptr == dest)) + { + *olen = n + 1; + return -1; + } + + n = (slen / 3) * 3; + + for (i = 0, p = dest; i < n; i += 3) + { + C1 = *src++; + C2 = *src++; + C3 = *src++; + + *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; + *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; + *p++ = base64_enc_map[(((C2 & 15) << 2) + (C3 >> 6)) & 0x3F]; + *p++ = base64_enc_map[C3 & 0x3F]; + } + + if (i < slen) + { + C1 = *src++; + C2 = ((i + 1) < slen) ? *src++ : 0; + + *p++ = base64_enc_map[(C1 >> 2) & 0x3F]; + *p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F]; + + if ((i + 1) < slen) + { + *p++ = base64_enc_map[((C2 & 15) << 2) & 0x3F]; + } + else if (padding_char) + { + *p++ = padding_char; + } + + if (padding_char) + { + *p++ = padding_char; + } + } + + *olen = static_cast(p - dest); + *p = 0; + + return 0; +} + +static inline int Base64EscapeInternal(std::string &dest, + const unsigned char *src, + std::size_t slen, + Base64EscapeChars &base64_enc_map, + unsigned char padding_char) +{ + std::size_t olen = 0; + Base64EscapeInternal(nullptr, 0, &olen, src, slen, base64_enc_map, padding_char); + dest.resize(olen); + + if (nullptr == src || 0 == slen) + { + return 0; + } + + int ret = Base64EscapeInternal(reinterpret_cast(&dest[0]), dest.size(), &olen, + src, slen, base64_enc_map, padding_char); +# if defined(HAVE_GSL) + Expects(0 != ret || dest.size() == olen + 1); +# else + assert(0 != ret || dest.size() == olen + 1); +# endif + // pop back last zero + if (!dest.empty() && *dest.rbegin() == 0) + { + dest.resize(dest.size() - 1); + } + return ret; +} + +static int Base64UnescapeInternal(unsigned char *dst, + std::size_t dlen, + std::size_t *olen, + const unsigned char *src, + std::size_t slen, + Base64UnescapeChars &base64_dec_map, + unsigned char padding_char) +{ + std::size_t i, n; + std::size_t j, x; + std::size_t valid_slen, line_len; + unsigned char *p; + + /* First pass: check for validity and get output length */ + for (i = n = j = valid_slen = line_len = 0; i < slen; i++) + { + /* Skip spaces before checking for EOL */ + x = 0; + while (i < slen && (src[i] == ' ' || src[i] == '\t')) + { + ++i; + ++x; + } + + /* Spaces at end of buffer are OK */ + if (i == slen) + break; + + if (src[i] == '\r' || src[i] == '\n') + { + line_len = 0; + continue; + } + + /* Space inside a line is an error */ + if (x != 0 && line_len != 0) + return -2; + + ++valid_slen; + ++line_len; + if (src[i] == padding_char) + { + if (++j > 2) + { + return -2; + } + else if ((valid_slen & 3) == 1 || (valid_slen & 3) == 2) + { + // First and second char of every group can not be padding char + return -2; + } + } + else + { + if (src[i] > 127 || base64_dec_map[src[i]] == 127) + return -2; + } + + if (base64_dec_map[src[i]] < 64 && j != 0) + return -2; + + n++; + } + + if (n == 0) + { + *olen = 0; + return 0; + } + + // no padding, add j to padding length + if (valid_slen & 3) + { + j += 4 - (valid_slen & 3); + n += 4 - (valid_slen & 3); + } + + /* The following expression is to calculate the following formula without + * risk of integer overflow in n: + * n = ( ( n * 6 ) + 7 ) >> 3; + */ + n = (6 * (n >> 3)) + ((6 * (n & 0x7) + 7) >> 3); + n -= j; + + if (dst == nullptr || dlen < n) + { + *olen = n; + return -1; + } + + for (n = x = 0, p = dst; i > 0; i--, src++) + { + if (*src == '\r' || *src == '\n' || *src == ' ' || *src == '\t') + continue; + if (*src == padding_char) + continue; + + x = (x << 6) | (base64_dec_map[*src] & 0x3F); + + if (++n == 4) + { + n = 0; + *p++ = (unsigned char)(x >> 16); + *p++ = (unsigned char)(x >> 8); + *p++ = (unsigned char)(x); + } + } + + // no padding, the tail code + if (n == 2) + { + *p++ = (unsigned char)(x >> 4); + } + else if (n == 3) + { + *p++ = (unsigned char)(x >> 10); + *p++ = (unsigned char)(x >> 2); + } + + *olen = static_cast(p - dst); + + return 0; +} + +} // namespace +#endif + +// Base64Escape() +// +// Encodes a `src` string into a base64-encoded 'dest' string with padding +// characters. This function conforms with RFC 4648 section 4 (base64) and RFC +// 2045. +OPENTELEMETRY_EXPORT void Base64Escape(opentelemetry::nostd::string_view src, std::string *dest) +{ + if (nullptr == dest) + { + return; + } + +#if defined(HAVE_ABSEIL) + absl::Base64Escape(absl::string_view{src.data(), src.size()}, dest); +#else + Base64EscapeInternal(*dest, reinterpret_cast(src.data()), src.size(), + kBase64EscapeCharsBasic, '='); +#endif +} + +OPENTELEMETRY_EXPORT std::string Base64Escape(opentelemetry::nostd::string_view src) +{ + std::string result; + + Base64Escape(src, &result); + return result; +} + +OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, std::string *dest) +{ + if (nullptr == dest) + { + return false; + } + +#if defined(HAVE_ABSEIL) + return absl::Base64Unescape(absl::string_view{src.data(), src.size()}, dest); +#else + std::size_t olen = 0; + + if (-2 == Base64UnescapeInternal(nullptr, 0, &olen, + reinterpret_cast(src.data()), src.size(), + kBase64UnescapeCharsBasic, '=')) + { + return false; + } + + if (src.empty()) + { + return true; + } + + dest->resize(olen); + Base64UnescapeInternal(reinterpret_cast(&(*dest)[0]), dest->size(), &olen, + reinterpret_cast(src.data()), src.size(), + kBase64UnescapeCharsBasic, '='); + return true; +#endif +} + +} // namespace common +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/test/common/BUILD b/sdk/test/common/BUILD index 0f886ec569..b08bcc0976 100644 --- a/sdk/test/common/BUILD +++ b/sdk/test/common/BUILD @@ -15,6 +15,18 @@ cc_test( ], ) +cc_test( + name = "base64_test", + srcs = [ + "base64_test.cc", + ], + tags = ["test"], + deps = [ + "//sdk/src/common:base64", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "fast_random_number_generator_test", srcs = [ @@ -27,6 +39,16 @@ cc_test( ], ) +otel_cc_benchmark( + name = "base64_benchmark", + srcs = ["base64_benchmark.cc"], + tags = [ + "benchmark", + "test", + ], + deps = ["//sdk/src/common:base64"], +) + otel_cc_benchmark( name = "random_benchmark", srcs = ["random_benchmark.cc"], diff --git a/sdk/test/common/CMakeLists.txt b/sdk/test/common/CMakeLists.txt index 0205061911..00ad67a5bf 100644 --- a/sdk/test/common/CMakeLists.txt +++ b/sdk/test/common/CMakeLists.txt @@ -24,11 +24,22 @@ foreach( TEST_LIST ${testname}) endforeach() +add_executable(base64_test base64_test.cc) +target_link_libraries(base64_test ${GTEST_BOTH_LIBRARIES} opentelemetry_common) +gtest_add_tests( + TARGET base64_test + TEST_PREFIX common. + TEST_LIST base64_test) + add_executable(random_fork_test random_fork_test.cc) target_link_libraries(random_fork_test opentelemetry_common) add_test(random_fork_test random_fork_test) if(WITH_BENCHMARK) + add_executable(base64_benchmark base64_benchmark.cc) + target_link_libraries(base64_benchmark benchmark::benchmark + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) + add_executable(random_benchmark random_benchmark.cc) target_link_libraries(random_benchmark benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) diff --git a/sdk/test/common/base64_benchmark.cc b/sdk/test/common/base64_benchmark.cc new file mode 100644 index 0000000000..24840042fc --- /dev/null +++ b/sdk/test/common/base64_benchmark.cc @@ -0,0 +1,49 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/sdk/common/base64.h" + +#include +#include + +#include + +namespace +{ + +static const unsigned char base64_test_dec[64] = { + 0x24, 0x48, 0x6E, 0x56, 0x87, 0x62, 0x5A, 0xBD, 0xBF, 0x17, 0xD9, 0xA2, 0xC4, 0x17, 0x1A, 0x01, + 0x94, 0xED, 0x8F, 0x1E, 0x11, 0xB3, 0xD7, 0x09, 0x0C, 0xB6, 0xE9, 0x10, 0x6F, 0x22, 0xEE, 0x13, + 0xCA, 0xB3, 0x07, 0x05, 0x76, 0xC9, 0xFA, 0x31, 0x6C, 0x08, 0x34, 0xFF, 0x8D, 0xC2, 0x6C, 0x38, + 0x00, 0x43, 0xE9, 0x54, 0x97, 0xAF, 0x50, 0x4B, 0xD1, 0x41, 0xBA, 0x95, 0x31, 0x5A, 0x0B, 0x97}; + +static const unsigned char base64_test_enc_rfc_2045[] = + "JEhuVodiWr2/F9mixBcaAZTtjx4Rs9cJDLbpEG8i7hPK" + "swcFdsn6MWwINP+Nwmw4AEPpVJevUEvRQbqVMVoLlw=="; + +void BM_Base64Escape(benchmark::State &state) +{ + while (state.KeepRunning()) + { + benchmark::DoNotOptimize( + opentelemetry::sdk::common::Base64Escape(opentelemetry::nostd::string_view{ + reinterpret_cast(base64_test_dec), sizeof(base64_test_dec)})); + } +} +BENCHMARK(BM_Base64Escape); + +void BM_Base64Unescape(benchmark::State &state) +{ + std::string dest; + while (state.KeepRunning()) + { + benchmark::DoNotOptimize(opentelemetry::sdk::common::Base64Unescape( + opentelemetry::nostd::string_view{reinterpret_cast(base64_test_enc_rfc_2045), + 88}, + &dest)); + } +} +BENCHMARK(BM_Base64Unescape); + +} // namespace +BENCHMARK_MAIN(); diff --git a/sdk/test/common/base64_test.cc b/sdk/test/common/base64_test.cc new file mode 100644 index 0000000000..353959d51d --- /dev/null +++ b/sdk/test/common/base64_test.cc @@ -0,0 +1,100 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/sdk/common/base64.h" + +#include + +#include + +static const unsigned char base64_test_dec[64] = { + 0x24, 0x48, 0x6E, 0x56, 0x87, 0x62, 0x5A, 0xBD, 0xBF, 0x17, 0xD9, 0xA2, 0xC4, 0x17, 0x1A, 0x01, + 0x94, 0xED, 0x8F, 0x1E, 0x11, 0xB3, 0xD7, 0x09, 0x0C, 0xB6, 0xE9, 0x10, 0x6F, 0x22, 0xEE, 0x13, + 0xCA, 0xB3, 0x07, 0x05, 0x76, 0xC9, 0xFA, 0x31, 0x6C, 0x08, 0x34, 0xFF, 0x8D, 0xC2, 0x6C, 0x38, + 0x00, 0x43, 0xE9, 0x54, 0x97, 0xAF, 0x50, 0x4B, 0xD1, 0x41, 0xBA, 0x95, 0x31, 0x5A, 0x0B, 0x97}; + +static const unsigned char base64_test_enc_rfc_2045[] = + "JEhuVodiWr2/F9mixBcaAZTtjx4Rs9cJDLbpEG8i7hPK" + "swcFdsn6MWwINP+Nwmw4AEPpVJevUEvRQbqVMVoLlw=="; + +TEST(Base64Test, EscapeRfc2045) +{ + std::string encoded = opentelemetry::sdk::common::Base64Escape(opentelemetry::nostd::string_view{ + reinterpret_cast(base64_test_dec), sizeof(base64_test_dec)}); + opentelemetry::nostd::string_view expected{ + reinterpret_cast(base64_test_enc_rfc_2045), 88}; + + EXPECT_EQ(encoded, expected); +}; + +TEST(Base64Test, UnescapeRfc2045) +{ + std::string decoded; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape( + opentelemetry::nostd::string_view{reinterpret_cast(base64_test_enc_rfc_2045), + 88}, + &decoded)); + opentelemetry::nostd::string_view expected{reinterpret_cast(base64_test_dec), + sizeof(base64_test_dec)}; + + EXPECT_EQ(decoded, expected); +}; + +TEST(Base64Test, UnescapeRfc2045InvalidInput) +{ + std::string std_str_out; + std::string std_str_in = "="; + + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "J="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhu="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhuV="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhuVodi="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JEhuVodiW="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " J="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhu="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhuV="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhuVodi="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JEhuVodiW="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "JE="; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JE=="; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JE==huVo"; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JE=huVo="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = " JE="; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JE=="; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JE==huVo"; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = " JE=huVo="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "JE= "; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "JE==\r\n"; + EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + + std_str_in = "J E="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); + std_str_in = "J E=="; + EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); +}; From 67dc6ff8bd0ead4f0c331932ab7b36b5ff13db3a Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 14:09:44 +0800 Subject: [PATCH 37/42] Early exit if base64 source is empty --- sdk/src/common/base64.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/src/common/base64.cc b/sdk/src/common/base64.cc index 866e7abfd7..28b863f565 100644 --- a/sdk/src/common/base64.cc +++ b/sdk/src/common/base64.cc @@ -291,7 +291,7 @@ static int Base64UnescapeInternal(unsigned char *dst, // 2045. OPENTELEMETRY_EXPORT void Base64Escape(opentelemetry::nostd::string_view src, std::string *dest) { - if (nullptr == dest) + if (nullptr == dest || src.empty()) { return; } @@ -322,6 +322,11 @@ OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, #if defined(HAVE_ABSEIL) return absl::Base64Unescape(absl::string_view{src.data(), src.size()}, dest); #else + if (src.empty()) + { + return true; + } + std::size_t olen = 0; if (-2 == Base64UnescapeInternal(nullptr, 0, &olen, @@ -331,11 +336,6 @@ OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, return false; } - if (src.empty()) - { - return true; - } - dest->resize(olen); Base64UnescapeInternal(reinterpret_cast(&(*dest)[0]), dest->size(), &olen, reinterpret_cast(src.data()), src.size(), From b8f2e5c6d07b26da23c9b9c4d7bedd0cdc9a709e Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 14:17:18 +0800 Subject: [PATCH 38/42] Fix including --- sdk/src/common/base64.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/src/common/base64.cc b/sdk/src/common/base64.cc index 28b863f565..11d9299d36 100644 --- a/sdk/src/common/base64.cc +++ b/sdk/src/common/base64.cc @@ -8,6 +8,7 @@ #else # include #endif +#include #include #if defined(HAVE_ABSEIL) @@ -326,7 +327,7 @@ OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, { return true; } - + std::size_t olen = 0; if (-2 == Base64UnescapeInternal(nullptr, 0, &olen, From c9967386a78cabfeed01388afbe71bab9ccf5480 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 14:39:11 +0800 Subject: [PATCH 39/42] Fix style --- exporters/otlp/BUILD | 1 + exporters/otlp/CMakeLists.txt | 2 +- sdk/include/opentelemetry/sdk/common/base64.h | 2 +- sdk/src/common/CMakeLists.txt | 7 ++++++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/exporters/otlp/BUILD b/exporters/otlp/BUILD index ccc79ad182..5099b43510 100644 --- a/exporters/otlp/BUILD +++ b/exporters/otlp/BUILD @@ -131,6 +131,7 @@ cc_library( "//api", "//ext/src/http/client/curl:http_client_curl", "//sdk:headers", + "//sdk/src/common:base64", "@com_github_opentelemetry_proto//:common_proto_cc", "@com_google_absl//absl/strings", "@github_nlohmann_json//:json", diff --git a/exporters/otlp/CMakeLists.txt b/exporters/otlp/CMakeLists.txt index 4e6d0766ce..8a93d34acc 100644 --- a/exporters/otlp/CMakeLists.txt +++ b/exporters/otlp/CMakeLists.txt @@ -240,7 +240,7 @@ if(BUILD_TESTING) unset(GMOCK_LIB CACHE) endif() endif() - if (NOT GMOCK_LIB) + if(NOT GMOCK_LIB) if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") find_library(GMOCK_LIB gmockd PATH_SUFFIXES lib) else() diff --git a/sdk/include/opentelemetry/sdk/common/base64.h b/sdk/include/opentelemetry/sdk/common/base64.h index 6d720a9649..918eaaf14a 100644 --- a/sdk/include/opentelemetry/sdk/common/base64.h +++ b/sdk/include/opentelemetry/sdk/common/base64.h @@ -28,7 +28,7 @@ OPENTELEMETRY_EXPORT std::string Base64Escape(opentelemetry::nostd::string_view // `src` contains invalid characters, `dest` is cleared and returns `false`. // If padding is included (note that `Base64Escape()` does produce it), it must // be correct. In the padding, '=' are treated identically. -OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, std::string* dest); +OPENTELEMETRY_EXPORT bool Base64Unescape(opentelemetry::nostd::string_view src, std::string *dest); } // namespace common } // namespace sdk diff --git a/sdk/src/common/CMakeLists.txt b/sdk/src/common/CMakeLists.txt index 0dc8a3be72..b0f44ac139 100644 --- a/sdk/src/common/CMakeLists.txt +++ b/sdk/src/common/CMakeLists.txt @@ -1,7 +1,8 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc base64.cc) +set(COMMON_SRCS random.cc core.cc global_log_handler.cc env_variables.cc + base64.cc) if(WIN32) list(APPEND COMMON_SRCS platform/fork_windows.cc) else() @@ -17,6 +18,10 @@ target_link_libraries( opentelemetry_common PUBLIC opentelemetry_api opentelemetry_sdk Threads::Threads) +if(WITH_ABSEIL) + target_link_libraries(opentelemetry_common PUBLIC absl::strings) +endif() + if(OPENTELEMETRY_INSTALL) install( TARGETS opentelemetry_common From ec5f39ee6bd1f2b1a30847896993460e5d48b691 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 15:01:57 +0800 Subject: [PATCH 40/42] Remove unsupport base64 format of absl --- sdk/test/common/base64_test.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sdk/test/common/base64_test.cc b/sdk/test/common/base64_test.cc index 353959d51d..468dd3b790 100644 --- a/sdk/test/common/base64_test.cc +++ b/sdk/test/common/base64_test.cc @@ -70,8 +70,6 @@ TEST(Base64Test, UnescapeRfc2045InvalidInput) std_str_in = " JEhuVodiW="; EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); - std_str_in = "JE="; - EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); std_str_in = "JE=="; EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); std_str_in = "JE==huVo"; @@ -79,8 +77,6 @@ TEST(Base64Test, UnescapeRfc2045InvalidInput) std_str_in = "JE=huVo="; EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); - std_str_in = " JE="; - EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); std_str_in = " JE=="; EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); std_str_in = " JE==huVo"; @@ -88,13 +84,9 @@ TEST(Base64Test, UnescapeRfc2045InvalidInput) std_str_in = " JE=huVo="; EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); - std_str_in = "JE= "; - EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); std_str_in = "JE==\r\n"; EXPECT_TRUE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); std_str_in = "J E="; EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); - std_str_in = "J E=="; - EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); }; From fd9dc99de00b50259ebacd796e375b769b25d308 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 15:45:13 +0800 Subject: [PATCH 41/42] Fix including --- sdk/src/common/base64.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/src/common/base64.cc b/sdk/src/common/base64.cc index 11d9299d36..04a865ffe8 100644 --- a/sdk/src/common/base64.cc +++ b/sdk/src/common/base64.cc @@ -10,6 +10,7 @@ #endif #include #include +#include #if defined(HAVE_ABSEIL) # include "absl/strings/escaping.h" From fdff03d9d4b845ba92202d18081db7428be2b828 Mon Sep 17 00:00:00 2001 From: owent Date: Fri, 4 Aug 2023 17:09:34 +0800 Subject: [PATCH 42/42] Restore protobuf and abseil checking --- .github/workflows/ci.yml | 24 ------------------- CMakeLists.txt | 9 +++++++ api/BUILD | 1 + sdk/include/opentelemetry/sdk/common/base64.h | 1 + sdk/test/common/base64_benchmark.cc | 8 +++---- sdk/test/common/base64_test.cc | 6 ++--- 6 files changed, 18 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7c5f0ae2f..39207e214c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -410,30 +410,6 @@ jobs: sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp ./ci/do_ci.sh cmake.exporter.otprotocol.test - cmake_modern_protobuf_grpc_without_abseil_test: - name: CMake test (with modern protobuf and grpc, without abseil) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: setup - env: - PROTOBUF_VERSION: '23.3' - ABSEIL_CPP_VERSION: '20230125.3' - CXX_STANDARD: '14' - run: | - sudo ./ci/setup_cmake.sh - sudo ./ci/setup_ci_environment.sh - sudo -E ./ci/install_abseil.sh - sudo -E ./ci/install_protobuf.sh - - name: run otlp exporter tests - env: - CXX_STANDARD: '14' - run: | - sudo -E ./ci/setup_grpc.sh -m -p protobuf -p abseil-cpp - ./ci/do_ci.sh cmake.exporter.otprotocol.test - cmake_do_not_install_test: name: CMake do not install test (with otlp-exporter) runs-on: ubuntu-20.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f318f8e47..dcf1034804 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,6 +351,15 @@ endif() if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) find_package(Protobuf) + if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") + if(NOT WITH_ABSEIL) + message( + FATAL_ERROR + "Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125 or upper)" + ) + endif() + endif() + if(WITH_OTLP_GRPC) find_package(gRPC) endif() diff --git a/api/BUILD b/api/BUILD index 421d26a626..e650d5f7ce 100644 --- a/api/BUILD +++ b/api/BUILD @@ -22,6 +22,7 @@ cc_library( deps = select({ ":with_external_abseil": [ "@com_google_absl//absl/base", + "@com_google_absl//absl/strings", "@com_google_absl//absl/types:variant", ], "//conditions:default": [], diff --git a/sdk/include/opentelemetry/sdk/common/base64.h b/sdk/include/opentelemetry/sdk/common/base64.h index 918eaaf14a..d88204675c 100644 --- a/sdk/include/opentelemetry/sdk/common/base64.h +++ b/sdk/include/opentelemetry/sdk/common/base64.h @@ -5,6 +5,7 @@ #include +#include "opentelemetry/common/macros.h" #include "opentelemetry/nostd/string_view.h" #include "opentelemetry/version.h" diff --git a/sdk/test/common/base64_benchmark.cc b/sdk/test/common/base64_benchmark.cc index 24840042fc..acb8b01ff4 100644 --- a/sdk/test/common/base64_benchmark.cc +++ b/sdk/test/common/base64_benchmark.cc @@ -1,12 +1,12 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -#include "opentelemetry/sdk/common/base64.h" +#include -#include -#include +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/sdk/common/base64.h" -#include +#include "benchmark/benchmark.h" namespace { diff --git a/sdk/test/common/base64_test.cc b/sdk/test/common/base64_test.cc index 468dd3b790..74ebe5cf27 100644 --- a/sdk/test/common/base64_test.cc +++ b/sdk/test/common/base64_test.cc @@ -25,7 +25,7 @@ TEST(Base64Test, EscapeRfc2045) reinterpret_cast(base64_test_enc_rfc_2045), 88}; EXPECT_EQ(encoded, expected); -}; +} TEST(Base64Test, UnescapeRfc2045) { @@ -38,7 +38,7 @@ TEST(Base64Test, UnescapeRfc2045) sizeof(base64_test_dec)}; EXPECT_EQ(decoded, expected); -}; +} TEST(Base64Test, UnescapeRfc2045InvalidInput) { @@ -89,4 +89,4 @@ TEST(Base64Test, UnescapeRfc2045InvalidInput) std_str_in = "J E="; EXPECT_FALSE(opentelemetry::sdk::common::Base64Unescape(std_str_in, &std_str_out)); -}; +}