From d0395fcf5dc31db9563a691231fac41aa918fa96 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Fri, 1 Sep 2023 10:28:27 -0500 Subject: [PATCH 1/4] Get autest working out of source --- CMakeLists.txt | 22 ++++++-- cmake/ConfigureTransparentProxy.cmake | 2 +- cmake/proxy-verifier.cmake | 79 +++++++++++++++++++++++++++ tests/CMakeLists.txt | 27 +++++++++ tests/tools/plugins/CMakeLists.txt | 10 ---- 5 files changed, 123 insertions(+), 17 deletions(-) create mode 100644 cmake/proxy-verifier.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 97ff35f0225..84dc2bc92c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ option(ENABLE_JEMALLOC "Use jemalloc (default OFF)") option(ENABLE_LUAJIT "Use LuaJIT (default OFF)") option(ENABLE_MIMALLOC "Use mimalloc (default OFF)") option(ENABLE_DOCS "Build docs (default OFF)") +option(ENABLE_AUTEST "Setup autest (default OFF)") option(ENABLE_BENCHMARKS "Build benchmarks (default OFF)") # Setup user @@ -314,11 +315,19 @@ check_symbol_exists( ) check_symbol_exists(TLS1_3_VERSION "openssl/ssl.h" TS_USE_TLS13) -# document tools +if(ENABLE_DOCS OR ENABLE_AUTEST) + find_package(Python3 REQUIRED) + find_program(PipEnv pipenv REQUIRED) +endif() + if(ENABLE_DOCS) - find_package(Python3 REQUIRED) - find_package(Java COMPONENTS Runtime REQUIRED) - find_program(PipEnv pipenv REQUIRED) + find_package(Java COMPONENTS Runtime REQUIRED) +endif() + +if(ENABLE_AUTEST) + set(PROXY_VERIFIER_VERSION "v2.10.1") + set(PROXY_VERIFIER_HASH "SHA1=0f189a37596d7488b5b81b5547df6fc1eadf56a1") + include(proxy-verifier) endif() include(CTest) @@ -414,7 +423,9 @@ if(ENABLE_WCCP) add_subdirectory(src/traffic_wccp) endif() add_subdirectory(src/tests) -add_subdirectory(tests) +if(ENABLE_AUTEST) + add_subdirectory(tests) +endif() add_subdirectory(plugins) add_subdirectory(configs) if(ENABLE_EXAMPLE) @@ -508,4 +519,3 @@ cmake_print_variables(TS_PKGSYSGROUP) cmake_print_variables(BUILD_MACHINE) cmake_print_variables(DEFAULT_STACK_SIZE) cmake_print_variables(CMAKE_INSTALL_RPATH) -cmake_print_variables(OPENSSL_VERSION) diff --git a/cmake/ConfigureTransparentProxy.cmake b/cmake/ConfigureTransparentProxy.cmake index c0964ecded0..e4e947e591e 100644 --- a/cmake/ConfigureTransparentProxy.cmake +++ b/cmake/ConfigureTransparentProxy.cmake @@ -58,7 +58,7 @@ endif() # If the read fails, it will print out a confusing error. This # is to make it clear why the error is happening in that case. -message("ENABLE_TPROXY enabled, looking for value in /usr/include/linux/in.h") +message(STATUS "ENABLE_TPROXY enabled, looking for value in /usr/include/linux/in.h") file(READ "/usr/include/linux/in.h" HEADER_CONTENTS) if(HEADER_CONTENTS MATCHES "#define[ \t]+IP_TRANSPARENT[ \t]+([0-9]+)") set(TS_USE_TPROXY TRUE) diff --git a/cmake/proxy-verifier.cmake b/cmake/proxy-verifier.cmake new file mode 100644 index 00000000000..a2e4f2487cc --- /dev/null +++ b/cmake/proxy-verifier.cmake @@ -0,0 +1,79 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +# This will download and extract proxy-verifier and setup variables to point to it. +# +# Required variables: +# PROXY_VERIFIER_VERSION +# PROXY_VERIFIER_HASH +# +# Defines variables: +# +# PROXY_VERIFIER_PATH Full path to the extracted proxy verifier for the build architecture +# PROXY_VERIFIER_CLIENT Full path to client-verifier +# PROXY_VERIFIER_SERVER Full path to server-verifier + +if(NOT PROXY_VERIFIER_VERSION) + message(FATAL_ERROR "PROXY_VERIFIER_VERSION Required") +endif() + +if(NOT PROXY_VERIFIER_HASH) + message(FATAL_ERROR "PROXY_VERIFIER_HASH Required") +endif() + +# Download proxy-verifier +set(PV_ARCHIVE ${CMAKE_BINARY_DIR}/proxy-verifier/proxy-verifier.tar.gz) +file(DOWNLOAD + https://ci.trafficserver.apache.org/bintray/proxy-verifier-${PROXY_VERIFIER_VERSION}.tar.gz + ${PV_ARCHIVE} + EXPECTED_HASH ${PROXY_VERIFIER_HASH} + SHOW_PROGRESS +) +file(ARCHIVE_EXTRACT + INPUT ${PV_ARCHIVE} +) + +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64" OR + CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "amd64" OR + CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "i686") + set(PV_SUBDIR "linux-amd64") + elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "arm64" OR + CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") + set(PV_SUBDIR "linux-arm64") + else() + message(FATAL_ERROR "Unknown processor ${CMAKE_HOST_SYSTEM_PROCESSOR}") + endif() +elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64" OR + CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "amd64" OR + CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "i686") + set(PV_SUBDIR "darwin-amd64") + elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "arm64") + set(PV_SUBDIR "darwin-arm64") + else() + message(FATAL_ERROR "Unknown processor ${CMAKE_HOST_SYSTEM_PROCESSOR}") + endif() +else() + message(FATAL_ERROR "Host ${CMAKE_HOST_SYSTEM_NAME} doesnt support running proxy verifier") +endif() + +set(PROXY_VERIFIER_PATH ${CMAKE_BINARY_DIR}/proxy-verifier-${PROXY_VERIFIER_VERSION}/${PV_SUBDIR}) +set(PROXY_VERIFIER_CLIENT ${PROXY_VERIFIER_PATH}/verifier-client) +set(PROXY_VERIFIER_SERVER ${PROXY_VERIFIER_PATH}/verifier-server) + +message(STATUS "proxy-verifier setup in ${PROXY_VERIFIER_PATH}") diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a7280798d1c..e17e0eb74c1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,16 @@ # ####################### +function(ADD_AUTEST_PLUGIN _NAME _SOURCES) + add_library(${_NAME} MODULE ${_SOURCES}) + set_target_properties(${_NAME} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/.libs" + PREFIX "" + SUFFIX ".so" + ) + target_link_libraries(${_NAME} PRIVATE traffic_server) +endfunction() + add_subdirectory(tools/plugins) add_subdirectory(gold_tests/bigobj) add_subdirectory(gold_tests/chunked_encoding) @@ -25,3 +35,20 @@ add_subdirectory(gold_tests/pluginTest/tsapi) add_subdirectory(gold_tests/pluginTest/TSVConnFd) add_subdirectory(gold_tests/timeout) add_subdirectory(gold_tests/tls) + +configure_file(Pipfile Pipfile COPYONLY) + +#set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True PIPENV_PIPFILE=${CMAKE_CURRENT_SOURCE_DIR}/Pipfile ${PipEnv}) +set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True ${PipEnv}) +add_custom_target( + autest + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install + COMMAND ${RUNPIPENV} install + COMMAND ${RUNPIPENV} run env autest + --directory ${CMAKE_CURRENT_SOURCE_DIR}/gold_tests + --ats-bin=${CMAKE_INSTALL_PREFIX}/bin + --proxy-verifier-bin ${PROXY_VERIFIER_PATH} + --build-root ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + USES_TERMINAL +) diff --git a/tests/tools/plugins/CMakeLists.txt b/tests/tools/plugins/CMakeLists.txt index 873d8694300..92487302405 100644 --- a/tests/tools/plugins/CMakeLists.txt +++ b/tests/tools/plugins/CMakeLists.txt @@ -15,16 +15,6 @@ # ####################### -function(ADD_AUTEST_PLUGIN _NAME _SOURCES) - add_library(${_NAME} MODULE ${_SOURCES}) - set_target_properties(${_NAME} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/.libs" - PREFIX "" - SUFFIX ".so" - ) - target_link_libraries(${_NAME} PRIVATE traffic_server) -endfunction() - add_autest_plugin(cert_update cert_update.cc) add_autest_plugin(conf_remap_stripped conf_remap_stripped.cc) add_autest_plugin(continuations_verify continuations_verify.cc) From 6590d4fd65b1b8d7d2846c43f2bd0288ad3ca6f4 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Fri, 1 Sep 2023 11:24:53 -0500 Subject: [PATCH 2/4] Fix installing configs with destdir. Create a helper script to run autest again after building the autest target --- cmake/install_configs.cmake | 5 +++-- configs/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/install_configs.cmake b/cmake/install_configs.cmake index f71b5656ba6..8562c549abe 100644 --- a/cmake/install_configs.cmake +++ b/cmake/install_configs.cmake @@ -20,12 +20,13 @@ # If the config file doesn't exist, the source config file is copied without the '.default' extension # If the config file already exists, the source config file is copied with the '.default' extension + file(GLOB CONFIG_FILES ${CONFIG_SOURCE_GLOBS}) -file(MAKE_DIRECTORY ${CONFIG_DEST_PATH}) +file(MAKE_DIRECTORY $ENV{DESTDIR}${CONFIG_DEST_PATH}) foreach(CONFIG_FILE ${CONFIG_FILES}) # remove the '.default' extension from the path cmake_path(GET CONFIG_FILE STEM LAST_ONLY CONFIG_FILE_NAME) - set(DEST_FILE "${CONFIG_DEST_PATH}/${CONFIG_FILE_NAME}") + set(DEST_FILE "$ENV{DESTDIR}${CONFIG_DEST_PATH}/${CONFIG_FILE_NAME}") if (EXISTS ${DEST_FILE}) set(DEST_FILE "${DEST_FILE}.default") endif() diff --git a/configs/CMakeLists.txt b/configs/CMakeLists.txt index 438d2372bea..8565f155e6d 100644 --- a/configs/CMakeLists.txt +++ b/configs/CMakeLists.txt @@ -17,7 +17,7 @@ # This is odd but seems to be the idiomatic way to pass arguments to install scripts install(CODE "set(CONFIG_SOURCE_GLOBS \"${CMAKE_BINARY_DIR}/configs/*.default\" \"${CMAKE_SOURCE_DIR}/configs/*.default\")") -install(CODE "set(CONFIG_DEST_PATH \"$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/etc/trafficserver\")") +install(CODE "set(CONFIG_DEST_PATH \"${CMAKE_INSTALL_PREFIX}/etc/trafficserver\")") install(SCRIPT ${CMAKE_SOURCE_DIR}/cmake/install_configs.cmake) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e17e0eb74c1..e9d4e81ba78 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,10 +36,12 @@ add_subdirectory(gold_tests/pluginTest/TSVConnFd) add_subdirectory(gold_tests/timeout) add_subdirectory(gold_tests/tls) +set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True ${PipEnv}) + configure_file(Pipfile Pipfile COPYONLY) +configure_file(autest.sh.in autest.sh) #set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True PIPENV_PIPFILE=${CMAKE_CURRENT_SOURCE_DIR}/Pipfile ${PipEnv}) -set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True ${PipEnv}) add_custom_target( autest COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install From ca9f5a289adb3b3583b648e66161e3a4b7a4b3fd Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Fri, 1 Sep 2023 11:27:24 -0500 Subject: [PATCH 3/4] cleanup, add missing autest.sh.in --- tests/CMakeLists.txt | 21 ++++++++++----------- tests/autest.sh.in | 13 +++++++++++++ 2 files changed, 23 insertions(+), 11 deletions(-) create mode 100755 tests/autest.sh.in diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e9d4e81ba78..4173e436a0a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -41,16 +41,15 @@ set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True ${PipEnv}) configure_file(Pipfile Pipfile COPYONLY) configure_file(autest.sh.in autest.sh) -#set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True PIPENV_PIPFILE=${CMAKE_CURRENT_SOURCE_DIR}/Pipfile ${PipEnv}) add_custom_target( - autest - COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install - COMMAND ${RUNPIPENV} install - COMMAND ${RUNPIPENV} run env autest - --directory ${CMAKE_CURRENT_SOURCE_DIR}/gold_tests - --ats-bin=${CMAKE_INSTALL_PREFIX}/bin - --proxy-verifier-bin ${PROXY_VERIFIER_PATH} - --build-root ${CMAKE_BINARY_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - USES_TERMINAL + autest + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install + COMMAND ${RUNPIPENV} install + COMMAND ${RUNPIPENV} run env autest + --directory ${CMAKE_CURRENT_SOURCE_DIR}/gold_tests + --ats-bin=${CMAKE_INSTALL_PREFIX}/bin + --proxy-verifier-bin ${PROXY_VERIFIER_PATH} + --build-root ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + USES_TERMINAL ) diff --git a/tests/autest.sh.in b/tests/autest.sh.in new file mode 100755 index 00000000000..5d0325a6bc8 --- /dev/null +++ b/tests/autest.sh.in @@ -0,0 +1,13 @@ +#!/bin/bash +# +# conveinience script for running autest after building the target +# +# + +export LD_LIBRARY_PATH=${CMAKE_INSTALL_PREFIX}/lib + +${RUNPIPENV} run env autest \ + --directory ${CMAKE_CURRENT_SOURCE_DIR}/gold_tests \ + --ats-bin=${CMAKE_INSTALL_PREFIX}/bin \ + --proxy-verifier-bin ${PROXY_VERIFIER_PATH} \ + --build-root ${CMAKE_BINARY_DIR} "$@" From f2ed0b122a4c06f78fae2bfe68c9f156da3ecf60 Mon Sep 17 00:00:00 2001 From: Chris McFarlen Date: Fri, 1 Sep 2023 16:35:21 -0500 Subject: [PATCH 4/4] allow overriding the autest sandbox --- CMakeLists.txt | 2 ++ tests/CMakeLists.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 84dc2bc92c4..c092567e94f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -325,6 +325,8 @@ if(ENABLE_DOCS) endif() if(ENABLE_AUTEST) + set(AUTEST_SANDBOX ${CMAKE_BINARY_DIR}/_sandbox CACHE STRING "Location for autest output (default + CMAKE_BINARY_DIR/_sandbox)") set(PROXY_VERIFIER_VERSION "v2.10.1") set(PROXY_VERIFIER_HASH "SHA1=0f189a37596d7488b5b81b5547df6fc1eadf56a1") include(proxy-verifier) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4173e436a0a..b6e20b090b7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,6 +50,7 @@ add_custom_target( --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} --build-root ${CMAKE_BINARY_DIR} + --sandbox ${AUTEST_SANDBOX} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} USES_TERMINAL )