Allow Cripts to be used directly as @plugin

doc/admin-guide/files/records.yaml.en.rst

@@ -4901,6 +4901,20 @@ Plug-in Configuration
    Enables (``1``) or disables (``0``) the dynamic reload feature for remap
    plugins (`remap.config`). Global plugins (`plugin.config`) do not have dynamic reload feature yet.
 
+.. ts:cv:: CONFIG proxy.config.plugin.compiler_path STRING ""
+
+   Specifies an optional compiler tool path for compiling plugins. This tool should
+   be an executable, which takes two arguments:
+
+   === ======================================================================
+   Arg Description
+   === ======================================================================
+   1   This is the path to the source file, which should be compiled
+   2   This is the path to the DSO file, which will be created and loaded
+   === ======================================================================
+
+   The script should exit with a status code of ``0`` if the compilation was successful.
+
 .. ts:cv:: CONFIG proxy.config.plugin.vc.default_buffer_index INT 8
    :reloadable:
    :overridable:

include/proxy/http/remap/PluginDso.h

@@ -67,7 +67,7 @@ class PluginDso : public PluginThreadContext
   virtual ~PluginDso();
 
   /* DSO Load, unload, get symbols from DSO */
-  virtual bool load(std::string &error);
+  virtual bool load(std::string &error, const fs::path &compilerPath);
   virtual bool unload(std::string &error);
   bool isLoaded();
   bool getSymbol(const char *symbol, void *&address, std::string &error) const;

include/proxy/http/remap/PluginFactory.h

@@ -95,6 +95,7 @@ class PluginFactory
   virtual ~PluginFactory();
 
   PluginFactory &setRuntimeDir(const fs::path &runtimeDir);
+  PluginFactory &setCompilerPath(const fs::path &compilerPath);
   PluginFactory &addSearchDir(const fs::path &searchDir);
 
   RemapPluginInst *getRemapPlugin(const fs::path &configPath, int argc, char **argv, std::string &error, bool dynamicReloadEnabled);
@@ -112,6 +113,7 @@ class PluginFactory
 
   std::vector<fs::path> _searchDirs; /** @brief ordered list of search paths where we look for plugins */
   fs::path _runtimeDir;              /** @brief the path where we would create a temporary copies of the plugins to load */
+  fs::path _compilerPath;            /** @brief the compilation script to use for cripts and other non-DSO plugins */
 
   PluginInstList _instList;
 

include/proxy/http/remap/RemapPluginInfo.h

@@ -83,7 +83,7 @@ class RemapPluginInfo : public PluginDso
   ~RemapPluginInfo();
 
   /* Overload to add / execute remap plugin specific tasks during the plugin loading */
-  bool load(std::string &error) override;
+  bool load(std::string &error, const fs::path &compilerPath) override;
 
   /* Used by the factory to invoke callbacks during plugin load, init and unload  */
   bool init(std::string &error) override;

src/proxy/http/remap/PluginDso.cc

@@ -38,6 +38,8 @@
 #define PluginError Error
 #endif
 
+#include <cstdlib>
+
 namespace
 {
 
@@ -69,7 +71,7 @@ PluginDso::~PluginDso()
 }
 
 bool
-PluginDso::load(std::string &error)
+PluginDso::load(std::string &error, const fs::path &compilerPath)
 {
   /* Clear all errors */
   error.clear();
@@ -89,13 +91,27 @@ PluginDso::load(std::string &error)
   } else {
     PluginDbg(_dbg_ctl(), "plugin '%s' effective path: %s", _configPath.c_str(), _effectivePath.c_str());
 
-    /* Copy the installed plugin DSO to a runtime directory if dynamic reload enabled */
     std::error_code ec;
-    if (isDynamicReloadEnabled() && !copy(_effectivePath, _runtimePath, ec)) {
+
+    if (!_effectivePath.string().ends_with(".so")) {
+      if (!isDynamicReloadEnabled()) {
+        concat_error(error, "Dynamic reload must be enabled for Cript files");
+        result = false;
+      } else {
+        std::string command = compilerPath.string() + " " + _effectivePath.string() + " " + _runtimePath.string();
+
+        if (std::system(command.c_str()) != 0) {
+          concat_error(error, "Compile script failed");
+          result = false;
+        }
+      }
+    } else if (isDynamicReloadEnabled() && !copy(_effectivePath, _runtimePath, ec)) {
       concat_error(error, "failed to create a copy");
       concat_error(error, ec.message());
       result = false;
-    } else {
+    }
+
+    if (result) {
       PluginDbg(_dbg_ctl(), "plugin '%s' runtime path: %s", _configPath.c_str(), _runtimePath.c_str());
 
       /* Save the time for later checking if DSO got modified in consecutive DSO reloads */

src/proxy/http/remap/PluginFactory.cc

@@ -124,6 +124,14 @@ PluginFactory::setRuntimeDir(const fs::path &runtimeDir)
   return *this;
 }
 
+PluginFactory &
+PluginFactory::setCompilerPath(const fs::path &compilerPath)
+{
+  _compilerPath = compilerPath;
+  PluginDbg(_dbg_ctl(), "set plugin compiler path %s", compilerPath.c_str());
+  return *this;
+}
+
 const char *
 PluginFactory::getUuid()
 {
@@ -175,6 +183,20 @@ PluginFactory::getRemapPlugin(const fs::path &configPath, int argc, char **argv,
       runtimePath /= _runtimeDir;
       runtimePath /= effectivePath.relative_path();
 
+      // Special case for Cripts
+      if (!runtimePath.string().ends_with(".so")) {
+        if (_compilerPath.empty()) {
+          error.assign("compiler path not set for compiling plugins");
+          return nullptr;
+        }
+
+        // Add .so to the source file, so e.g. example.cc.so. ToDo: libswoc doesn't allow appending to the extension.
+        std::string newPath = runtimePath.string();
+
+        newPath.append(".so");
+        runtimePath = newPath;
+      }
+
       fs::path parent = runtimePath.parent_path();
       PluginDbg(_dbg_ctl(), "Using effectivePath: [%s] runtimePath: [%s] parent: [%s]", effectivePath.c_str(), runtimePath.c_str(),
                 parent.c_str());
@@ -189,7 +211,7 @@ PluginFactory::getRemapPlugin(const fs::path &configPath, int argc, char **argv,
 
     plugin = new RemapPluginInfo(configPath, effectivePath, runtimePath);
     if (nullptr != plugin) {
-      if (plugin->load(error)) {
+      if (plugin->load(error, _compilerPath)) {
         if (plugin->init(error)) {
           PluginDso::loadedPlugins()->add(plugin);
           inst = RemapPluginInst::init(plugin, argc, argv, error);

src/proxy/http/remap/RemapPluginInfo.cc

@@ -77,11 +77,11 @@ RemapPluginInfo::RemapPluginInfo(const fs::path &configPath, const fs::path &eff
 }
 
 bool
-RemapPluginInfo::load(std::string &error)
+RemapPluginInfo::load(std::string &error, const fs::path &compilerPath)
 {
   error.clear();
 
-  if (!PluginDso::load(error)) {
+  if (!PluginDso::load(error, compilerPath)) {
     return false;
   }
 

src/proxy/http/remap/UrlRewrite.cc

@@ -87,6 +87,25 @@ UrlRewrite::load()
   /* Initialize the plugin factory */
   pluginFactory.setRuntimeDir(RecConfigReadRuntimeDir()).addSearchDir(RecConfigReadPluginDir());
 
+  // This is "optional", and this configuration is not set by default.
+  char buf[PATH_NAME_MAX];
+
+  buf[0] = '\0';
+  RecGetRecordString("proxy.config.plugin.compiler_path", buf, PATH_NAME_MAX);
+  if (strlen(buf) > 0) {
+    std::error_code ec;
+    fs::path compilerPath  = fs::path(buf);
+    fs::file_status status = fs::status(compilerPath, ec);
+
+    if (ec || !swoc::file::is_regular_file(status)) {
+      Error("Configured plugin compiler path '%s' is not a regular file", buf);
+      return false;
+    } else {
+      // This also adds the configuration directory (etc/trafficserver) to find Cripts etc.
+      pluginFactory.setCompilerPath(compilerPath).addSearchDir(RecConfigReadConfigDir());
+    }
+  }
+
   /* Initialize the next hop strategy factory */
   std::string sf = RecConfigReadConfigPath("proxy.config.url_remap.strategies.filename", "strategies.yaml");
   Dbg(dbg_ctl_url_rewrite_regex, "strategyFactory file: %s", sf.c_str());

src/records/RecordsConfig.cc

@@ -53,6 +53,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.bin_path", RECD_STRING, TS_BUILD_BINDIR, RECU_NULL, RR_REQUIRED, RECC_NULL, nullptr, RECA_READ_ONLY}
   ,
+  {RECT_CONFIG, "proxy.config.plugin.compiler_path", RECD_STRING, "", RECU_NULL, RR_REQUIRED, RECC_NULL, nullptr, RECA_READ_ONLY}
+  ,
   // Jira TS-21
   {RECT_CONFIG, "proxy.config.local_state_dir", RECD_STRING, TS_BUILD_RUNTIMEDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_READ_ONLY}
   ,

src/traffic_server/traffic_server.cc

@@ -1036,7 +1036,7 @@ try_loading_plugin(plugin_type_t plugin_type, const fs::path &plugin_path, std::
     const auto runtime_path = temporary_directory / plugin_path.filename();
     const fs::path unused_config;
     auto plugin_info = std::make_unique<RemapPluginInfo>(unused_config, plugin_path, runtime_path);
-    bool loaded      = plugin_info->load(error);
+    bool loaded      = plugin_info->load(error, unused_config); // ToDo: Will this ever need support for cripts
     if (!fs::remove(temporary_directory, ec)) {
       fprintf(stderr, "ERROR: could not remove temporary directory '%s': %s\n", temporary_directory.c_str(), ec.message().c_str());
     }

tools/cripts/compiler.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+##############################################################################
+# This is an example compiler script for compiling Cripts and other plugins
+# on the fly. You would set proxy.config.plugin.compiler_path to point to a
+# customized version of this script, and it will be called whenever a plugin
+# needs to be compiled via remap.config.
+##############################################################################
+
+# Configurable parts
+ATS_ROOT=/opt/ats
+CXX=clang++
+CXXFLAGS="-std=c++20 -I/opt/homebrew/include -undefined dynamic_lookup"
+
+# Probably don't need to change these ?
+STDFLAGS="-shared -fPIC -Wall -Werror -I${ATS_ROOT}/include -L${ATS_ROOT}/lib -lcripts"
+
+# This is optional, but if set, the script will cache the compiled shared objects for faster restarts/reloads
+CACHE_DIR=/tmp/ats-cache
+
+# Extract the arguments, and do some sanity checks
+SOURCE=$1
+DEST=$2
+
+SOURCE_DIR=$(dirname $SOURCE)
+DEST_DIR=$(dirname $DEST)
+SOURCE_FILE=$(basename $SOURCE)
+DEST_FILE=$(basename $DEST)
+
+cd "$SOURCE_DIR"
+if [ $(pwd) != "$SOURCE_DIR" ]; then
+    echo "Failed to cd to $SOURCE_DIR"
+    exit 1
+fi
+
+cd "$DEST_DIR"
+if [ $(pwd) != "$DEST_DIR" ]; then
+    echo "Failed to cd to $DEST_DIR"
+    exit 1
+fi
+
+if [ ! -f "$SOURCE" ]; then
+    echo "Source file $SOURCE does not exist"
+    exit 1
+fi
+
+if [ "${DEST_FILE}" != "${SOURCE_FILE}.so" ]; then
+    echo "Destination file name must match source file name with .so extension"
+    exit 1
+fi
+
+cache_file=""
+if [ -d "$CACHE_DIR" ]; then
+    cache_tree="${CACHE_DIR}${SOURCE_DIR}"
+    cache_file="${cache_tree}/${DEST_FILE}"
+
+    [ -d "$cache_tree" ] || mkdir -p "$cache_tree"
+
+    if [ "$SOURCE" -ot "$cache_file" ]; then
+        cp "$cache_file" "$DEST"
+        exit 0
+    fi
+
+    DEST="$cache_file"
+fi
+
+# Compile the plugin
+${CXX} ${CXXFLAGS} ${STDFLAGS} -o $DEST $SOURCE
+exit_code=$?
+
+if [ $exit_code -ne 0 ]; then
+    echo "Compilation failed with exit code $exit_code"
+    exit $exit_code
+fi
+
+# In case we compiled to the cache, copy from the cache to the runtime destination
+if [ -f "$cache_file" ]; then
+    cp "$cache_file" "${DEST_DIR}/${DEST_FILE}"
+fi
+
+exit 0