Adding a log pipe buffer size configuration test

proxy/logging/LogFile.cc

@@ -199,21 +199,21 @@ LogFile::open_file()
     if (m_pipe_buffer_size) {
       long pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
       if (pipe_size == -1) {
-        Error("get pipe size failed for pipe %s", m_name);
+        Error("Get pipe size failed for pipe %s: %s", m_name, strerror(errno));
       } else {
-        Debug("log-file", "Default pipe size for pipe %s = %ld", m_name, pipe_size);
+        Debug("log-file", "Previous buffer size for pipe %s: %ld", m_name, pipe_size);
       }
 
       int ret = fcntl(m_fd, F_SETPIPE_SZ, m_pipe_buffer_size);
       if (ret == -1) {
-        Error("set pipe size failed for pipe %s", m_name);
+        Error("Set pipe size failed for pipe %s to size %d: %s", m_name, m_pipe_buffer_size, strerror(errno));
       }
 
       pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
       if (pipe_size == -1) {
-        Error("get pipe size failed for pipe %s", m_name);
+        Error("Get pipe size after setting it failed for pipe %s: %s", m_name, strerror(errno));
       } else {
-        Debug("log-file", "NEW pipe size for pipe %s = %ld", m_name, pipe_size);
+        Debug("log-file", "New buffer size for pipe %s: %ld", m_name, pipe_size);
       }
     }
 #endif // F_GETPIPE_SZ

src/traffic_layout/info.cc

@@ -21,6 +21,7 @@
   limitations under the License.
  */
 
+#include <fcntl.h>
 #include <openssl/crypto.h>
 #include "tscore/I_Layout.h"
 #include "tscore/Filenames.h"
@@ -88,6 +89,11 @@ produce_features(bool json)
 #else
   print_feature("TS_HAS_BROTLI", 0, json);
 #endif
+#ifdef F_GETPIPE_SZ
+  print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json);
+#else
+  print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 0, json);
+#endif /* F_GETPIPE_SZ */
   print_feature("TS_HAS_JEMALLOC", TS_HAS_JEMALLOC, json);
   print_feature("TS_HAS_TCMALLOC", TS_HAS_TCMALLOC, json);
   print_feature("TS_HAS_IN6_IS_ADDR_UNSPECIFIED", TS_HAS_IN6_IS_ADDR_UNSPECIFIED, json);

tests/gold_tests/autest-site/when.test.ext

@@ -0,0 +1,36 @@
+'''
+When extensions.
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from autest.api import AddWhenFunction
+import hosts.output as host
+
+
+def FileContains(haystack, needle):
+    with open(haystack) as f:
+        result = needle in f.read()
+
+        host.WriteDebug(
+            ['FileExists', 'when'],
+            "Testing for file content '{0}' in '{1}' : {2}".format(
+                needle, haystack, result))
+
+        return result
+
+
+AddWhenFunction(FileContains)

tests/gold_tests/logging/log_pipe.test.py

@@ -0,0 +1,186 @@
+'''
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+
+Test.Summary = '''
+Test custom log file format
+'''
+Test.SkipUnless(
+    Condition.HasATSFeature('TS_HAS_PIPE_BUFFER_SIZE_CONFIG')
+)
+
+ts_counter = 1
+
+
+def get_ts(logging_config):
+    """
+    Create a Traffic Server process.
+    """
+    global ts_counter
+    ts = Test.MakeATSProcess("ts{}".format(ts_counter))
+    ts_counter += 1
+
+    ts.Disk.records_config.update({
+        'proxy.config.diags.debug.enabled': 1,
+        'proxy.config.diags.debug.tags': 'log-file',
+        })
+
+    # Since we're only verifying logs and not traffic, we don't need an origin
+    # server. The following will simply deny the requests and emit a log
+    # message.
+    ts.Disk.remap_config.AddLine(
+        'map / http://www.linkedin.com/ @action=deny'
+    )
+
+    ts.Disk.logging_yaml.AddLines(logging_config)
+
+    return ts
+
+
+#
+# Test 1: Default configured log pipe size.
+#
+tr = Test.AddTestRun()
+pipe_name = "default_pipe_size.pipe"
+ts = get_ts(
+        '''
+logging:
+  formats:
+    - name: custom
+      format: "%<hii> %<hiih>"
+  logs:
+    - filename: '{}'
+      mode: ascii_pipe
+      format: custom
+'''.format(pipe_name).split("\n")
+)
+
+pipe_path = os.path.join(ts.Variables.LOGDIR, pipe_name)
+
+ts.Streams.All += Testers.ContainsExpression(
+    "Created named pipe .*{}".format(pipe_name),
+    "Verify that the named pipe was created")
+
+ts.Streams.All += Testers.ContainsExpression(
+    "no readers for pipe .*{}".format(pipe_name),
+    "Verify that no readers for the pipe was detected.")
+
+ts.Streams.All += Testers.ExcludesExpression(
+    "New buffer size for pipe".format(pipe_name),
+    "Verify that the default pipe size was used.")
+
+curl = tr.Processes.Process("client_request", 'curl "http://127.0.0.1:{0}" --verbose'.format(
+    ts.Variables.port))
+
+reader_output = os.path.join(ts.Variables.LOGDIR, "reader_output")
+pipe_reader = tr.Processes.Process("pipe_reader", 'cat {} | tee {}'.format(pipe_path, reader_output))
+curl_ready = tr.Processes.Process("curl_ready", 'sleep 30')
+# In the AuTest environment, it can take more than 10 seconds for the log file
+# to be created.
+curl_ready.StartupTimeout = 30
+curl_ready.Ready = When.FileContains(reader_output, '127.0.0.1')
+
+tr.Processes.Default.Command = "sleep 10"
+tr.Processes.Default.Return = 0
+
+# Process ordering.
+tr.Processes.Default.StartBefore(curl_ready)
+curl_ready.StartBefore(curl)
+curl.StartBefore(pipe_reader)
+pipe_reader.StartBefore(ts)
+
+
+#
+# Test 2: Change the log's buffer size.
+#
+tr = Test.AddTestRun()
+pipe_name = "change_pipe_size.pipe"
+# 64 KB is the default, so set the size larger than that to verify we can
+# increase the size.
+pipe_size = 75000
+ts = get_ts(
+        '''
+logging:
+  formats:
+    - name: custom
+      format: "%<hii> %<hiih>"
+  logs:
+    - filename: '{}'
+      mode: ascii_pipe
+      format: custom
+      pipe_buffer_size: {}
+      '''.format(pipe_name, pipe_size).split("\n")
+)
+
+pipe_path = os.path.join(ts.Variables.LOGDIR, pipe_name)
+
+ts.Streams.All += Testers.ContainsExpression(
+    "Created named pipe .*{}".format(pipe_name),
+    "Verify that the named pipe was created")
+
+ts.Streams.All += Testers.ContainsExpression(
+    "no readers for pipe .*{}".format(pipe_name),
+    "Verify that no readers for the pipe was detected.")
+
+ts.Streams.All += Testers.ContainsExpression(
+    "Previous buffer size for pipe .*{}".format(pipe_name),
+    "Verify that the named pipe's size was adjusted")
+
+# See fcntl:
+#   "Attempts to set the pipe capacity below the page size
+#    are silently rounded up to the page size."
+#
+# As a result of this, we cannot check that the pipe size is the exact size we
+# requested, but it should be at least that big. We use the
+# pipe_buffer_is_larger_than.py helper script to verify that the pipe grew in
+# size.
+ts.Streams.All += Testers.ContainsExpression(
+    "New buffer size for pipe.*{}".format(pipe_name),
+    "Verify that the named pipe's size was adjusted")
+buffer_verifier = "pipe_buffer_is_larger_than.py"
+tr.Setup.Copy(buffer_verifier)
+verify_buffer_size = tr.Processes.Process(
+        "verify_buffer_size",
+        "python3 {} {} {}".format(buffer_verifier, pipe_path, pipe_size))
+verify_buffer_size.Return = 0
+verify_buffer_size.Streams.All += Testers.ContainsExpression(
+    "Success",
+    "The buffer size verifier should report success.")
+
+curl = tr.Processes.Process("client_request", 'curl "http://127.0.0.1:{0}" --verbose'.format(
+    ts.Variables.port))
+
+reader_output = os.path.join(ts.Variables.LOGDIR, "reader_output")
+pipe_reader = tr.Processes.Process("pipe_reader", 'cat {} | tee {}'.format(pipe_path, reader_output))
+curl_ready = tr.Processes.Process("curl_ready", 'sleep 30')
+# In the AuTest environment, it can take more than 10 seconds for the log file
+# to be created.
+curl_ready.StartupTimeout = 30
+curl_ready.Ready = When.FileContains(reader_output, '127.0.0.1')
+
+tr.Processes.Default.Command = "sleep 10"
+tr.Processes.Default.Return = 0
+
+
+# Process ordering.
+tr.Processes.Default.StartBefore(verify_buffer_size)
+verify_buffer_size.StartBefore(curl_ready)
+curl_ready.StartBefore(curl)
+curl.StartBefore(pipe_reader)
+pipe_reader.StartBefore(ts)

tests/gold_tests/logging/pipe_buffer_is_larger_than.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+'''
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import argparse
+import fcntl
+import sys
+
+F_SETPIPE_SZ = 1031  # Linux 2.6.35+
+F_GETPIPE_SZ = 1032  # Linux 2.6.35+
+
+
+def parse_args():
+    parser = parser = argparse.ArgumentParser(
+            description='Verify that a FIFO has a buffer of at least a certain size')
+
+    parser.add_argument(
+            'pipe_name',
+            help='The pipe name upon which to verify the size is large enough.')
+
+    parser.add_argument(
+            'minimum_buffer_size',
+            help='The minimu buffer size for the pipe to expect.')
+
+    return parser.parse_args()
+
+
+def test_fifo(fifo, minimum_buffer_size):
+    try:
+        fifo_fd = open(fifo, "rb+", buffering=0)
+        buffer_size = fcntl.fcntl(fifo_fd, F_GETPIPE_SZ)
+
+        if buffer_size >= int(minimum_buffer_size):
+            print("Success. Size is: {} which is larger than: {}".format(
+                buffer_size,
+                minimum_buffer_size))
+            return 0
+        else:
+            print("Fail. Size is: {} which is smaller than: {}".format(
+                buffer_size,
+                minimum_buffer_size))
+            return 1
+    except Exception as e:
+        print("Unable to open fifo, error: {}".format(str(e)))
+        return 2
+
+
+def main():
+    args = parse_args()
+    return test_fifo(args.pipe_name, args.minimum_buffer_size)
+
+
+if __name__ == '__main__':
+    sys.exit(main())