From d6c858cc52b62c88c1418f81128493d0bfc7eea8 Mon Sep 17 00:00:00 2001
From: Eric Lunderberg <elunderberg@octoml.ai>
Date: Wed, 10 Jan 2024 08:49:33 -0600
Subject: [PATCH] [CI] In jenkins.cmd_utils.Sh.tee, check for failing
 subprocess

Prior to this commit, the `Sh.tee` method was implemented by calling
`f"{cmd} | tee"` in `subprocess.run`.  While the `check=True` flag was
used, the return code was from `tee`, not from the command itself.
This causes failures in the command itself to be silently ignored,
such as in [this CI
pipeline](https://ci.tlcpack.ai/blue/organizations/jenkins/tvm-i386/detail/PR-16183/37/pipeline)
in the `ci/scripts/jenkins/s3.py` step.

This commit updates `Sh.tee` to call `subprocess.Popen` for `cmd`, tee
the stdout, and check the return code.  (Roughly adapted from [this
stackoverflow post](https://stackoverflow.com/a/56484734).)
---
 ci/scripts/jenkins/cmd_utils.py | 44 ++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 9 deletions(-)

diff --git a/ci/scripts/jenkins/cmd_utils.py b/ci/scripts/jenkins/cmd_utils.py
index 1b282c50ba0f..57ec39973114 100644
--- a/ci/scripts/jenkins/cmd_utils.py
+++ b/ci/scripts/jenkins/cmd_utils.py
@@ -58,24 +58,50 @@ def tee(self, cmd: str, **kwargs):
         """
         Run 'cmd' in a shell then return the (process, stdout) as a tuple
         """
-        with tempfile.NamedTemporaryFile(delete=False) as f:
-            proc = self.run(f"{cmd} | tee {f.name}", **kwargs)
-            with open(f.name, "r") as f:
-                output = f.read()
-            return proc, output
+
+        logging.info(f"+ {cmd}")
+
+        kwargs = {
+            **self._default_popen_flags(),
+            **kwargs,
+            "stdout": subprocess.PIPE,
+        }
+        proc = subprocess.Popen(cmd, **kwargs)
+
+        stdout = []
+
+        def _tee_output(s):
+            stdout.append(s)
+            print(s, end="")
+
+        while proc.poll() is None:
+            _tee_output(proc.stdout.readline())
+        _tee_output(proc.stdout.read())
+
+        stdout = "".join(stdout)
+        if proc.returncode:
+            raise subprocess.CalledProcessError(proc.returncode, proc.args, stdout)
+
+        return proc, stdout
 
     def run(self, cmd: str, **kwargs):
         logging.info(f"+ {cmd}")
-        defaults = {
+
+        kwargs = {
+            **self._default_popen_flags(),
             "check": True,
+            **kwargs,
+        }
+
+        return subprocess.run(cmd, **kwargs)
+
+    def _default_popen_flags(self):
+        return {
             "shell": True,
             "env": self.env,
             "encoding": "utf-8",
             "cwd": self.cwd,
         }
-        defaults.update(kwargs)
-
-        return subprocess.run(cmd, **defaults)
 
 
 def tags_from_title(title: str) -> List[str]: