From 89314a1a7e7f73123598532cfd83ca1a72f1ec8a Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 14:36:54 +0100
Subject: [PATCH 01/45] Move testrunner deps to separate dir

---
 build_defs/python.build_defs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index aa4f979d..33540194 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -325,17 +325,24 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
         pre_cmd = worker_cmd,
     )
 
+    cmd = ['mv $SRCS_MAIN __main__.py',
+           'mkdir .bootstrap && mv $SRCS_TESTRUNNERDEPS .bootstrap',
+            f'$TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict']
+
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
     return build_rule(
         name=name,
-        srcs=[pex_rule],
+        srcs={
+            "main": pex_rule,
+            "testrunnerdeps" : CONFIG.PYTHON.TESTRUNNER_DEPS
+        },
         deps=deps,
         # N.B. the actual test sources are passed as data files as well. This is needed for pytest but
         #      is faster for unittest as well (because we don't need to rebuild the pex if they change).
         data=data | {'_srcs': srcs} if isinstance(data, dict) else data + srcs,
         outs=[f'{name}.pex'],
         labels=labels + ['test_results_dir'],
-        cmd=f'mv $SRC __main__.py && $TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict',
+        cmd=' && '.join(cmd),
         test_cmd=test_cmd,
         debug_cmd=debug_cmd,
         needs_transitive_deps=True,

From f6d4869891eefada8941495439222e41d77371f5 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 15:23:56 +0100
Subject: [PATCH 02/45] Don't rename files in pip_library

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 33540194..49370477 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -464,7 +464,7 @@ def pip_library(name:str, version:str, hashes:list=None, package_name:str=None,
     )
 
     # Don't include the dependency whl's into our whl. They are later picked up by py_binary anyway.
-    cmd = f'$TOOLS_JARCAT z --suffix="" --exclude_suffix=whl --include_other -i . -r $PKG/{name}:$PKG'
+    cmd = f'$TOOLS_JARCAT z --suffix="" --exclude_suffix=whl --include_other -i .'
     if not licences:
         cmd += ' && find . -name METADATA -or -name PKG-INFO | grep -v "^./build/" | xargs grep -E "License ?:" | grep -v UNKNOWN | cat || true'
 

From 0c71d7b2c18b494d5f864aa8f086c8e1d45b4778 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 15:44:02 +0100
Subject: [PATCH 03/45] If interpreter is build label, then resolve

---
 build_defs/python.build_defs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 49370477..f9debe0a 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -218,6 +218,10 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     return f'#!{shebang}\n'
 
 
+def _resolve_interpreter(interpreter:str):
+    return f'$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
+
+
 def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:list=[], worker:str='',
                 labels:list&features&tags=[], size:str=None, flags:str='', visibility:list=None,
                 sandbox:bool=None, timeout:int=0, flaky:bool|int=0,
@@ -259,7 +263,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
       test_runner (str): Specify which Python test runner to use for these tests. One of
                          `unittest`, `pytest`, or a custom test runner entry point.
     """
-    interpreter = interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER
+    interpreter = _resolve_interpreter(interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER)
     test_runner = test_runner or CONFIG.PYTHON.TEST_RUNNER
     module_dir = CONFIG.PYTHON.MODULE_DIR
     interpreter_options = CONFIG.PYTHON.INTERPRETER_OPTIONS
@@ -309,6 +313,8 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
     deps = [pex_rule, lib_rule]
     shebang = _set_shebang(interpreter, interpreter_options, site)
 
+
+
     # If there are resources specified, they have to get built into the pex.
     # Also add the the value of CONFIG.PYTHON.TESTRUNNER_BOOTSTRAP as a dependency.
     deps += [CONFIG.PYTHON.TESTRUNNER_DEPS]

From 1a706c78101dcd25214fdce370fd45c5eb3a01ff Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:03:46 +0100
Subject: [PATCH 04/45] Pass the substitution to _set_shebang

---
 build_defs/python.build_defs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index f9debe0a..124611ad 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -210,7 +210,7 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
 
 def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     """Returns a shebang to pass to arcat to inject at the top of a pex"""
-    # Assume the path to the interpreter is relative
+    interpreter = _interpreter_cmd(interpreter)
     shebang = f'/usr/bin/env {interpreter}'
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
@@ -218,7 +218,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     return f'#!{shebang}\n'
 
 
-def _resolve_interpreter(interpreter:str):
+def _interpreter_cmd(interpreter:str):
     return f'$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
 
 

From e0f0ad19e00fe7fb8a469aae239a2e72c8be31aa Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:05:17 +0100
Subject: [PATCH 05/45] Woops

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 124611ad..014a44b2 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -263,7 +263,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
       test_runner (str): Specify which Python test runner to use for these tests. One of
                          `unittest`, `pytest`, or a custom test runner entry point.
     """
-    interpreter = _resolve_interpreter(interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER)
+    interpreter = _interpreter_cmd(interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER)
     test_runner = test_runner or CONFIG.PYTHON.TEST_RUNNER
     module_dir = CONFIG.PYTHON.MODULE_DIR
     interpreter_options = CONFIG.PYTHON.INTERPRETER_OPTIONS

From dbb09177e29963df8832cfa4c8ef775c473a3ca9 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:16:22 +0100
Subject: [PATCH 06/45] Don't call here

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 014a44b2..2509c2cf 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -263,7 +263,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
       test_runner (str): Specify which Python test runner to use for these tests. One of
                          `unittest`, `pytest`, or a custom test runner entry point.
     """
-    interpreter = _interpreter_cmd(interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER)
+    interpreter = interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER
     test_runner = test_runner or CONFIG.PYTHON.TEST_RUNNER
     module_dir = CONFIG.PYTHON.MODULE_DIR
     interpreter_options = CONFIG.PYTHON.INTERPRETER_OPTIONS

From c61437aaf4136f6910accb77ee5c18b5da6c9d2f Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:25:32 +0100
Subject: [PATCH 07/45] Put debug in

---
 build_defs/python.build_defs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 2509c2cf..3e7962ea 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -210,6 +210,7 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
 
 def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     """Returns a shebang to pass to arcat to inject at the top of a pex"""
+    fail(f"in _set_shebang. interpreter={interpreter}")
     interpreter = _interpreter_cmd(interpreter)
     shebang = f'/usr/bin/env {interpreter}'
     if not site:

From 6993e40485338669a8b7aa16d5e153f417192d48 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:26:37 +0100
Subject: [PATCH 08/45] Put debug in

---
 build_defs/python.build_defs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 3e7962ea..7b7b115d 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -210,8 +210,9 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
 
 def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     """Returns a shebang to pass to arcat to inject at the top of a pex"""
-    fail(f"in _set_shebang. interpreter={interpreter}")
+    # interpreter=///python/python39/python39//:interpreter
     interpreter = _interpreter_cmd(interpreter)
+    fail(f"in _set_shebang. interpreter={interpreter}")
     shebang = f'/usr/bin/env {interpreter}'
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'

From d7d37f61f8732f86271b65341022e716275d92aa Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:28:08 +0100
Subject: [PATCH 09/45] Put debug in

---
 build_defs/python.build_defs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 7b7b115d..d64b9e0f 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -212,8 +212,9 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     """Returns a shebang to pass to arcat to inject at the top of a pex"""
     # interpreter=///python/python39/python39//:interpreter
     interpreter = _interpreter_cmd(interpreter)
-    fail(f"in _set_shebang. interpreter={interpreter}")
+    # interpreter=$(out ///python/python39/python39//:interpreter)
     shebang = f'/usr/bin/env {interpreter}'
+    fail(f"in _set_shebang. shebang={interpreter}")
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
 

From 63f90d5b87e337bdbee0069e6b645b93fe28a9df Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:28:53 +0100
Subject: [PATCH 10/45] Put debug in

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index d64b9e0f..98e4c3ff 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -214,7 +214,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     interpreter = _interpreter_cmd(interpreter)
     # interpreter=$(out ///python/python39/python39//:interpreter)
     shebang = f'/usr/bin/env {interpreter}'
-    fail(f"in _set_shebang. shebang={interpreter}")
+    fail(f"in _set_shebang. shebang={shebang}")
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
 

From f4c4fb10b8baac2608d03944df93cbf1968a4cab Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:30:58 +0100
Subject: [PATCH 11/45] Take debug out

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 98e4c3ff..be106c68 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -214,7 +214,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     interpreter = _interpreter_cmd(interpreter)
     # interpreter=$(out ///python/python39/python39//:interpreter)
     shebang = f'/usr/bin/env {interpreter}'
-    fail(f"in _set_shebang. shebang={shebang}")
+    # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
 

From f11e2e073f21309edc7ee5f2f0bfd2844ebbef88 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:34:00 +0100
Subject: [PATCH 12/45] Put in quotes

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index be106c68..a89eefec 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -213,7 +213,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     # interpreter=///python/python39/python39//:interpreter
     interpreter = _interpreter_cmd(interpreter)
     # interpreter=$(out ///python/python39/python39//:interpreter)
-    shebang = f'/usr/bin/env {interpreter}'
+    shebang = f'/usr/bin/env "{interpreter}"'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'

From 92dab07a441db0869590ae30969be51ad3e05f39 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:42:22 +0100
Subject: [PATCH 13/45] Escape quotes

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index a89eefec..8783d4c3 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -213,7 +213,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     # interpreter=///python/python39/python39//:interpreter
     interpreter = _interpreter_cmd(interpreter)
     # interpreter=$(out ///python/python39/python39//:interpreter)
-    shebang = f'/usr/bin/env "{interpreter}"'
+    shebang = f'/usr/bin/env \\\"{interpreter}\\\"'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'

From 8062cfd8f3202391964b8eba3d40581b3832a3a5 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 16:45:14 +0100
Subject: [PATCH 14/45] Escape more quotes

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 8783d4c3..ceca7f02 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -222,7 +222,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
 
 
 def _interpreter_cmd(interpreter:str):
-    return f'$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
+    return f'\\\\$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
 
 
 def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:list=[], worker:str='',

From a206822a65559fe008bd7df1a3537a26161a2b65 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 17:01:13 +0100
Subject: [PATCH 15/45] Un-escape quotes

---
 build_defs/python.build_defs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index ceca7f02..52622ba9 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -213,7 +213,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     # interpreter=///python/python39/python39//:interpreter
     interpreter = _interpreter_cmd(interpreter)
     # interpreter=$(out ///python/python39/python39//:interpreter)
-    shebang = f'/usr/bin/env \\\"{interpreter}\\\"'
+    shebang = f'/usr/bin/env {interpreter}'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
@@ -316,8 +316,6 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
     deps = [pex_rule, lib_rule]
     shebang = _set_shebang(interpreter, interpreter_options, site)
 
-
-
     # If there are resources specified, they have to get built into the pex.
     # Also add the the value of CONFIG.PYTHON.TESTRUNNER_BOOTSTRAP as a dependency.
     deps += [CONFIG.PYTHON.TESTRUNNER_DEPS]

From 9278942d56664d47277ba24fa00862d0e7c7803c Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 17:22:05 +0100
Subject: [PATCH 16/45] Escape quotes

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 52622ba9..f4129269 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -216,7 +216,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     shebang = f'/usr/bin/env {interpreter}'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
-        return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
+        return f'#!/bin/sh\nexec \"{shebang}\" -S \\\\$0 \\\"\\\\$@\\\"\n'
 
     return f'#!{shebang}\n'
 

From 4d210932e967e71bd1f5a56051367293c799e1ee Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 17:25:56 +0100
Subject: [PATCH 17/45] Unescape dollar

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index f4129269..4f667040 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -222,7 +222,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
 
 
 def _interpreter_cmd(interpreter:str):
-    return f'\\\\$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
+    return f'$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
 
 
 def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:list=[], worker:str='',

From 11b98d2eb85f2dc6d4bdbc90b20869fb698b2de8 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 1 Apr 2022 17:27:56 +0100
Subject: [PATCH 18/45] More escape

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 4f667040..6ac8e5c3 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -216,7 +216,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     shebang = f'/usr/bin/env {interpreter}'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
-        return f'#!/bin/sh\nexec \"{shebang}\" -S \\\\$0 \\\"\\\\$@\\\"\n'
+        return f'#!/bin/sh\nexec \\"{shebang}\\" -S \\\\$0 \\\"\\\\$@\\\"\n'
 
     return f'#!{shebang}\n'
 

From 5467cbeb3967749efa9f96f9d63ead41353b1408 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Mon, 4 Apr 2022 15:35:04 +0100
Subject: [PATCH 19/45] Remove quotes entirely

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 6ac8e5c3..4ea08268 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -216,7 +216,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     shebang = f'/usr/bin/env {interpreter}'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
-        return f'#!/bin/sh\nexec \\"{shebang}\\" -S \\\\$0 \\\"\\\\$@\\\"\n'
+        return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
 
     return f'#!{shebang}\n'
 

From 753b7e94a63e8e5d3f64b1b5c513fb4b77396b65 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Mon, 4 Apr 2022 15:49:57 +0100
Subject: [PATCH 20/45] Try this

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 4ea08268..4f667040 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -216,7 +216,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     shebang = f'/usr/bin/env {interpreter}'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
-        return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
+        return f'#!/bin/sh\nexec \"{shebang}\" -S \\\\$0 \\\"\\\\$@\\\"\n'
 
     return f'#!{shebang}\n'
 

From 0b366b5b5b609d47796269c76c65504ec5e702bf Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Mon, 4 Apr 2022 15:54:56 +0100
Subject: [PATCH 21/45] Try something else

---
 build_defs/python.build_defs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 4f667040..e3f5a4e7 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -216,13 +216,13 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     shebang = f'/usr/bin/env {interpreter}'
     # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
-        return f'#!/bin/sh\nexec \"{shebang}\" -S \\\\$0 \\\"\\\\$@\\\"\n'
+        return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
 
     return f'#!{shebang}\n'
 
 
 def _interpreter_cmd(interpreter:str):
-    return f'$(out {interpreter})' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
+    return f'\"$(out {interpreter})\"' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
 
 
 def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:list=[], worker:str='',

From 957de1d8e7c581fda463456b58e06411cdb08587 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 5 Apr 2022 10:27:59 +0100
Subject: [PATCH 22/45] Set shebang in config

---
 .plzconfig                   | 6 ++++++
 build_defs/python.build_defs | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index 21d02ee0..b498094e 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -94,6 +94,12 @@ DefaultValue = //tools/wheel_resolver
 Optional = true
 Inherit = true
 
+[PluginConfig "pex_shebang"]
+ConfigKey = PexShebang
+DefaultValue = python3
+Optional = true
+Inherit = true
+
 [featureflags]
 PythonWheelHashing = true
 ExcludePythonRules = true
diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index a4033908..64cfab19 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -131,7 +131,7 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
       labels (list): Labels to apply to this rule.
     """
     interpreter = interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER
-    shebang = shebang or _set_shebang(interpreter, CONFIG.PYTHON.INTERPRETER_OPTIONS, site)
+    shebang = shebang or CONFIG.PYTHON.PEX_SHEBANG
     zipsafe_flag = '' if zip_safe is False else '--zip_safe'
     cmd = '$TOOLS_PEX -m "%s" %s --interpreter_options="%s" --stamp="$RULE_HASH"' % (
         CONFIG.PYTHON.MODULE_DIR, zipsafe_flag, CONFIG.PYTHON.INTERPRETER_OPTIONS)
@@ -307,7 +307,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
     )
 
     deps = [pex_rule, lib_rule]
-    shebang = _set_shebang(interpreter, interpreter_options, site)
+    shebang = shebang or CONFIG.PYTHON.PEX_SHEBANG
 
     # If there are resources specified, they have to get built into the pex.
     # Also add the the value of CONFIG.PYTHON.TESTRUNNER_BOOTSTRAP as a dependency.

From eeda479d03a08cd841313d3418b005f12fe33f56 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 5 Apr 2022 10:52:57 +0100
Subject: [PATCH 23/45] Add shebang as an argument

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 64cfab19..19f62b75 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -220,7 +220,7 @@ def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
 
 def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:list=[], worker:str='',
                 labels:list&features&tags=[], size:str=None, flags:str='', visibility:list=None,
-                sandbox:bool=None, timeout:int=0, flaky:bool|int=0,
+                sandbox:bool=None, timeout:int=0, flaky:bool|int=0, shebang:str=None,
                 test_outputs:list=None, zip_safe:bool=None, interpreter:str=None, site:bool=False,
                 test_runner:str=None):
     """Generates a Python test target.

From 5913c6e2290049dc1739577086020b7ce34ef730 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 5 Apr 2022 15:13:39 +0100
Subject: [PATCH 24/45] Add site argument to interpreter options if needed

---
 .plzconfig                   | 2 +-
 build_defs/python.build_defs | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index b498094e..32f60491 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -96,7 +96,7 @@ Inherit = true
 
 [PluginConfig "pex_shebang"]
 ConfigKey = PexShebang
-DefaultValue = python3
+DefaultValue = "#!/usr/bin/python3"
 Optional = true
 Inherit = true
 
diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 19f62b75..54a12ded 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -288,7 +288,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
         pre_build=_handle_zip_safe,
         deps=deps,
         tools={
-            'interpreter': [interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER],
+            'interpreter': [interpreter],
             'pex': [CONFIG.PYTHON.PEX_TOOL],
         },
         labels = labels,
@@ -309,11 +309,14 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
     deps = [pex_rule, lib_rule]
     shebang = shebang or CONFIG.PYTHON.PEX_SHEBANG
 
+    if not site:
+        interpreter_options = f'{interpreter_options} -S'
+
     # If there are resources specified, they have to get built into the pex.
     # Also add the the value of CONFIG.PYTHON.TESTRUNNER_BOOTSTRAP as a dependency.
     deps += [CONFIG.PYTHON.TESTRUNNER_DEPS]
 
-    test_cmd = f'$TEST {flags}'
+    test_cmd = f'$TOOL {interpreter_options} $TEST {flags}'
     worker_cmd = f'$(worker {worker})' if worker else ""
     if worker_cmd:
         test_cmd = f'{worker_cmd} && {test_cmd} '
@@ -351,6 +354,8 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
         test_outputs=test_outputs,
         requires=['py', 'test', interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER],
         tools=[CONFIG.JARCAT_TOOL],
+        test_tools=[CONFIG.PYTHON.DEFAULT_INTERPRETER],
+            
     )
 
 def pip_library(name:str, version:str, hashes:list=None, package_name:str=None,

From 4dd0f838ce755b481d4ad3a73451c6c1f3051616 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 5 Apr 2022 15:41:03 +0100
Subject: [PATCH 25/45] Revert some stuff that we don't need anymore

---
 build_defs/python.build_defs | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 1691eb3b..e2c3636a 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -210,21 +210,14 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
 
 def _set_shebang(interpreter:str, interpreter_options:str, site:bool):
     """Returns a shebang to pass to arcat to inject at the top of a pex"""
-    # interpreter=///python/python39/python39//:interpreter
-    interpreter = _interpreter_cmd(interpreter)
-    # interpreter=$(out ///python/python39/python39//:interpreter)
+    # Assume the path to the interpreter is relative
     shebang = f'/usr/bin/env {interpreter}'
-    # shebang=/usr/bin/env $(out ///python/python39/python39//:interpreter)
     if not site:
         return f'#!/bin/sh\nexec {shebang} -S \\\\$0 \\\"\\\\$@\\\"\n'
 
     return f'#!{shebang}\n'
 
 
-def _interpreter_cmd(interpreter:str):
-    return f'\"$(out {interpreter})\"' if interpreter.startswith('//') or interpreter.startswith(':') else interpreter
-
-
 def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:list=[], worker:str='',
                 labels:list&features&tags=[], size:str=None, flags:str='', visibility:list=None,
                 sandbox:bool=None, timeout:int=0, flaky:bool|int=0,

From a7f233b3cb734d0a35df7abbeea2bed35602bd62 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Fri, 8 Apr 2022 10:54:39 +0100
Subject: [PATCH 26/45] Get path finder to look in right place

---
 .plzconfig                            |  4 ++--
 build_defs/python.build_defs          |  3 ++-
 tools/BUILD                           | 14 ++++++++++++++
 tools/please_pex/pex/pex_main.py      | 25 +++++++++++++++++++++++++
 tools/please_pex/pex/pex_test_main.py |  2 ++
 5 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index 32f60491..0604bb9e 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -13,7 +13,7 @@ ImportPath = github.com/please-build/python-rules
 gotool = //third_party/go:toolchain|go
 
 [Java]
-JarCatTool = //tools:arcat
+JarCatTool = //tools:arcat_local
 
 [PluginDefinition]
 name = python
@@ -90,7 +90,7 @@ Repeatable = true
 
 [PluginConfig "wheel_tool"]
 ConfigKey = WheelTool
-DefaultValue = //tools/wheel_resolver
+DefaultValue = //tools:wheel_resolver
 Optional = true
 Inherit = true
 
diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index b18152c9..84fdb289 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -328,9 +328,10 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
         pre_cmd = worker_cmd,
     )
 
+    #'cp -r $SRCS_TESTRUNNERDEPS .',
     cmd = ['mv $SRCS_MAIN __main__.py',
            'mkdir .bootstrap && mv $SRCS_TESTRUNNERDEPS .bootstrap',
-            f'$TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict']
+           f'$TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict']
 
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
     return build_rule(
diff --git a/tools/BUILD b/tools/BUILD
index cd186eb8..9e2fe7bb 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -5,3 +5,17 @@ remote_file(
     binary = True,
     visibility = ["PUBLIC"],
 )
+
+filegroup(
+    name = "wheel_resolver",
+    system_srcs = ["/home/swestmoreland/wheel_tool.pex",],
+    binary = True,
+    visibility = ["PUBLIC"],
+)
+
+filegroup(
+    name = "arcat_local",
+    system_srcs = ["/home/swestmoreland/git/arcat/plz-out/bin/arcat"],
+    binary = True,
+    visibility = ["PUBLIC"],
+)
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 24cd1651..a1bac047 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -5,6 +5,7 @@
 import os
 import runpy
 import sys
+import traceback
 
 
 PY_VERSION = sys.version_info
@@ -113,6 +114,8 @@ class SoImport(object):
     """So import. Much binary. Such dynamic. Wow."""
 
     def __init__(self):
+        print("Calling SoImport __init__()")
+        traceback.print_stack(limit=10)
 
         if PY_VERSION.major < 3:
             self.suffixes = {x[0]: x for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION}
@@ -127,6 +130,8 @@ def __init__(self):
             for name in zf.namelist():
                 path, _ = self.splitext(name)
                 if path:
+                    if path.startswith('.bootstrap/'):
+                        path = path[len('.bootstrap/'):]
                     importpath = path.replace('/', '.')
                     self.modules.setdefault(importpath, name)
                     if path.startswith(MODULE_DIR):
@@ -174,20 +179,29 @@ class ModuleDirImport(object):
     """
 
     def __init__(self, module_dir=MODULE_DIR):
+        print("Calling ModuleDirImport __init__()", module_dir)
         self.prefix = module_dir.replace('/', '.') + '.'
+        print("Added prefix", self.prefix)
 
     def find_module(self, fullname, path=None):
+        print("Calling ModuleDirImport.find_module() on", fullname, "with path", path)
+        print("checking against", self.prefix)
+        #traceback.print_stack(limit=10)
         """Attempt to locate module. Returns self if found, None if not."""
         if fullname.startswith(self.prefix):
+            print("found!")
             return self
+        print("not found")
 
     def load_module(self, fullname):
+        print("Calling load_module()")
         """Actually load a module that we said we'd handle in find_module."""
         module = import_module(fullname[len(self.prefix):])
         sys.modules[fullname] = module
         return module
 
     def find_distributions(self, context):
+        print("Calling find_distributions()")
         """Return an iterable of all Distribution instances capable of
         loading the metadata for packages for the indicated ``context``.
         """
@@ -203,6 +217,7 @@ class PexDistribution(Distribution):
                 template = r"{path}(-.*)?\.(dist|egg)-info/{filename}"
 
                 def __init__(self, name, prefix=MODULE_DIR):
+                    print("PexDistribution __init__()")
                     """Construct a distribution for a pex file to the metadata directory.
 
                     :param name: A module name
@@ -243,11 +258,16 @@ def _has_distribution(self):
                     yield distribution
 
     def get_code(self, fullname):
+        print("get_code()")
         module = self.load_module(fullname)
         return module.__loader__.get_code(fullname)
 
 
 def add_module_dir_to_sys_path(dirname):
+    print("Calling add_module_dir_to_sys_path with dirname", dirname)
+    print("called from")
+    #traceback.print_stack(limit=10)
+
     """Adds the given dirname to sys.path if it's nonempty."""
     if dirname:
         sys.path = sys.path[:1] + [os.path.join(sys.path[0], dirname)] + sys.path[1:]
@@ -255,6 +275,7 @@ def add_module_dir_to_sys_path(dirname):
 
 
 def pex_basepath(temp=False):
+    print("Calling pex_basepath()")
     if temp:
         import tempfile
         return tempfile.mkdtemp(dir=os.environ.get('TEMP_DIR'), prefix='pex_')
@@ -267,6 +288,7 @@ def pex_uniquedir():
 
 
 def pex_paths():
+    print("Calling pex_paths()")
     no_cache = os.environ.get('PEX_NOCACHE')
     no_cache = no_cache and no_cache.lower() == 'true'
     basepath, uniquedir = pex_basepath(no_cache), pex_uniquedir()
@@ -280,6 +302,7 @@ def explode_zip():
     This is primarily used for binary extensions which can't be imported directly from
     inside a zipfile.
     """
+    print("Calling explode_zip()")
     # Temporarily add bootstrap to sys path
     ### THIS SHOULD MAYBE COME FROM CONFIG FILE?
     sys.path = [os.path.join(sys.path[0], 'third_party/python')] + sys.path[1:]
@@ -360,6 +383,8 @@ def main():
 
     N.B. This gets redefined by pex_test_main to run tests instead.
     """
+    sys.path = sys.path[:1] + [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
+    print("sys.path =", sys.path)
     # Starts a debugging session, if defined, before running the entry point.
     start_debugger()
     # Must run this as __main__ so it executes its own __name__ == '__main__' block.
diff --git a/tools/please_pex/pex/pex_test_main.py b/tools/please_pex/pex/pex_test_main.py
index 647d9626..ad06b6c8 100644
--- a/tools/please_pex/pex/pex_test_main.py
+++ b/tools/please_pex/pex/pex_test_main.py
@@ -31,6 +31,8 @@ def _xml_file(self, fr, analysis, *args, **kvargs):
 def main():
     """Runs the tests. Returns an appropriate exit code."""
     args = [arg for arg in sys.argv[1:]]
+    sys.path = sys.path[:1] + [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
+    print("sys.path =", sys.path)
     if os.getenv('COVERAGE'):
         # It's important that we run coverage while we load the tests otherwise
         # we get no coverage for import statements etc.

From 80e4f2d1b14d6ba6bf24c4b02cd90df0c854a5e7 Mon Sep 17 00:00:00 2001
From: Sam Westmoreland <samwestmoreland@gmail.com>
Date: Fri, 8 Apr 2022 18:26:29 +0100
Subject: [PATCH 27/45] cd into srcs dir before zipping

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 84fdb289..3ce259e1 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -471,7 +471,7 @@ def pip_library(name:str, version:str, hashes:list=None, package_name:str=None,
     )
 
     # Don't include the dependency whl's into our whl. They are later picked up by py_binary anyway.
-    cmd = f'$TOOLS_JARCAT z --suffix="" --exclude_suffix=whl --include_other -i .'
+    cmd = f'cd $SRCS && $TOOLS_JARCAT z --suffix="" --exclude_suffix=whl --include_other -i .'
     if not licences:
         cmd += ' && find . -name METADATA -or -name PKG-INFO | grep -v "^./build/" | xargs grep -E "License ?:" | grep -v UNKNOWN | cat || true'
 

From 68e26158f81c8c31160d3e4fe88b28bf372b5346 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Mon, 11 Apr 2022 10:56:20 +0100
Subject: [PATCH 28/45] Revert debug changes

---
 .plzconfig                       |  4 ++--
 tools/BUILD                      | 14 --------------
 tools/please_pex/pex/pex_main.py | 22 ----------------------
 3 files changed, 2 insertions(+), 38 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index 0604bb9e..32f60491 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -13,7 +13,7 @@ ImportPath = github.com/please-build/python-rules
 gotool = //third_party/go:toolchain|go
 
 [Java]
-JarCatTool = //tools:arcat_local
+JarCatTool = //tools:arcat
 
 [PluginDefinition]
 name = python
@@ -90,7 +90,7 @@ Repeatable = true
 
 [PluginConfig "wheel_tool"]
 ConfigKey = WheelTool
-DefaultValue = //tools:wheel_resolver
+DefaultValue = //tools/wheel_resolver
 Optional = true
 Inherit = true
 
diff --git a/tools/BUILD b/tools/BUILD
index 9e2fe7bb..cd186eb8 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -5,17 +5,3 @@ remote_file(
     binary = True,
     visibility = ["PUBLIC"],
 )
-
-filegroup(
-    name = "wheel_resolver",
-    system_srcs = ["/home/swestmoreland/wheel_tool.pex",],
-    binary = True,
-    visibility = ["PUBLIC"],
-)
-
-filegroup(
-    name = "arcat_local",
-    system_srcs = ["/home/swestmoreland/git/arcat/plz-out/bin/arcat"],
-    binary = True,
-    visibility = ["PUBLIC"],
-)
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index a1bac047..7efadba9 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -5,7 +5,6 @@
 import os
 import runpy
 import sys
-import traceback
 
 
 PY_VERSION = sys.version_info
@@ -114,9 +113,6 @@ class SoImport(object):
     """So import. Much binary. Such dynamic. Wow."""
 
     def __init__(self):
-        print("Calling SoImport __init__()")
-        traceback.print_stack(limit=10)
-
         if PY_VERSION.major < 3:
             self.suffixes = {x[0]: x for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION}
         else:
@@ -179,29 +175,21 @@ class ModuleDirImport(object):
     """
 
     def __init__(self, module_dir=MODULE_DIR):
-        print("Calling ModuleDirImport __init__()", module_dir)
         self.prefix = module_dir.replace('/', '.') + '.'
-        print("Added prefix", self.prefix)
 
     def find_module(self, fullname, path=None):
-        print("Calling ModuleDirImport.find_module() on", fullname, "with path", path)
-        print("checking against", self.prefix)
         #traceback.print_stack(limit=10)
         """Attempt to locate module. Returns self if found, None if not."""
         if fullname.startswith(self.prefix):
-            print("found!")
             return self
-        print("not found")
 
     def load_module(self, fullname):
-        print("Calling load_module()")
         """Actually load a module that we said we'd handle in find_module."""
         module = import_module(fullname[len(self.prefix):])
         sys.modules[fullname] = module
         return module
 
     def find_distributions(self, context):
-        print("Calling find_distributions()")
         """Return an iterable of all Distribution instances capable of
         loading the metadata for packages for the indicated ``context``.
         """
@@ -217,7 +205,6 @@ class PexDistribution(Distribution):
                 template = r"{path}(-.*)?\.(dist|egg)-info/{filename}"
 
                 def __init__(self, name, prefix=MODULE_DIR):
-                    print("PexDistribution __init__()")
                     """Construct a distribution for a pex file to the metadata directory.
 
                     :param name: A module name
@@ -258,16 +245,11 @@ def _has_distribution(self):
                     yield distribution
 
     def get_code(self, fullname):
-        print("get_code()")
         module = self.load_module(fullname)
         return module.__loader__.get_code(fullname)
 
 
 def add_module_dir_to_sys_path(dirname):
-    print("Calling add_module_dir_to_sys_path with dirname", dirname)
-    print("called from")
-    #traceback.print_stack(limit=10)
-
     """Adds the given dirname to sys.path if it's nonempty."""
     if dirname:
         sys.path = sys.path[:1] + [os.path.join(sys.path[0], dirname)] + sys.path[1:]
@@ -275,7 +257,6 @@ def add_module_dir_to_sys_path(dirname):
 
 
 def pex_basepath(temp=False):
-    print("Calling pex_basepath()")
     if temp:
         import tempfile
         return tempfile.mkdtemp(dir=os.environ.get('TEMP_DIR'), prefix='pex_')
@@ -288,7 +269,6 @@ def pex_uniquedir():
 
 
 def pex_paths():
-    print("Calling pex_paths()")
     no_cache = os.environ.get('PEX_NOCACHE')
     no_cache = no_cache and no_cache.lower() == 'true'
     basepath, uniquedir = pex_basepath(no_cache), pex_uniquedir()
@@ -302,7 +282,6 @@ def explode_zip():
     This is primarily used for binary extensions which can't be imported directly from
     inside a zipfile.
     """
-    print("Calling explode_zip()")
     # Temporarily add bootstrap to sys path
     ### THIS SHOULD MAYBE COME FROM CONFIG FILE?
     sys.path = [os.path.join(sys.path[0], 'third_party/python')] + sys.path[1:]
@@ -384,7 +363,6 @@ def main():
     N.B. This gets redefined by pex_test_main to run tests instead.
     """
     sys.path = sys.path[:1] + [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
-    print("sys.path =", sys.path)
     # Starts a debugging session, if defined, before running the entry point.
     start_debugger()
     # Must run this as __main__ so it executes its own __name__ == '__main__' block.

From 12186251b1de40f7f5893a27ea12a7bb88207127 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Mon, 11 Apr 2022 10:59:10 +0100
Subject: [PATCH 29/45] Delete more debug

---
 build_defs/python.build_defs          | 1 -
 tools/please_pex/pex/pex_main.py      | 1 -
 tools/please_pex/pex/pex_test_main.py | 1 -
 3 files changed, 3 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 3ce259e1..5887db58 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -363,7 +363,6 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
         requires=['py', 'test', interpreter or CONFIG.PYTHON.DEFAULT_INTERPRETER],
         tools=[CONFIG.JARCAT_TOOL],
         test_tools=[CONFIG.PYTHON.DEFAULT_INTERPRETER],
-            
     )
 
 def pip_library(name:str, version:str, hashes:list=None, package_name:str=None,
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 7efadba9..067e51ca 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -178,7 +178,6 @@ def __init__(self, module_dir=MODULE_DIR):
         self.prefix = module_dir.replace('/', '.') + '.'
 
     def find_module(self, fullname, path=None):
-        #traceback.print_stack(limit=10)
         """Attempt to locate module. Returns self if found, None if not."""
         if fullname.startswith(self.prefix):
             return self
diff --git a/tools/please_pex/pex/pex_test_main.py b/tools/please_pex/pex/pex_test_main.py
index ad06b6c8..07cddd5c 100644
--- a/tools/please_pex/pex/pex_test_main.py
+++ b/tools/please_pex/pex/pex_test_main.py
@@ -32,7 +32,6 @@ def main():
     """Runs the tests. Returns an appropriate exit code."""
     args = [arg for arg in sys.argv[1:]]
     sys.path = sys.path[:1] + [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
-    print("sys.path =", sys.path)
     if os.getenv('COVERAGE'):
         # It's important that we run coverage while we load the tests otherwise
         # we get no coverage for import statements etc.

From a73e10cff8557dda8cf591ac339c61ab6ead4788 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Wed, 13 Apr 2022 17:14:03 +0100
Subject: [PATCH 30/45] Think this works now

---
 build_defs/python.build_defs     |  6 +++---
 third_party/python/BUILD         | 20 +++++++++++++++++++-
 tools/please_pex/pex/pex_main.py |  4 ++--
 tools/please_pex/pex/pex_run.py  |  9 ++++++---
 4 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 5887db58..baf287a4 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -330,8 +330,8 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
 
     #'cp -r $SRCS_TESTRUNNERDEPS .',
     cmd = ['mv $SRCS_MAIN __main__.py',
-           'mkdir .bootstrap && mv $SRCS_TESTRUNNERDEPS .bootstrap',
-           f'$TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict']
+           'mv $SRCS_TESTRUNNERDEPS .',
+           f'$TOOL z -i . -s .pex.zip -s .zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict']
 
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
     return build_rule(
@@ -470,7 +470,7 @@ def pip_library(name:str, version:str, hashes:list=None, package_name:str=None,
     )
 
     # Don't include the dependency whl's into our whl. They are later picked up by py_binary anyway.
-    cmd = f'cd $SRCS && $TOOLS_JARCAT z --suffix="" --exclude_suffix=whl --include_other -i .'
+    cmd = f'$TOOLS_JARCAT z --suffix="" --exclude_suffix=whl --include_other -i . -r $PKG/{name}:$PKG'
     if not licences:
         cmd += ' && find . -name METADATA -or -name PKG-INFO | grep -v "^./build/" | xargs grep -E "License ?:" | grep -v UNKNOWN | cat || true'
 
diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index 2f9a5f34..03dcd344 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -468,8 +468,26 @@ pip_library(
     version = "1.5.0",
 )
 
-filegroup(
+genrule(
     name = "unittest_bootstrap",
+    cmd = [
+        "mkdir .bootstrap",
+        "ls -d $PKG_DIR/* | grep -v \.whl | xargs -I{} mv {} .bootstrap",
+        "ls -d $PKG_DIR/* | xargs -I{} -n1 $TOOLS x {} -o {}.unzip",
+        "for dir in $PKG_DIR/*.unzip; do mv $dir/$PKG_DIR/* .bootstrap; done",
+        "$TOOLS z -i .bootstrap/ -o .bootstrap.zip  --include_other",
+    ],
+    srcs = [
+        ":unittest_deps"
+    ],
+    outs = [
+        ".bootstrap.zip",
+    ],
+    tools = [CONFIG.JARCAT_TOOL],
+)
+
+filegroup(
+    name = "unittest_deps",
     srcs = [
         ":coverage",
         ":portalocker",
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 067e51ca..2cc93aea 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -282,8 +282,7 @@ def explode_zip():
     inside a zipfile.
     """
     # Temporarily add bootstrap to sys path
-    ### THIS SHOULD MAYBE COME FROM CONFIG FILE?
-    sys.path = [os.path.join(sys.path[0], 'third_party/python')] + sys.path[1:]
+    sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
     import contextlib, portalocker
     sys.path = sys.path[1:]
 
@@ -361,6 +360,7 @@ def main():
 
     N.B. This gets redefined by pex_test_main to run tests instead.
     """
+    # Add .bootstrap dir to path, after the initial pex entry
     sys.path = sys.path[:1] + [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
     # Starts a debugging session, if defined, before running the entry point.
     start_debugger()
diff --git a/tools/please_pex/pex/pex_run.py b/tools/please_pex/pex/pex_run.py
index e47ca881..c81cc146 100644
--- a/tools/please_pex/pex/pex_run.py
+++ b/tools/please_pex/pex/pex_run.py
@@ -1,4 +1,4 @@
-def run():
+def run(explode=False):
     if not ZIP_SAFE:
         with explode_zip()():
             add_module_dir_to_sys_path(MODULE_DIR)
@@ -10,6 +10,9 @@ def run():
 
 
 if __name__ == '__main__':
+     # If PEX_EXPLODE is set, then it should always be exploded.
+    explode = os.environ.get('PEX_EXPLODE', '0') != '0'
+
     # If PEX_INTERPRETER is set, then it starts an interactive console.
     if os.environ.get('PEX_INTERPRETER', '0') != '0':
         import code
@@ -17,8 +20,8 @@ def run():
     # If PEX_PROFILE_FILENAME is set, then it collects profile information into the filename.
     elif os.environ.get('PEX_PROFILE_FILENAME'):
         with profile(os.environ['PEX_PROFILE_FILENAME'])():
-            result = run()
+            result = run(explode)
     else:
-        result = run()
+        result = run(explode)
 
     sys.exit(result)

From 0e89e60b2dcef7112cbf3b632bed76553e06bd1e Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Wed, 13 Apr 2022 17:50:18 +0100
Subject: [PATCH 31/45] Add genrule for pytest deps

---
 third_party/python/BUILD | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index 03dcd344..4e3435c6 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -475,13 +475,13 @@ genrule(
         "ls -d $PKG_DIR/* | grep -v \.whl | xargs -I{} mv {} .bootstrap",
         "ls -d $PKG_DIR/* | xargs -I{} -n1 $TOOLS x {} -o {}.unzip",
         "for dir in $PKG_DIR/*.unzip; do mv $dir/$PKG_DIR/* .bootstrap; done",
-        "$TOOLS z -i .bootstrap/ -o .bootstrap.zip  --include_other",
+        "$TOOLS z -i .bootstrap/ -o .unittest_bootstrap.zip  --include_other",
     ],
     srcs = [
         ":unittest_deps"
     ],
     outs = [
-        ".bootstrap.zip",
+        ".unittest_bootstrap.zip",
     ],
     tools = [CONFIG.JARCAT_TOOL],
 )
@@ -496,8 +496,26 @@ filegroup(
     ],
 )
 
-filegroup(
+genrule(
     name = "pytest_bootstrap",
+    cmd = [
+        "mkdir .bootstrap",
+        "ls -d $PKG_DIR/* | grep -v \.whl | xargs -I{} mv {} .bootstrap;"
+        "ls -d $PKG_DIR/* | xargs -I{} -n1 $TOOLS x {} -o {}.unzip;"
+        "for dir in $PKG_DIR/*.unzip; do mv $dir/$PKG_DIR/* .bootstrap; done",
+        "$TOOLS z -i .bootstrap/ -o .pytest_bootstrap.zip  --include_other",
+    ],
+    srcs = [
+        ":pytest_deps"
+    ],
+    outs = [
+        ".pytest_bootstrap.zip",
+    ],
+    tools = [CONFIG.JARCAT_TOOL],
+)
+
+filegroup(
+    name = "pytest_deps",
     srcs = [
         ":attrs",
         ":funcsigs",

From 7f39e9b338bc774e40a27f6dd10353690e33a9b2 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 10:55:26 +0100
Subject: [PATCH 32/45] Add patch to importlib_metadata target

---
 .plzconfig                       | 2 +-
 third_party/python/BUILD         | 8 +++++++-
 tools/BUILD                      | 7 +++++++
 tools/please_pex/pex/pex_main.py | 1 +
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index 32f60491..c1c6a64c 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -90,7 +90,7 @@ Repeatable = true
 
 [PluginConfig "wheel_tool"]
 ConfigKey = WheelTool
-DefaultValue = //tools/wheel_resolver
+DefaultValue = //tools:wheel_resolver
 Optional = true
 Inherit = true
 
diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index 4e3435c6..d5c0770b 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -175,8 +175,14 @@ pip_library(
     deps = [":importlib_metadata"],
 )
 
-pip_library(
+python_wheel(
     name = "importlib_metadata",
+    outs = [
+        "importlib_metadata",
+        "importlib_metadata-1.5.0.dist-info",
+    ],
+    hashes = ["b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"],
+    patch = "importlib_metadata.patch",
     version = "1.5.0",
     deps = [":zipp"],
 )
diff --git a/tools/BUILD b/tools/BUILD
index cd186eb8..25a44892 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -5,3 +5,10 @@ remote_file(
     binary = True,
     visibility = ["PUBLIC"],
 )
+
+filegroup(
+    name = "wheel_resolver",
+    system_srcs = ["/home/swestmoreland/wheel_tool.pex"],
+    binary = True,
+    visibility = ["PUBLIC"],
+)
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 2cc93aea..e0c38e2a 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -113,6 +113,7 @@ class SoImport(object):
     """So import. Much binary. Such dynamic. Wow."""
 
     def __init__(self):
+
         if PY_VERSION.major < 3:
             self.suffixes = {x[0]: x for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION}
         else:

From a9d8a02a4afee485edbd804a34872e2bad6226ed Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 10:55:57 +0100
Subject: [PATCH 33/45] Revert "Add patch to importlib_metadata target"

This reverts commit 7f39e9b338bc774e40a27f6dd10353690e33a9b2.
---
 .plzconfig                       | 2 +-
 third_party/python/BUILD         | 8 +-------
 tools/BUILD                      | 7 -------
 tools/please_pex/pex/pex_main.py | 1 -
 4 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index c1c6a64c..32f60491 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -90,7 +90,7 @@ Repeatable = true
 
 [PluginConfig "wheel_tool"]
 ConfigKey = WheelTool
-DefaultValue = //tools:wheel_resolver
+DefaultValue = //tools/wheel_resolver
 Optional = true
 Inherit = true
 
diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index d5c0770b..4e3435c6 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -175,14 +175,8 @@ pip_library(
     deps = [":importlib_metadata"],
 )
 
-python_wheel(
+pip_library(
     name = "importlib_metadata",
-    outs = [
-        "importlib_metadata",
-        "importlib_metadata-1.5.0.dist-info",
-    ],
-    hashes = ["b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"],
-    patch = "importlib_metadata.patch",
     version = "1.5.0",
     deps = [":zipp"],
 )
diff --git a/tools/BUILD b/tools/BUILD
index 25a44892..cd186eb8 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -5,10 +5,3 @@ remote_file(
     binary = True,
     visibility = ["PUBLIC"],
 )
-
-filegroup(
-    name = "wheel_resolver",
-    system_srcs = ["/home/swestmoreland/wheel_tool.pex"],
-    binary = True,
-    visibility = ["PUBLIC"],
-)
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index e0c38e2a..2cc93aea 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -113,7 +113,6 @@ class SoImport(object):
     """So import. Much binary. Such dynamic. Wow."""
 
     def __init__(self):
-
         if PY_VERSION.major < 3:
             self.suffixes = {x[0]: x for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION}
         else:

From 0367aa435b9450c329965496d9656ccb716fe675 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 10:56:57 +0100
Subject: [PATCH 34/45] Add patch for importlib_metadata target

---
 third_party/python/BUILD | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index 4e3435c6..d5c0770b 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -175,8 +175,14 @@ pip_library(
     deps = [":importlib_metadata"],
 )
 
-pip_library(
+python_wheel(
     name = "importlib_metadata",
+    outs = [
+        "importlib_metadata",
+        "importlib_metadata-1.5.0.dist-info",
+    ],
+    hashes = ["b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"],
+    patch = "importlib_metadata.patch",
     version = "1.5.0",
     deps = [":zipp"],
 )

From 426397e593362eb56936d0ec477140d0a10d701a Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 12:06:35 +0100
Subject: [PATCH 35/45] Try adding full path

---
 tools/please_pex/pex/pex_main.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 2cc93aea..766ac5df 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -283,7 +283,7 @@ def explode_zip():
     """
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
-    import contextlib, portalocker
+    import contextlib, .bootstrap.portalocker
     sys.path = sys.path[1:]
 
     @contextlib.contextmanager
@@ -292,10 +292,10 @@ def pex_lockfile(basepath, uniquedir):
         lockfile_path = os.path.join(basepath, '.lock-%s' % uniquedir)
         lockfile = open(lockfile_path, "a+")
         # Block until we can acquire the lockfile.
-        portalocker.lock(lockfile, portalocker.LOCK_EX)
+        portalocker.lock(lockfile, .bootstrap.portalocker.LOCK_EX)
         lockfile.seek(0)
         yield lockfile
-        portalocker.lock(lockfile, portalocker.LOCK_UN)
+        portalocker.lock(lockfile, .bootstrap.portalocker.LOCK_UN)
 
     @contextlib.contextmanager
     def _explode_zip():

From a3a0a90947a24b06b8611f8353980cd6c77a71e5 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 12:08:36 +0100
Subject: [PATCH 36/45] Try removing this path line

---
 tools/please_pex/pex/pex_main.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 766ac5df..8e1fe966 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -283,8 +283,8 @@ def explode_zip():
     """
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
-    import contextlib, .bootstrap.portalocker
-    sys.path = sys.path[1:]
+    import contextlib, portalocker
+    # sys.path = sys.path[1:]
 
     @contextlib.contextmanager
     def pex_lockfile(basepath, uniquedir):
@@ -292,10 +292,10 @@ def pex_lockfile(basepath, uniquedir):
         lockfile_path = os.path.join(basepath, '.lock-%s' % uniquedir)
         lockfile = open(lockfile_path, "a+")
         # Block until we can acquire the lockfile.
-        portalocker.lock(lockfile, .bootstrap.portalocker.LOCK_EX)
+        portalocker.lock(lockfile, portalocker.LOCK_EX)
         lockfile.seek(0)
         yield lockfile
-        portalocker.lock(lockfile, .bootstrap.portalocker.LOCK_UN)
+        portalocker.lock(lockfile, portalocker.LOCK_UN)
 
     @contextlib.contextmanager
     def _explode_zip():

From b0655c6decf8dc2b7ebf757fd44f59aac551a6c7 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 12:10:12 +0100
Subject: [PATCH 37/45] Print some debug

---
 tools/please_pex/pex/pex_main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 8e1fe966..e3e48ae6 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -283,8 +283,9 @@ def explode_zip():
     """
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
+    print("sys.path =", sys.path)
     import contextlib, portalocker
-    # sys.path = sys.path[1:]
+    sys.path = sys.path[1:]
 
     @contextlib.contextmanager
     def pex_lockfile(basepath, uniquedir):

From fbd16705d88eb6c6b0107a5a232309cc069fe3b9 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 12:19:02 +0100
Subject: [PATCH 38/45] Try this

---
 third_party/python/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index d5c0770b..2bfe59e0 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -299,8 +299,9 @@ python_wheel(
     deps = [":six"],
 )
 
-pip_library(
+python_wheel(
     name = "portalocker",
+    hashes = ["874d6063c6ceb185fe4771da41b01872d2c56d292db746698f8ad7bf1833c905"],
     version = "1.7.0",
 )
 

From fe5da59ed36700988cc9cb845ae4c9cc1560d317 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 15:47:11 +0100
Subject: [PATCH 39/45] Add portalocker to bootstrap for python_binaries

---
 .plzconfig                       | 4 ++++
 build_defs/python.build_defs     | 9 +++++++--
 third_party/python/BUILD         | 3 +--
 tools/please_pex/pex/pex_main.py | 1 +
 tools/please_pex/pex/pex_run.py  | 2 +-
 5 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index 32f60491..4616f42a 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -100,6 +100,10 @@ DefaultValue = "#!/usr/bin/python3"
 Optional = true
 Inherit = true
 
+[PluginConfig "portalocker"]
+ConfigKey = Portalocker
+DefaultValue = //third_party/python:portalocker
+
 [featureflags]
 PythonWheelHashing = true
 ExcludePythonRules = true
diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index baf287a4..d7f77dbf 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -175,15 +175,20 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
     )
 
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
-    cmd = f'mv $SRC __main__.py && $TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict'
+    cmd = f'mv $SRCS_MAIN __main__.py && mkdir .bootstrap && mv $SRCS_BOOTSTRAP .bootstrap && $TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict'
     if strip:
         cmd += ' --strip_py'
 
     debug_cmd = _debug_cmd("./$OUT")
 
+    srcs = {
+        'main': [pex_rule],
+        'bootstrap': [CONFIG.PYTHON.PORTALOCKER],
+        }
+
     return build_rule(
         name=name,
-        srcs=[pex_rule],
+        srcs=srcs,
         deps=[lib_rule],
         outs=[out or (name + '.pex')],
         data=data,
diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index 2bfe59e0..d5c0770b 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -299,9 +299,8 @@ python_wheel(
     deps = [":six"],
 )
 
-python_wheel(
+pip_library(
     name = "portalocker",
-    hashes = ["874d6063c6ceb185fe4771da41b01872d2c56d292db746698f8ad7bf1833c905"],
     version = "1.7.0",
 )
 
diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index e3e48ae6..7a7e989e 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -184,6 +184,7 @@ def find_module(self, fullname, path=None):
 
     def load_module(self, fullname):
         """Actually load a module that we said we'd handle in find_module."""
+        print("load_module", fullname, "as", fullname[len(self.prefix):])
         module = import_module(fullname[len(self.prefix):])
         sys.modules[fullname] = module
         return module
diff --git a/tools/please_pex/pex/pex_run.py b/tools/please_pex/pex/pex_run.py
index c81cc146..afd23543 100644
--- a/tools/please_pex/pex/pex_run.py
+++ b/tools/please_pex/pex/pex_run.py
@@ -1,5 +1,5 @@
 def run(explode=False):
-    if not ZIP_SAFE:
+    if explode or not ZIP_SAFE:
         with explode_zip()():
             add_module_dir_to_sys_path(MODULE_DIR)
             return main()

From c18c940230208a8cb368dc7efedfd685fc9e8b6e Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 16:53:18 +0100
Subject: [PATCH 40/45] Genrule to output bootstrap

---
 .plzconfig                   |  2 +-
 build_defs/python.build_defs |  3 ++-
 third_party/python/BUILD     | 22 ++++++++++++++++++++--
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/.plzconfig b/.plzconfig
index 4616f42a..dc2390f7 100644
--- a/.plzconfig
+++ b/.plzconfig
@@ -102,7 +102,7 @@ Inherit = true
 
 [PluginConfig "portalocker"]
 ConfigKey = Portalocker
-DefaultValue = //third_party/python:portalocker
+DefaultValue = //third_party/python:py_binary_bootstrap
 
 [featureflags]
 PythonWheelHashing = true
diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index d7f77dbf..1c034f19 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -175,7 +175,8 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
     )
 
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
-    cmd = f'mv $SRCS_MAIN __main__.py && mkdir .bootstrap && mv $SRCS_BOOTSTRAP .bootstrap && $TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict'
+    cmd = f'mv $SRCS_MAIN __main__.py && mv $SRCS_BOOTSTRAP . && $TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict'
+
     if strip:
         cmd += ' --strip_py'
 
diff --git a/third_party/python/BUILD b/third_party/python/BUILD
index d5c0770b..622df2e9 100644
--- a/third_party/python/BUILD
+++ b/third_party/python/BUILD
@@ -304,6 +304,24 @@ pip_library(
     version = "1.7.0",
 )
 
+genrule(
+    name = "py_binary_bootstrap",
+    cmd = [
+        "mkdir .bootstrap",
+        "ls -d $PKG_DIR/* | grep -v \.whl | xargs -I{} mv {} .bootstrap;"
+        "ls -d $PKG_DIR/* | xargs -I{} -n1 $TOOLS x {} -o {}.unzip;"
+        "for dir in $PKG_DIR/*.unzip; do mv $dir/$PKG_DIR/* .bootstrap; done",
+        "$TOOLS z -i .bootstrap/ -o .py_binary_bootstrap.zip  --include_other",
+    ],
+    srcs = [
+        ":portalocker"
+    ],
+    outs = [
+        ".py_binary_bootstrap.zip",
+    ],
+    tools = [CONFIG.JARCAT_TOOL],
+)
+
 pip_library(
     name = "numpy",
     test_only = True,
@@ -478,8 +496,8 @@ genrule(
     name = "unittest_bootstrap",
     cmd = [
         "mkdir .bootstrap",
-        "ls -d $PKG_DIR/* | grep -v \.whl | xargs -I{} mv {} .bootstrap",
-        "ls -d $PKG_DIR/* | xargs -I{} -n1 $TOOLS x {} -o {}.unzip",
+        "ls -d $PKG_DIR/* | grep -v \.whl | xargs -I{} mv {} .bootstrap;"
+        "ls -d $PKG_DIR/* | xargs -I{} -n1 $TOOLS x {} -o {}.unzip;"
         "for dir in $PKG_DIR/*.unzip; do mv $dir/$PKG_DIR/* .bootstrap; done",
         "$TOOLS z -i .bootstrap/ -o .unittest_bootstrap.zip  --include_other",
     ],

From e1a49754b2117b611ce6fb3d744d1ef9b940e231 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 17:00:34 +0100
Subject: [PATCH 41/45] Include .zip

---
 build_defs/python.build_defs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs
index 1c034f19..2a36ee49 100644
--- a/build_defs/python.build_defs
+++ b/build_defs/python.build_defs
@@ -175,7 +175,7 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
     )
 
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
-    cmd = f'mv $SRCS_MAIN __main__.py && mv $SRCS_BOOTSTRAP . && $TOOL z -i . -s .pex.zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict'
+    cmd = f'mv $SRCS_MAIN __main__.py && mv $SRCS_BOOTSTRAP . && $TOOL z -i . -s .zip -s .whl --preamble="{shebang}" --include_other --add_init_py --strict'
 
     if strip:
         cmd += ' --strip_py'

From 63da81219b437d269033ec59e283b2517f73fdd7 Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 17:15:43 +0100
Subject: [PATCH 42/45] Remove debug

---
 tools/please_pex/pex/pex_main.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 7a7e989e..2cc93aea 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -184,7 +184,6 @@ def find_module(self, fullname, path=None):
 
     def load_module(self, fullname):
         """Actually load a module that we said we'd handle in find_module."""
-        print("load_module", fullname, "as", fullname[len(self.prefix):])
         module = import_module(fullname[len(self.prefix):])
         sys.modules[fullname] = module
         return module
@@ -284,7 +283,6 @@ def explode_zip():
     """
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
-    print("sys.path =", sys.path)
     import contextlib, portalocker
     sys.path = sys.path[1:]
 

From f39111830f795aed30bea1f53691468710e8394c Mon Sep 17 00:00:00 2001
From: swestmoreland <swestmoreland@thoughtmachine.net>
Date: Tue, 19 Apr 2022 17:35:08 +0100
Subject: [PATCH 43/45] Add debug again...

---
 tools/please_pex/pex/pex_main.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 2cc93aea..e3e48ae6 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -283,6 +283,7 @@ def explode_zip():
     """
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
+    print("sys.path =", sys.path)
     import contextlib, portalocker
     sys.path = sys.path[1:]
 

From 05d81c89d6f05c4e2704001dc040013c616ae467 Mon Sep 17 00:00:00 2001
From: Sam Westmoreland <samwestmoreland@gmail.com>
Date: Mon, 25 Apr 2022 21:30:02 +0100
Subject: [PATCH 44/45] Try different path for portalocker

---
 tools/please_pex/pex/pex_main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index e3e48ae6..12d76b12 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -284,7 +284,7 @@ def explode_zip():
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
     print("sys.path =", sys.path)
-    import contextlib, portalocker
+    import contextlib, portalocker.portalocker
     sys.path = sys.path[1:]
 
     @contextlib.contextmanager

From 3a939221f23c538c699676a03d8c2f84665794c6 Mon Sep 17 00:00:00 2001
From: Sam Westmoreland <samwestmoreland@gmail.com>
Date: Mon, 25 Apr 2022 21:54:16 +0100
Subject: [PATCH 45/45] Old path was fine

---
 tools/please_pex/pex/pex_main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/please_pex/pex/pex_main.py b/tools/please_pex/pex/pex_main.py
index 12d76b12..e3e48ae6 100644
--- a/tools/please_pex/pex/pex_main.py
+++ b/tools/please_pex/pex/pex_main.py
@@ -284,7 +284,7 @@ def explode_zip():
     # Temporarily add bootstrap to sys path
     sys.path = [os.path.join(sys.path[0], '.bootstrap')] + sys.path[1:]
     print("sys.path =", sys.path)
-    import contextlib, portalocker.portalocker
+    import contextlib, portalocker
     sys.path = sys.path[1:]
 
     @contextlib.contextmanager