From d936b1e15e465b3ad5f7c5d449875c7ca5cc0b01 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 03:16:59 +0000 Subject: [PATCH 1/5] Add deterministic prefix in Library.get_flags() (NFC) This factors duplicate routines to add deterministic prefix paths out to `Library.get_cflags()`. Suggested in https://github.com/emscripten-core/emscripten/pull/23222#discussion_r1891063416. --- tools/system_libs.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 3b01a85fbf651..73b9df9a44542 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -471,14 +471,9 @@ def get_files(self): def generate_ninja(self, build_dir, libname): ensure_sysroot() utils.safe_ensure_dirs(build_dir) + self.batch_inputs = True cflags = self.get_cflags() - if self.deterministic_paths: - source_dir = utils.path_from_root() - relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', - f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}', - f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] asflags = get_base_cflags(preprocess=False) input_files = self.get_files() ninja_file = os.path.join(build_dir, 'build.ninja') @@ -491,18 +486,11 @@ def build_objects(self, build_dir): By default, this builds all the source files returned by `self.get_files()`, with the `cflags` returned by `self.get_cflags()`. """ - batch_inputs = int(os.environ.get('EMCC_BATCH_BUILD', '1')) + self.batch_inputs = int(os.environ.get('EMCC_BATCH_BUILD', '1')) batches = {} commands = [] objects = set() cflags = self.get_cflags() - if self.deterministic_paths: - source_dir = utils.path_from_root() - if batch_inputs: - relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}'] - cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', - f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] case_insensitive = is_case_insensitive(build_dir) for src in self.get_files(): ext = shared.suffix(src) @@ -537,7 +525,7 @@ def build_objects(self, build_dir): object_uuid += 1 o = os.path.join(build_dir, f'{object_basename}__{object_uuid}.o') commands.append(cmd + [src, '-o', o]) - elif batch_inputs: + elif self.batch_inputs: # Use relative paths to reduce the length of the command line. # This allows to avoid switching to a response file as often. src = os.path.relpath(src, build_dir) @@ -547,7 +535,7 @@ def build_objects(self, build_dir): commands.append(cmd + [src, '-o', o]) objects.add(o) - if batch_inputs: + if self.batch_inputs: # Choose a chunk size that is large enough to avoid too many subprocesses # but not too large to avoid task starvation. # For now the heuristic is to split inputs by 2x number of cores. @@ -611,6 +599,13 @@ def get_cflags(self): if self.includes: cflags += ['-I' + utils.path_from_root(i) for i in self._inherit_list('includes')] + if self.deterministic_paths: + source_dir = utils.path_from_root() + if self.batch_inputs: + relative_source_dir = os.path.relpath(source_dir, build_dir) + cflags += [f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}'] + cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', + f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] return cflags def get_base_name_prefix(self): From 8a16f48f26c73c5baa8bdf0d76602301fe763d97 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 04:48:32 +0000 Subject: [PATCH 2/5] Add self.build_dir --- tools/system_libs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 73b9df9a44542..19bd7b4a6e099 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -472,6 +472,7 @@ def generate_ninja(self, build_dir, libname): ensure_sysroot() utils.safe_ensure_dirs(build_dir) self.batch_inputs = True + self.build_dir = build_dir cflags = self.get_cflags() asflags = get_base_cflags(preprocess=False) @@ -487,6 +488,7 @@ def build_objects(self, build_dir): with the `cflags` returned by `self.get_cflags()`. """ self.batch_inputs = int(os.environ.get('EMCC_BATCH_BUILD', '1')) + self.build_dir = build_dir batches = {} commands = [] objects = set() @@ -602,7 +604,7 @@ def get_cflags(self): if self.deterministic_paths: source_dir = utils.path_from_root() if self.batch_inputs: - relative_source_dir = os.path.relpath(source_dir, build_dir) + relative_source_dir = os.path.relpath(source_dir, self.build_dir) cflags += [f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}'] cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] From cfff15bf91d99185ad85e27299bfc7c5070f04d3 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 04:49:07 +0000 Subject: [PATCH 3/5] Add missing self --- tools/system_libs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 19bd7b4a6e099..e47643a80719d 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -530,7 +530,7 @@ def build_objects(self, build_dir): elif self.batch_inputs: # Use relative paths to reduce the length of the command line. # This allows to avoid switching to a response file as often. - src = os.path.relpath(src, build_dir) + src = os.path.relpath(src, self.build_dir) src = utils.normalize_path(src) batches.setdefault(tuple(cmd), []).append(src) else: From ff18a897ece7a5a552520e0f5b7bb0da33d2a3fd Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 05:08:30 +0000 Subject: [PATCH 4/5] Always replace relative_source_dir --- tools/system_libs.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index e47643a80719d..6ef19ea06c656 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -471,7 +471,6 @@ def get_files(self): def generate_ninja(self, build_dir, libname): ensure_sysroot() utils.safe_ensure_dirs(build_dir) - self.batch_inputs = True self.build_dir = build_dir cflags = self.get_cflags() @@ -487,7 +486,7 @@ def build_objects(self, build_dir): By default, this builds all the source files returned by `self.get_files()`, with the `cflags` returned by `self.get_cflags()`. """ - self.batch_inputs = int(os.environ.get('EMCC_BATCH_BUILD', '1')) + batch_inputs = int(os.environ.get('EMCC_BATCH_BUILD', '1')) self.build_dir = build_dir batches = {} commands = [] @@ -527,7 +526,7 @@ def build_objects(self, build_dir): object_uuid += 1 o = os.path.join(build_dir, f'{object_basename}__{object_uuid}.o') commands.append(cmd + [src, '-o', o]) - elif self.batch_inputs: + elif batch_inputs: # Use relative paths to reduce the length of the command line. # This allows to avoid switching to a response file as often. src = os.path.relpath(src, self.build_dir) @@ -537,7 +536,7 @@ def build_objects(self, build_dir): commands.append(cmd + [src, '-o', o]) objects.add(o) - if self.batch_inputs: + if batch_inputs: # Choose a chunk size that is large enough to avoid too many subprocesses # but not too large to avoid task starvation. # For now the heuristic is to split inputs by 2x number of cores. @@ -603,9 +602,8 @@ def get_cflags(self): if self.deterministic_paths: source_dir = utils.path_from_root() - if self.batch_inputs: - relative_source_dir = os.path.relpath(source_dir, self.build_dir) - cflags += [f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}'] + relative_source_dir = os.path.relpath(source_dir, self.build_dir) + cflags += [f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}'] cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}', f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}'] return cflags From eb034d797a99919cd90e528f8d6e494674109be5 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 19 Dec 2024 05:09:37 +0000 Subject: [PATCH 5/5] Remove unnecessary self --- tools/system_libs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 6ef19ea06c656..ef896f6ec96ac 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -529,7 +529,7 @@ def build_objects(self, build_dir): elif batch_inputs: # Use relative paths to reduce the length of the command line. # This allows to avoid switching to a response file as often. - src = os.path.relpath(src, self.build_dir) + src = os.path.relpath(src, build_dir) src = utils.normalize_path(src) batches.setdefault(tuple(cmd), []).append(src) else: