From 86365fd160970c02d4e54014d267b7e752a0e758 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Fri, 11 Dec 2020 10:49:56 -0800 Subject: [PATCH 1/3] Make symlinks so that RBE will work for Xcode toolchains --- build/config/ios/ios_sdk.gni | 22 ++++++++++-- build/config/ios/ios_sdk.py | 56 ++++++++++++++++++++++------- build/config/mac/mac_sdk.gni | 36 ++++++++++--------- build/config/mac/xcode_toolchain.py | 33 ++++++++++++++--- build/config/sysroot.gni | 11 +++++- build/mac/find_sdk.py | 56 +++++++++++++++++++---------- 6 files changed, 159 insertions(+), 55 deletions(-) diff --git a/build/config/ios/ios_sdk.gni b/build/config/ios/ios_sdk.gni index a2bc330399..bf92a51da4 100644 --- a/build/config/ios/ios_sdk.gni +++ b/build/config/ios/ios_sdk.gni @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/toolchain/goma.gni") + declare_args() { # SDK path to use. When empty this will use the default SDK based on the # value of use_ios_simulator. @@ -27,15 +29,29 @@ declare_args() { } if (ios_sdk_path == "") { + ios_sdk_args = [] + if (use_goma) { + ios_sdk_args += [ + "--symlink", + rebase_path(root_gen_dir), + ] + } if (!use_ios_simulator && ios_device_sdk_path == "") { + ios_sdk_args += [ + "--sdk", + "iphoneos", + ] _ios_device_sdk_result = - exec_script("ios_sdk.py", [ "iphoneos" ], "list lines") + exec_script("ios_sdk.py", ios_sdk_args, "list lines") ios_device_sdk_path = _ios_device_sdk_result[0] } if (use_ios_simulator && ios_simulator_sdk_path == "") { - _ios_sim_sdk_result = - exec_script("ios_sdk.py", [ "iphonesimulator" ], "list lines") + ios_sdk_args += [ + "--sdk", + "iphonesimulator", + ] + _ios_sim_sdk_result = exec_script("ios_sdk.py", ios_sdk_args, "list lines") ios_simulator_sdk_path = _ios_sim_sdk_result[0] } diff --git a/build/config/ios/ios_sdk.py b/build/config/ios/ios_sdk.py index d1fb50e973..d88dc299ce 100644 --- a/build/config/ios/ios_sdk.py +++ b/build/config/ios/ios_sdk.py @@ -2,22 +2,54 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import argparse +import errno +import os import subprocess import sys # This script returns the path to the SDK of the given type. Pass the type of -# SDK you want, which is typically "iphone" or "iphonesimulator". +# SDK you want, which is typically 'iphone' or 'iphonesimulator'. -if len(sys.argv) != 2: - print "Takes one arg (SDK to find)" - sys.exit(1) +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise -command = [ - 'xcodebuild', - '-version', - '-sdk', - sys.argv[1], - 'Path' -] -print subprocess.check_output(command).strip() +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('--symlink', + help='Whether to create a symlink in the buildroot to the SDK.') + parser.add_argument('--sdk', + choices=['iphone', 'iphonesimulator'], + help='Which SDK to find.') + args = parser.parse_args() + + command = [ + 'xcodebuild', + '-version', + '-sdk', + args.sdk, + 'Path' + ] + + sdk_output = subprocess.check_output(command).strip() + if args.symlink: + mkdir_p(args.symlink) + symlink_target = os.path.join(args.symlink, os.path.basename(sdk_output)) + if not os.path.exists(symlink_target): + os.symlink(sdk_output, symlink_target) + sdk_output = symlink_target + + print(sdk_output) + return 0 + +if __name__ == '__main__': + if sys.platform != 'darwin': + raise Exception('This script only runs on Mac') + sys.exit(main(sys.argv)) diff --git a/build/config/mac/mac_sdk.gni b/build/config/mac/mac_sdk.gni index 715abd122c..3564add9a3 100644 --- a/build/config/mac/mac_sdk.gni +++ b/build/config/mac/mac_sdk.gni @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/toolchain/goma.gni") + declare_args() { # Minimum supported version of the Mac SDK. mac_sdk_min = "10.12" @@ -16,24 +18,24 @@ declare_args() { mac_sdk_path = "" } -find_sdk_args = [ - "--print_sdk_path", - mac_sdk_min, -] - -# The tool will print the SDK path on the first line, and the version on the -# second line. -find_sdk_lines = - exec_script("//build/mac/find_sdk.py", find_sdk_args, "list lines") +if (mac_sdk_path == "") { + find_sdk_args = [] + if (use_goma) { + # RBE has a restriction that paths cannot come from outside the build root. + find_sdk_args += [ + "--symlink", + rebase_path(root_gen_dir), + ] + } + find_sdk_args += [ + "--print_sdk_path", + mac_sdk_min, + ] -mac_sdk_version = find_sdk_lines[1] + # The tool will print the SDK path on the first line, and the version on the + # second line. + find_sdk_lines = + exec_script("//build/mac/find_sdk.py", find_sdk_args, "list lines") -if (mac_sdk_path == "") { - # TODO(brettw) http://crbug.com/335325 when everybody moves to XCode 5 we - # can remove the --print_sdk_path argument to find_sdk and instead just use - # the following two lines to get the path. Although it looks longer here, it - # saves forking a process in find_sdk.py so will be faster. - #mac_sdk_root = "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX" - #mac_sdk_path = mac_sdk_root + mac_sdk_version + ".sdk" mac_sdk_path = find_sdk_lines[0] } diff --git a/build/config/mac/xcode_toolchain.py b/build/config/mac/xcode_toolchain.py index 9b25b29953..2089519f7c 100755 --- a/build/config/mac/xcode_toolchain.py +++ b/build/config/mac/xcode_toolchain.py @@ -3,15 +3,40 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import argparse +import errno import sys import os import subprocess -def Main(): - path = subprocess.check_output(['/usr/bin/env', 'xcode-select', '-p']).strip(); +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('--symlink', + help='Whether to create a symlink in the buildroot to the SDK.') + args = parser.parse_args() + + path = subprocess.check_output(['/usr/bin/env', 'xcode-select', '-p']).strip() path = os.path.join(path, "Toolchains", "XcodeDefault.xctoolchain") assert os.path.exists(path) - print path + + if args.symlink: + mkdir_p(args.symlink) + symlink_target = os.path.join(args.symlink, os.path.basename(path)) + if not os.path.exists(symlink_target): + os.symlink(path, symlink_target) + path = symlink_target + + print(path) if __name__ == '__main__': - sys.exit(Main()) + sys.exit(main(sys.argv)) diff --git a/build/config/sysroot.gni b/build/config/sysroot.gni index 4f32e8f63b..192b7b18dd 100644 --- a/build/config/sysroot.gni +++ b/build/config/sysroot.gni @@ -5,6 +5,8 @@ # This header file defines the "sysroot" variable which is the absolute path # of the sysroot. If no sysroot applies, the variable will be an empty string. +import("//build/toolchain/goma.gni") + declare_args() { # The absolute path of the sysroot that is applied when compiling using # the target toolchain. @@ -50,8 +52,15 @@ if (current_toolchain == default_toolchain && target_sysroot != "") { } if ((is_mac || is_ios) && xcode_toolchain == "") { + xcode_toolchain_args = [] + if (use_goma) { + xcode_toolchain_args += [ + "--symlink", + rebase_path(root_gen_dir), + ] + } xcode_toolchain = exec_script("//build/config/mac/xcode_toolchain.py", - [], + xcode_toolchain_args, "trim string", []) } diff --git a/build/mac/find_sdk.py b/build/mac/find_sdk.py index 1b229060e9..1a91aaa6c4 100755 --- a/build/mac/find_sdk.py +++ b/build/mac/find_sdk.py @@ -10,15 +10,25 @@ python find_sdk.py 10.6 # Ignores SDKs < 10.6 """ +import errno import os import re import subprocess import sys - from optparse import OptionParser +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + def parse_version(version_str): """'10.6' => [10, 6]""" return map(int, re.findall(r'(\d+)', version_str)) @@ -35,6 +45,9 @@ def main(): parser.add_option("--print_sdk_path", action="store_true", dest="print_sdk_path", default=False, help="Additionaly print the path the SDK (appears first).") + parser.add_option("--symlink", + action="store", type="string", dest="symlink", default="", + help="Whether to create a symlink in the buildroot to the SDK.") (options, args) = parser.parse_args() min_sdk_version = args[0] @@ -43,8 +56,7 @@ def main(): stderr=subprocess.STDOUT) out, err = job.communicate() if job.returncode != 0: - print >> sys.stderr, out - print >> sys.stderr, err + sys.stderr.writelines([out, err]) raise Exception(('Error %d running xcode-select, you might have to run ' '|sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer| ' 'if you are using Xcode 4.') % job.returncode) @@ -64,22 +76,30 @@ def main(): best_sdk = sorted(sdks, key=parse_version)[0] if options.verify and best_sdk != min_sdk_version and not options.sdk_path: - print >> sys.stderr, '' - print >> sys.stderr, ' vvvvvvv' - print >> sys.stderr, '' - print >> sys.stderr, \ - 'This build requires the %s SDK, but it was not found on your system.' \ - % min_sdk_version - print >> sys.stderr, \ - 'Either install it, or explicitly set mac_sdk in your GYP_DEFINES.' - print >> sys.stderr, '' - print >> sys.stderr, ' ^^^^^^^' - print >> sys.stderr, '' + sys.stderr.writelines([ + '', + ' vvvvvvv', + '', + 'This build requires the %s SDK, but it was not found on your system.' \ + % min_sdk_version, + 'Either install it, or explicitly set mac_sdk in your gn args.', + '', + ' ^^^^^^^', + '']) return min_sdk_version - if options.print_sdk_path: - print subprocess.check_output(['xcodebuild', '-version', '-sdk', - 'macosx' + best_sdk, 'Path']).strip() + if options.symlink or options.print_sdk_path: + sdk_output = subprocess.check_output(['xcodebuild', '-version', '-sdk', + 'macosx' + best_sdk, 'Path']).strip() + if options.symlink: + mkdir_p(options.symlink) + symlink_target = os.path.join(options.symlink, os.path.basename(sdk_output)) + if not os.path.exists(symlink_target): + os.symlink(sdk_output, symlink_target) + sdk_output = symlink_target + + if options.print_sdk_path: + print(sdk_output) return best_sdk @@ -87,4 +107,4 @@ def main(): if __name__ == '__main__': if sys.platform != 'darwin': raise Exception("This script only runs on Mac") - print main() + print(main()) From df4fae5051ee8b02bde64a650f9189fabeb10ef2 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Fri, 11 Dec 2020 12:16:52 -0800 Subject: [PATCH 2/3] refactor and make it overwrite --- build/config/ios/ios_sdk.py | 17 ++++------------- build/config/mac/xcode_toolchain.py | 15 +++------------ build/mac/find_sdk.py | 16 +++------------- build/pyutil/__init__.py | 3 +++ build/pyutil/file_util.py | 26 ++++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 38 deletions(-) create mode 100644 build/pyutil/__init__.py create mode 100644 build/pyutil/file_util.py diff --git a/build/config/ios/ios_sdk.py b/build/config/ios/ios_sdk.py index d88dc299ce..13dc9a7758 100644 --- a/build/config/ios/ios_sdk.py +++ b/build/config/ios/ios_sdk.py @@ -8,19 +8,12 @@ import subprocess import sys +sys.path.insert(1, '../../build') +from pyutil.file_util import symlink + # This script returns the path to the SDK of the given type. Pass the type of # SDK you want, which is typically 'iphone' or 'iphonesimulator'. -def mkdir_p(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--symlink', @@ -40,10 +33,8 @@ def main(argv): sdk_output = subprocess.check_output(command).strip() if args.symlink: - mkdir_p(args.symlink) symlink_target = os.path.join(args.symlink, os.path.basename(sdk_output)) - if not os.path.exists(symlink_target): - os.symlink(sdk_output, symlink_target) + symlink(sdk_output, symlink_target) sdk_output = symlink_target print(sdk_output) diff --git a/build/config/mac/xcode_toolchain.py b/build/config/mac/xcode_toolchain.py index 2089519f7c..36551b9731 100755 --- a/build/config/mac/xcode_toolchain.py +++ b/build/config/mac/xcode_toolchain.py @@ -4,19 +4,12 @@ # found in the LICENSE file. import argparse -import errno import sys import os import subprocess -def mkdir_p(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise +sys.path.insert(1, '../../build') +from pyutil.file_util import symlink def main(argv): @@ -30,10 +23,8 @@ def main(argv): assert os.path.exists(path) if args.symlink: - mkdir_p(args.symlink) symlink_target = os.path.join(args.symlink, os.path.basename(path)) - if not os.path.exists(symlink_target): - os.symlink(path, symlink_target) + symlink(path, symlink_target) path = symlink_target print(path) diff --git a/build/mac/find_sdk.py b/build/mac/find_sdk.py index 1a91aaa6c4..53860e08df 100755 --- a/build/mac/find_sdk.py +++ b/build/mac/find_sdk.py @@ -10,7 +10,6 @@ python find_sdk.py 10.6 # Ignores SDKs < 10.6 """ -import errno import os import re import subprocess @@ -18,15 +17,8 @@ from optparse import OptionParser - -def mkdir_p(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise +sys.path.insert(1, '../../build') +from pyutil.file_util import symlink def parse_version(version_str): @@ -92,10 +84,8 @@ def main(): sdk_output = subprocess.check_output(['xcodebuild', '-version', '-sdk', 'macosx' + best_sdk, 'Path']).strip() if options.symlink: - mkdir_p(options.symlink) symlink_target = os.path.join(options.symlink, os.path.basename(sdk_output)) - if not os.path.exists(symlink_target): - os.symlink(sdk_output, symlink_target) + symlink(sdk_output, symlink_target) sdk_output = symlink_target if options.print_sdk_path: diff --git a/build/pyutil/__init__.py b/build/pyutil/__init__.py new file mode 100644 index 0000000000..96196cffb2 --- /dev/null +++ b/build/pyutil/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/build/pyutil/file_util.py b/build/pyutil/file_util.py new file mode 100644 index 0000000000..e5ce3f8c7d --- /dev/null +++ b/build/pyutil/file_util.py @@ -0,0 +1,26 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import errno +import os + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +def symlink(target, link): + mkdir_p(os.path.dirname(link)) + tmp_link = link + '.tmp' + try: + os.remove(tmp_link) + except OSError: + pass + os.symlink(target, tmp_link) + os.rename(tmp_link, link) From 5fedd811e842d44dd64d68d48e530ff54f973820 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Fri, 11 Dec 2020 13:52:52 -0800 Subject: [PATCH 3/3] comments --- build/pyutil/file_util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/pyutil/file_util.py b/build/pyutil/file_util.py index e5ce3f8c7d..9711abfb82 100644 --- a/build/pyutil/file_util.py +++ b/build/pyutil/file_util.py @@ -5,6 +5,9 @@ import errno import os +"""Creates a directory and its parents (i.e. `mkdir -p`). + +If the directory already exists, does nothing.""" def mkdir_p(path): try: os.makedirs(path) @@ -15,6 +18,7 @@ def mkdir_p(path): raise +"""Creates or ovewrites a symlink from `link` to `target`.""" def symlink(target, link): mkdir_p(os.path.dirname(link)) tmp_link = link + '.tmp'