From 4047c00f6e513c4481b725fa71fef9134e175cc5 Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Thu, 24 Oct 2019 14:12:11 -0700
Subject: [PATCH 1/7] [fuchsia] [packaging] Layout debug symbols for Fuchsia

This packages and moves the relevant symbols to the right
locations in root_out_dir, but doesn't upload them to CIPD
yet. That will be done in a following change.

Refer: go/flutter-fuchsia-packaging for more information.
---
 tools/fuchsia/copy_debug_symbols.py | 73 +++++++++++++++++++++++++
 tools/fuchsia/debug_symbols.gni     | 82 +++++++++++++++++++++++++++++
 tools/fuchsia/fuchsia_archive.gni   | 14 ++++-
 3 files changed, 168 insertions(+), 1 deletion(-)
 create mode 100755 tools/fuchsia/copy_debug_symbols.py
 create mode 100644 tools/fuchsia/debug_symbols.gni

diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py
new file mode 100755
index 0000000000000..5fd0938bb65df
--- /dev/null
+++ b/tools/fuchsia/copy_debug_symbols.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+""" Gather the build_id, prefix_dir, and exec_name given the path to executable
+    also copies to the specified destination.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+
+def Touch(fname):
+  with open(fname, 'a'):
+    os.utime(fname, None)
+
+
+def GetBuildIdParts(exec_path):
+  file_out = subprocess.check_output(['file', exec_path])
+  build_id = re.match('.*=(.*?),', file_out).groups()[0]
+  return {
+      'build_id': build_id,
+      'prefix_dir': build_id[:2],
+      'exec_name': build_id[2:]
+  }
+
+
+def main():
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument(
+      '--executable-name', dest='exec_name', action='store', required=True)
+  parser.add_argument(
+      '--executable-path', dest='exec_path', action='store', required=True)
+  parser.add_argument(
+      '--destination-base', dest='dest', action='store', required=True)
+
+  parser.add_argument('--stripped', dest='stripped', action='store_true')
+  parser.add_argument('--unstripped', dest='stripped', action='store_false')
+  parser.set_defaults(stripped=True)
+
+  args = parser.parse_args()
+  assert os.path.exists(args.exec_path)
+  assert os.path.exists(args.dest)
+
+  parts = GetBuildIdParts(args.exec_path)
+  dbg_prefix_base = '%s/%s' % (args.dest, parts['prefix_dir'])
+
+  if not os.path.exists(dbg_prefix_base):
+    os.makedirs(dbg_prefix_base)
+
+  dbg_suffix = ''
+  if not args.stripped:
+    dbg_suffix = '.debug'
+  dbg_file_path = '%s/%s%s' % (dbg_prefix_base, parts['exec_name'], dbg_suffix)
+
+  shutil.copyfile(args.exec_path, dbg_file_path)
+
+  # Note this needs to be in sync with debug_symbols.gni
+  completion_file = '%s/.%s_dbg_success' % (args.dest, args.exec_name)
+  Touch(completion_file)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/tools/fuchsia/debug_symbols.gni b/tools/fuchsia/debug_symbols.gni
new file mode 100644
index 0000000000000..aa6bc538723ff
--- /dev/null
+++ b/tools/fuchsia/debug_symbols.gni
@@ -0,0 +1,82 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# The inputs to this template are 'binary_path' and a boolean 'unstripped'.
+# If 'unstripped' is specified, we append '.debug' to the symbols name.
+template("_copy_debug_symbols") {
+  assert(defined(invoker.binary_path), "'binary_path' needs to be defined.")
+
+  bin_path = invoker.binary_path
+
+  _args = []
+  if (defined(invoker.unstripped) && invoker.unstripped) {
+    _args += [ "--unstripped" ]
+  }
+
+  action(target_name) {
+    testonly = defined(invoker.testonly) && invoker.testonly
+
+    deps = []
+    if (defined(invoker.deps)) {
+      deps = invoker.deps
+    }
+
+    script = "$flutter_root/tools/fuchsia/copy_debug_symbols.py"
+
+    sources = [
+      bin_path,
+    ]
+
+    _dest_base =
+        "${root_out_dir}/flutter-debug-symbols-${target_os}-${target_cpu}"
+
+    args = _args + [
+             "--executable-name",
+             target_name,
+             "--executable-path",
+             rebase_path(bin_path),
+             "--destination-base",
+             rebase_path(_dest_base),
+           ]
+
+    outputs = [
+      "${_dest_base}/.${target_name}_success",
+    ]
+  }
+}
+
+# Takes a binary and generates its debug symbols following
+# the Fuchsia packaging convention.
+template("debug_symbols") {
+  assert(defined(invoker.binary), "'binary' needs to be defined.")
+  _testonly = defined(invoker.testonly) && invoker.testonly
+
+  _deps = []
+  if (defined(invoker.deps)) {
+    _deps = invoker.deps
+  }
+
+  bin = invoker.binary
+
+  _copy_debug_symbols("_${target_name}_stripped") {
+    testonly = _testonly
+    binary_path = rebase_path("${root_out_dir}/$bin")
+    deps = _deps
+  }
+
+  _copy_debug_symbols("_${target_name}_unstripped") {
+    testonly = _testonly
+    binary_path = "${root_out_dir}/exe.unstripped/$bin"
+    unstripped = true
+    deps = _deps
+  }
+
+  group(target_name) {
+    testonly = _testonly
+    deps = [
+      ":_${target_name}_stripped",
+      ":_${target_name}_unstripped",
+    ]
+  }
+}
diff --git a/tools/fuchsia/fuchsia_archive.gni b/tools/fuchsia/fuchsia_archive.gni
index a1e306c843767..53dbbf16c0390 100644
--- a/tools/fuchsia/fuchsia_archive.gni
+++ b/tools/fuchsia/fuchsia_archive.gni
@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("$flutter_root/tools/fuchsia/debug_symbols.gni")
+
 # Creates a Fuchsia archive (.far) file using PM from the Fuchsia SDK.
 template("fuchsia_archive") {
   assert(defined(invoker.binary), "package must define binary")
@@ -72,6 +74,13 @@ template("fuchsia_archive") {
              },
              "json")
 
+  _dbg_symbols_target = "${target_name}_dbg_symbols"
+  debug_symbols(_dbg_symbols_target) {
+    deps = pkg.deps
+    testonly = pkg_testonly
+    binary = invoker.binary
+  }
+
   pkg_dir_deps = pkg.deps + [ ":$cmx_target" ]
 
   action("${target_name}_dir") {
@@ -86,7 +95,10 @@ template("fuchsia_archive") {
 
   action(target_name) {
     script = "$flutter_root/tools/fuchsia/gen_package.py"
-    deps = pkg_dir_deps + [ ":${target_name}_dir" ]
+    deps = pkg_dir_deps + [
+             ":${target_name}_dir",
+             ":${_dbg_symbols_target}",
+           ]
     sources = copy_outputs
 
     inputs = []

From 34098744d2015c79bb29294dc4bbf233a62119a4 Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Thu, 24 Oct 2019 15:22:53 -0700
Subject: [PATCH 2/7] Add retry to makedirs

---
 tools/fuchsia/copy_debug_symbols.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py
index 5fd0938bb65df..fa1ad15973d6a 100755
--- a/tools/fuchsia/copy_debug_symbols.py
+++ b/tools/fuchsia/copy_debug_symbols.py
@@ -14,6 +14,7 @@
 import shutil
 import subprocess
 import sys
+import time
 
 
 def Touch(fname):
@@ -52,8 +53,21 @@ def main():
   parts = GetBuildIdParts(args.exec_path)
   dbg_prefix_base = '%s/%s' % (args.dest, parts['prefix_dir'])
 
-  if not os.path.exists(dbg_prefix_base):
-    os.makedirs(dbg_prefix_base)
+  success = False
+  for _ in range(3):
+    try:
+      if not os.path.exists(dbg_prefix_base):
+        os.makedirs(dbg_prefix_base)
+      success = True
+      break
+    except OSError as error:
+      print 'Failed to create dir %s, error: %s. sleeping...' % (
+          dbg_prefix_base, error)
+      time.sleep(3)
+
+  if not success:
+    print 'Unable to create directory: %s.' % dbg_prefix_base
+    return 1
 
   dbg_suffix = ''
   if not args.stripped:

From 90fff548bf1ac87d53d935d942027abc37cf37e1 Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Thu, 24 Oct 2019 15:49:08 -0700
Subject: [PATCH 3/7] use forward_args_from and also rename to
 "fuchsia_debug_symbols"

---
 tools/fuchsia/fuchsia_archive.gni             |  4 +-
 ..._symbols.gni => fuchsia_debug_symbols.gni} | 66 ++++++++-----------
 2 files changed, 30 insertions(+), 40 deletions(-)
 rename tools/fuchsia/{debug_symbols.gni => fuchsia_debug_symbols.gni} (54%)

diff --git a/tools/fuchsia/fuchsia_archive.gni b/tools/fuchsia/fuchsia_archive.gni
index 53dbbf16c0390..1dc65a66d0983 100644
--- a/tools/fuchsia/fuchsia_archive.gni
+++ b/tools/fuchsia/fuchsia_archive.gni
@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import("$flutter_root/tools/fuchsia/debug_symbols.gni")
+import("$flutter_root/tools/fuchsia/fuchsia_debug_symbols.gni")
 
 # Creates a Fuchsia archive (.far) file using PM from the Fuchsia SDK.
 template("fuchsia_archive") {
@@ -75,7 +75,7 @@ template("fuchsia_archive") {
              "json")
 
   _dbg_symbols_target = "${target_name}_dbg_symbols"
-  debug_symbols(_dbg_symbols_target) {
+  fuchsia_debug_symbols(_dbg_symbols_target) {
     deps = pkg.deps
     testonly = pkg_testonly
     binary = invoker.binary
diff --git a/tools/fuchsia/debug_symbols.gni b/tools/fuchsia/fuchsia_debug_symbols.gni
similarity index 54%
rename from tools/fuchsia/debug_symbols.gni
rename to tools/fuchsia/fuchsia_debug_symbols.gni
index aa6bc538723ff..cee11e934604e 100644
--- a/tools/fuchsia/debug_symbols.gni
+++ b/tools/fuchsia/fuchsia_debug_symbols.gni
@@ -6,39 +6,38 @@
 # If 'unstripped' is specified, we append '.debug' to the symbols name.
 template("_copy_debug_symbols") {
   assert(defined(invoker.binary_path), "'binary_path' needs to be defined.")
-
-  bin_path = invoker.binary_path
-
-  _args = []
-  if (defined(invoker.unstripped) && invoker.unstripped) {
-    _args += [ "--unstripped" ]
-  }
+  assert(defined(invoker.unstripped), "'unstripped' needs to be defined.")
 
   action(target_name) {
-    testonly = defined(invoker.testonly) && invoker.testonly
-
-    deps = []
-    if (defined(invoker.deps)) {
-      deps = invoker.deps
-    }
+    forward_variables_from(invoker,
+                           [
+                             "deps",
+                             "unstripped",
+                             "binary_path",
+                             "testonly",
+                           ])
 
     script = "$flutter_root/tools/fuchsia/copy_debug_symbols.py"
 
     sources = [
-      bin_path,
+      binary_path,
     ]
 
     _dest_base =
         "${root_out_dir}/flutter-debug-symbols-${target_os}-${target_cpu}"
 
-    args = _args + [
-             "--executable-name",
-             target_name,
-             "--executable-path",
-             rebase_path(bin_path),
-             "--destination-base",
-             rebase_path(_dest_base),
-           ]
+    args = [
+      "--executable-name",
+      target_name,
+      "--executable-path",
+      rebase_path(binary_path),
+      "--destination-base",
+      rebase_path(_dest_base),
+    ]
+
+    if (unstripped) {
+      args += [ "--unstripped" ]
+    }
 
     outputs = [
       "${_dest_base}/.${target_name}_success",
@@ -48,32 +47,23 @@ template("_copy_debug_symbols") {
 
 # Takes a binary and generates its debug symbols following
 # the Fuchsia packaging convention.
-template("debug_symbols") {
+template("fuchsia_debug_symbols") {
   assert(defined(invoker.binary), "'binary' needs to be defined.")
-  _testonly = defined(invoker.testonly) && invoker.testonly
-
-  _deps = []
-  if (defined(invoker.deps)) {
-    _deps = invoker.deps
-  }
-
-  bin = invoker.binary
 
   _copy_debug_symbols("_${target_name}_stripped") {
-    testonly = _testonly
-    binary_path = rebase_path("${root_out_dir}/$bin")
-    deps = _deps
+    forward_variables_from(invoker, "*")
+    binary_path = rebase_path("${root_out_dir}/$binary")
+    unstripped = false
   }
 
   _copy_debug_symbols("_${target_name}_unstripped") {
-    testonly = _testonly
-    binary_path = "${root_out_dir}/exe.unstripped/$bin"
+    forward_variables_from(invoker, "*")
+    binary_path = "${root_out_dir}/exe.unstripped/$binary"
     unstripped = true
-    deps = _deps
   }
 
   group(target_name) {
-    testonly = _testonly
+    forward_variables_from(invoker, [ "testonly" ])
     deps = [
       ":_${target_name}_stripped",
       ":_${target_name}_unstripped",

From 230b8bc0c65f348009acd449dd6c4b73f421eb31 Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Thu, 24 Oct 2019 16:03:49 -0700
Subject: [PATCH 4/7] add help statements and use os.path.join the copy debug
 symbols script

---
 tools/fuchsia/copy_debug_symbols.py | 45 ++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py
index fa1ad15973d6a..29eeacec49320 100755
--- a/tools/fuchsia/copy_debug_symbols.py
+++ b/tools/fuchsia/copy_debug_symbols.py
@@ -5,6 +5,9 @@
 # found in the LICENSE file.
 """ Gather the build_id, prefix_dir, and exec_name given the path to executable
     also copies to the specified destination.
+
+    The structure of debug symbols is as follows:
+      .build-id/<prefix>/<exec_name>[.debug]
 """
 
 import argparse
@@ -36,22 +39,43 @@ def main():
   parser = argparse.ArgumentParser()
 
   parser.add_argument(
-      '--executable-name', dest='exec_name', action='store', required=True)
+      '--executable-name',
+      dest='exec_name',
+      action='store',
+      required=True,
+      help='This is the name of the executable that we wish to layout debug symbols for.'
+  )
   parser.add_argument(
-      '--executable-path', dest='exec_path', action='store', required=True)
+      '--executable-path',
+      dest='exec_path',
+      action='store',
+      required=True,
+      help='Path to the executable on the filesystem.')
   parser.add_argument(
-      '--destination-base', dest='dest', action='store', required=True)
-
-  parser.add_argument('--stripped', dest='stripped', action='store_true')
-  parser.add_argument('--unstripped', dest='stripped', action='store_false')
-  parser.set_defaults(stripped=True)
+      '--destination-base',
+      dest='dest',
+      action='store',
+      required=True,
+      help='Path to the base directory where the debug symbols are to be laid out.'
+  )
+  parser.add_argument(
+      '--stripped',
+      dest='stripped',
+      action='store_true',
+      default=True,
+      help='Executable at the specified path is stripped.')
+  parser.add_argument(
+      '--unstripped',
+      dest='stripped',
+      action='store_false',
+      help='Executable at the specified path is unstripped.')
 
   args = parser.parse_args()
   assert os.path.exists(args.exec_path)
   assert os.path.exists(args.dest)
 
   parts = GetBuildIdParts(args.exec_path)
-  dbg_prefix_base = '%s/%s' % (args.dest, parts['prefix_dir'])
+  dbg_prefix_base = os.path.join(args.dest, parts['prefix_dir'])
 
   success = False
   for _ in range(3):
@@ -72,12 +96,13 @@ def main():
   dbg_suffix = ''
   if not args.stripped:
     dbg_suffix = '.debug'
-  dbg_file_path = '%s/%s%s' % (dbg_prefix_base, parts['exec_name'], dbg_suffix)
+  dbg_file_name = '%s%s' % (parts['exec_name'], dbg_suffix)
+  dbg_file_path = os.path.join(dbg_prefix_base, dbg_file_name)
 
   shutil.copyfile(args.exec_path, dbg_file_path)
 
   # Note this needs to be in sync with debug_symbols.gni
-  completion_file = '%s/.%s_dbg_success' % (args.dest, args.exec_name)
+  completion_file = os.path.join(args.dest, '%s_dbg_success' % args.exec_name)
   Touch(completion_file)
 
   return 0

From cc7fc9fe860f2e9e62c4cd49d808a6683c938e26 Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Fri, 25 Oct 2019 10:09:21 -0700
Subject: [PATCH 5/7] Add debug logs for "file" command

---
 tools/fuchsia/copy_debug_symbols.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py
index 29eeacec49320..8b71857e37ecf 100755
--- a/tools/fuchsia/copy_debug_symbols.py
+++ b/tools/fuchsia/copy_debug_symbols.py
@@ -20,12 +20,33 @@
 import time
 
 
+def IsExecutable(fpath):
+  return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+
+def Which(program):
+  fpath, _ = os.path.split(program)
+  if fpath:
+    if IsExecutable(program):
+      return program
+  else:
+    for path in os.environ["PATH"].split(os.pathsep):
+      exe_file = os.path.join(path, program)
+      if IsExecutable(exe_file):
+        return exe_file
+
+  return None
+
+
 def Touch(fname):
   with open(fname, 'a'):
     os.utime(fname, None)
 
 
 def GetBuildIdParts(exec_path):
+  if not Which('file'):
+    print "'file' command is not present on PATH."
+    sys.exit(1)
   file_out = subprocess.check_output(['file', exec_path])
   build_id = re.match('.*=(.*?),', file_out).groups()[0]
   return {

From ab7ca90ca14638ff1cee8e58547c4b3f0178b278 Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Fri, 25 Oct 2019 10:19:45 -0700
Subject: [PATCH 6/7] use llvm-readelf from Fuchsia toolchain

---
 tools/fuchsia/copy_debug_symbols.py     | 38 +++++++++----------------
 tools/fuchsia/fuchsia_debug_symbols.gni |  2 ++
 2 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py
index 8b71857e37ecf..41e2aa809e5ef 100755
--- a/tools/fuchsia/copy_debug_symbols.py
+++ b/tools/fuchsia/copy_debug_symbols.py
@@ -20,35 +20,16 @@
 import time
 
 
-def IsExecutable(fpath):
-  return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
-
-def Which(program):
-  fpath, _ = os.path.split(program)
-  if fpath:
-    if IsExecutable(program):
-      return program
-  else:
-    for path in os.environ["PATH"].split(os.pathsep):
-      exe_file = os.path.join(path, program)
-      if IsExecutable(exe_file):
-        return exe_file
-
-  return None
-
-
 def Touch(fname):
   with open(fname, 'a'):
     os.utime(fname, None)
 
 
-def GetBuildIdParts(exec_path):
-  if not Which('file'):
-    print "'file' command is not present on PATH."
-    sys.exit(1)
-  file_out = subprocess.check_output(['file', exec_path])
-  build_id = re.match('.*=(.*?),', file_out).groups()[0]
+def GetBuildIdParts(exec_path, read_elf):
+  file_out = subprocess.check_output(
+      [read_elf, '--hex-dump=.note.gnu.build-id', exec_path])
+  second_line = file_out.splitlines()[-1].split()
+  build_id = second_line[1] + second_line[2]
   return {
       'build_id': build_id,
       'prefix_dir': build_id[:2],
@@ -90,12 +71,19 @@ def main():
       dest='stripped',
       action='store_false',
       help='Executable at the specified path is unstripped.')
+  parser.add_argument(
+      '--read-elf',
+      dest='read_elf',
+      action='store',
+      required=True,
+      help='Path to read-elf executable.')
 
   args = parser.parse_args()
   assert os.path.exists(args.exec_path)
   assert os.path.exists(args.dest)
+  assert os.path.exists(args.read_elf)
 
-  parts = GetBuildIdParts(args.exec_path)
+  parts = GetBuildIdParts(args.exec_path, args.read_elf)
   dbg_prefix_base = os.path.join(args.dest, parts['prefix_dir'])
 
   success = False
diff --git a/tools/fuchsia/fuchsia_debug_symbols.gni b/tools/fuchsia/fuchsia_debug_symbols.gni
index cee11e934604e..fb02d70eeea41 100644
--- a/tools/fuchsia/fuchsia_debug_symbols.gni
+++ b/tools/fuchsia/fuchsia_debug_symbols.gni
@@ -33,6 +33,8 @@ template("_copy_debug_symbols") {
       rebase_path(binary_path),
       "--destination-base",
       rebase_path(_dest_base),
+      "--read-elf",
+      rebase_path("//fuchsia/toolchain/$host_os/bin/llvm-readelf"),
     ]
 
     if (unstripped) {

From 65e17778223c78ba9eae813bae41bf4b0f01324f Mon Sep 17 00:00:00 2001
From: Kaushik Iska <kaushikiska@google.com>
Date: Fri, 25 Oct 2019 10:47:02 -0700
Subject: [PATCH 7/7] make them hidden again

---
 tools/fuchsia/copy_debug_symbols.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/fuchsia/copy_debug_symbols.py b/tools/fuchsia/copy_debug_symbols.py
index 41e2aa809e5ef..e7f2ad7810aec 100755
--- a/tools/fuchsia/copy_debug_symbols.py
+++ b/tools/fuchsia/copy_debug_symbols.py
@@ -111,7 +111,7 @@ def main():
   shutil.copyfile(args.exec_path, dbg_file_path)
 
   # Note this needs to be in sync with debug_symbols.gni
-  completion_file = os.path.join(args.dest, '%s_dbg_success' % args.exec_name)
+  completion_file = os.path.join(args.dest, '.%s_dbg_success' % args.exec_name)
   Touch(completion_file)
 
   return 0