From b621b2cfe2a3c1571119b822af6fcd620e4b9aea Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Wed, 12 Apr 2017 10:59:59 -0400
Subject: [PATCH 1/8] bazel/ci: unify developer-local and CI build generation.

This patch dedupes the distinct approaches that existed previously, where we assembled hand curated
BUILD files for external dependencies to be used in developer-local builds and used prebuilt
artifacts compiled under the external dependency's native build system for CI.

In the new approach, the CI flow continues to prebuild artifacts with the build recipes and
recursive make in ci/build_container/{Makefile,build_recipes}, ahead of time and prior to any
invocation of Bazel.

Developer-local builds will not prebuild, but instead invoke the same build recipes and recursive
make under a Bazel repository_rule.

A significant improvement that is also provided by this patch is the automatic workspace population
via repositories.bzl. See the changes to WORKSPACE and ci/{WORKSPACE,WORKSPACE.consumer}. Projects
that consume Envoy, e.g. to link in additional filters, no longer need to track Envoy's
dependencies and maintain their own bind rules, this is now automagic.

WARNING: Any external consumer of the Bazel build will need to update their WORKSPACE definitions
after this patch is merged.
---
 .gitignore                                   |   1 +
 WORKSPACE                                    |  75 ---
 bazel/EXTERNAL_DEPS.md                       |  19 +
 bazel/envoy_build_system.bzl                 |   2 +-
 bazel/recipes.bzl                            |  16 +
 bazel/repositories.bzl                       | 626 ++++---------------
 bazel/repositories.sh                        |  22 +
 ci/WORKSPACE                                 |  71 +--
 ci/WORKSPACE.consumer                        |  71 +--
 ci/build_container/BUILD                     |   3 +
 ci/build_container/Dockerfile                |   2 +-
 ci/build_container/Makefile                  |  15 +-
 ci/build_container/build_container.sh        |   6 -
 ci/build_container/build_recipes/BUILD       |   3 +
 ci/build_container/docker_build_container.sh |   4 +
 ci/build_container/print_recipe_deps.sh      |   3 +
 ci/build_container/update_build_container.sh |   3 +-
 17 files changed, 206 insertions(+), 736 deletions(-)
 create mode 100644 bazel/EXTERNAL_DEPS.md
 create mode 100644 bazel/recipes.bzl
 create mode 100755 bazel/repositories.sh
 create mode 100644 ci/build_container/BUILD
 create mode 100644 ci/build_container/build_recipes/BUILD
 create mode 100755 ci/build_container/docker_build_container.sh
 create mode 100755 ci/build_container/print_recipe_deps.sh

diff --git a/.gitignore b/.gitignore
index 0fc989004d778..d4b22ca90b0ec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 /generated
 /bazel-*
 /ci/bazel-*
+/ci/build_container/recipes.bzl
 /ci/prebuilt/thirdparty
 /ci/prebuilt/thirdparty_build
 /test/coverage/BUILD
diff --git a/WORKSPACE b/WORKSPACE
index add3951ebe0af..b993737d20ec4 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -3,78 +3,3 @@ workspace(name = "envoy")
 load("//bazel:repositories.bzl", "envoy_dependencies")
 
 envoy_dependencies()
-
-bind(
-    name = "ares",
-    actual = "@cares_git//:ares",
-)
-
-bind(
-    name = "cc_wkt_protos_genproto",
-    actual = "@protobuf_git//:cc_wkt_protos_genproto",
-)
-
-bind(
-    name = "cc_wkt_protos",
-    actual = "@protobuf_git//:cc_wkt_protos",
-)
-
-bind(
-    name = "event",
-    actual = "@libevent_git//:event",
-)
-
-bind(
-    name = "event_pthreads",
-    actual = "@libevent_git//:event_pthreads",
-)
-
-bind(
-    name = "googletest",
-    actual = "@googletest_git//:googletest",
-)
-
-bind(
-    name = "http_parser",
-    actual = "@http_parser_git//:http_parser",
-)
-
-bind(
-    name = "lightstep",
-    actual = "@lightstep_git//:lightstep_core",
-)
-
-bind(
-    name = "nghttp2",
-    actual = "@nghttp2_tar//:nghttp2",
-)
-
-bind(
-    name = "protobuf",
-    actual = "@protobuf_git//:protobuf",
-)
-
-bind(
-    name = "protoc",
-    actual = "@protobuf_git//:protoc",
-)
-
-bind(
-    name = "rapidjson",
-    actual = "@rapidjson_git//:rapidjson",
-)
-
-bind(
-    name = "spdlog",
-    actual = "@spdlog_git//:spdlog",
-)
-
-bind(
-    name = "ssl",
-    actual = "@boringssl//:ssl",
-)
-
-bind(
-    name = "tclap",
-    actual = "@tclap_archive//:tclap",
-)
diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md
new file mode 100644
index 0000000000000..2f7c3fcbe0f63
--- /dev/null
+++ b/bazel/EXTERNAL_DEPS.md
@@ -0,0 +1,19 @@
+# Adding external dependencies to Envoy
+
+1. Specify name and version in [external documentation](../docs/install/requirements.rst).
+2. Add a build recipe in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes)
+   for developer-local and CI external dependency build flows.
+3. Add the build recipe to the list in [`bazel/recipes.bzl`](recipes.bzl).
+4. Add a build target X in [`ci/prebuilt/BUILD`](../ci/prebuilt/BUILD) to consume the headers and
+   libraries.
+5. Add a bind target in [`ci/repositories.bzl`](repositories.bzl#L72) to allow the new dependency to be
+   consumed by WORKSPACE.
+6. Reference your new external dependency in some `envoy_cc_library` via X in the `external_deps`
+   attribute.
+7. `bazel test //test/...`
+
+# Updating an external dependency version
+
+1. Specify the new version in [external documentation](../docs/install/requirements.rst).
+2. Update the build recipe in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes).
+3. `bazel test //test/...`
diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl
index 9e0c35c519d3d..3078656b763d2 100644
--- a/bazel/envoy_build_system.bzl
+++ b/bazel/envoy_build_system.bzl
@@ -1,4 +1,4 @@
-load("@protobuf_git//:protobuf.bzl", "cc_proto_library")
+load("@protobuf_bzl//:protobuf.bzl", "cc_proto_library")
 
 ENVOY_COPTS = [
     # TODO(htuch): Remove this when Bazel bringup is done.
diff --git a/bazel/recipes.bzl b/bazel/recipes.bzl
new file mode 100644
index 0000000000000..3cb07d5ebb124
--- /dev/null
+++ b/bazel/recipes.bzl
@@ -0,0 +1,16 @@
+RECIPES = [
+    "boringssl",
+    "cares",
+    "cotire",
+    "gcovr",
+    "googletest",
+    "gperftools",
+    "http-parser",
+    "libevent",
+    "lightstep",
+    "nghttp2",
+    "protobuf",
+    "rapidjson",
+    "spdlog",
+    "tclap",
+]
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
index 4219e2313e5ba..28be9f11b62c1 100644
--- a/bazel/repositories.bzl
+++ b/bazel/repositories.bzl
@@ -1,532 +1,150 @@
-# The build rules below for external dependencies build rules are maintained on a best effort basis.
-# The rules are provided for developer convenience. For production builds, we recommend building the
-# libraries according to their canonical build systems and expressing the dependencies in a manner
-# similar to ci/WORKSPACE.
-
-def ares_repositories():
-    BUILD = """
-cc_library(
-    name = "ares",
-    srcs = [
-        "ares__close_sockets.c",
-        "ares__get_hostent.c",
-        "ares__read_line.c",
-        "ares__timeval.c",
-        "ares_cancel.c",
-        "ares_create_query.c",
-        "ares_data.c",
-        "ares_destroy.c",
-        "ares_expand_name.c",
-        "ares_expand_string.c",
-        "ares_fds.c",
-        "ares_free_hostent.c",
-        "ares_free_string.c",
-        "ares_getenv.c",
-        "ares_gethostbyaddr.c",
-        "ares_gethostbyname.c",
-        "ares_getnameinfo.c",
-        "ares_getopt.c",
-        "ares_getsock.c",
-        "ares_init.c",
-        "ares_library_init.c",
-        "ares_llist.c",
-        "ares_mkquery.c",
-        "ares_nowarn.c",
-        "ares_options.c",
-        "ares_parse_a_reply.c",
-        "ares_parse_aaaa_reply.c",
-        "ares_parse_mx_reply.c",
-        "ares_parse_naptr_reply.c",
-        "ares_parse_ns_reply.c",
-        "ares_parse_ptr_reply.c",
-        "ares_parse_soa_reply.c",
-        "ares_parse_srv_reply.c",
-        "ares_parse_txt_reply.c",
-        "ares_platform.c",
-        "ares_process.c",
-        "ares_query.c",
-        "ares_search.c",
-        "ares_send.c",
-        "ares_strcasecmp.c",
-        "ares_strdup.c",
-        "ares_strerror.c",
-        "ares_timeout.c",
-        "ares_version.c",
-        "ares_writev.c",
-        "bitncmp.c",
-        "inet_net_pton.c",
-        "inet_ntop.c",
-        "windows_port.c",
-    ],
-    hdrs = [
-        "ares.h",
-        "ares_build.h",
-        "ares_config.h",
-        "ares_data.h",
-        "ares_dns.h",
-        "ares_getenv.h",
-        "ares_getopt.h",
-        "ares_inet_net_pton.h",
-        "ares_iphlpapi.h",
-        "ares_ipv6.h",
-        "ares_library_init.h",
-        "ares_llist.h",
-        "ares_nowarn.h",
-        "ares_platform.h",
-        "ares_private.h",
-        "ares_rules.h",
-        "ares_setup.h",
-        "ares_strcasecmp.h",
-        "ares_strdup.h",
-        "ares_version.h",
-        "ares_writev.h",
-        "bitncmp.h",
-        "nameser.h",
-        "setup_once.h",
-    ],
-    copts = [
-        "-DHAVE_CONFIG_H",
-    ],
-    includes = ["."],
-    visibility = ["//visibility:public"],
-)
-
-# There is some moderate evil here. buildconf needs to be invoked in its source
-# location, where it also generates temp files (no option to place in TMPDIR).
-# Also, configure does a cd relative dance that doesn't play nice with the
-# symlink execroot in the Bazel build. So, copy everything prior to building.
-ARES_CONFIG_SH_CMDS = [
-    "export TMPDIR=$$(realpath $(@D))",
-    "rm -rf $(@D)/cares_build",
-    "mkdir -p $(@D)/cares_build",
-    "cp --parents -L $(SRCS) $(@D)/cares_build",
-    "pushd $(@D)/cares_build/external/cares_git",
-    "./buildconf",
-    "./configure",
-    "popd",
-    "cp $(@D)/cares_build/external/cares_git/ares_config.h $@",
-]
-
-genrule(
-    name = "config",
-    srcs = glob(["**/*"]),
-    outs = ["ares_config.h"],
-    cmd = "&& ".join(ARES_CONFIG_SH_CMDS),
-)
-
-genrule(
-    name = "ares_build",
-    srcs = ["ares_build.h.dist"],
-    outs = ["ares_build.h"],
-    cmd = "cp $(SRCS) $@",
-)
-"""
-
-    native.new_git_repository(
-        name = "cares_git",
-        remote = "https://github.com/c-ares/c-ares.git",
-        commit = "7691f773af79bf75a62d1863fd0f13ebf9dc51b1", # v1.12.0
-        build_file_content = BUILD,
+load(":recipes.bzl", "RECIPES")
+
+def _repository_impl(ctxt):
+    # Setup the build directory with links to the relevant files.
+    ctxt.symlink(Label("//bazel:repositories.sh"), "repositories.sh")
+    ctxt.symlink(Label("//ci/build_container:build_and_install_deps.sh"),
+                 "build_and_install_deps.sh")
+    ctxt.symlink(Label("//ci/build_container:recipes.bzl"), "recipes.bzl")
+    ctxt.symlink(Label("//ci/build_container:print_recipe_deps.sh"), "print_recipe_deps.sh")
+    ctxt.symlink(Label("//ci/build_container:Makefile"), "Makefile")
+    for r in RECIPES:
+        ctxt.symlink(Label("//ci/build_container/build_recipes:" + r + ".sh"),
+                     "build_recipes/" + r + ".sh")
+    ctxt.symlink(Label("//ci/prebuilt:BUILD"), "BUILD")
+
+    # Run the build script.
+    environment = {}
+    if ctxt.attr.debug:
+        environment["DEBUG"] = "1"
+    result = ctxt.execute(
+        ["./repositories.sh"],
+        environment = environment,
+        # Ideally we would print progress, but instead this hangs on "INFO: Loading
+        # complete.  Analyzing.." today, see
+        # https://github.com/bazelbuild/bazel/issues/1289. We could set quiet=False
+        # as well to indicate progress, but that isn't supported in versions folks
+        # are using right now (0.4.5).
+        # TODO(htuch): Revisit this when everyone is on newer Bazel versions.
+        #
+        # quiet = False,
     )
-
-def boringssl_repositories():
-    native.git_repository(
-        name = "boringssl",
-        commit = "bfd36df3da38dbf8828e712f42fbab2a0034bc40",  # 2017-02-02
-        remote = "https://boringssl.googlesource.com/boringssl",
+    if result.return_code != 0 or ctxt.attr.debug:
+        print("External dep build exited with return code: %d" % result.return_code)
+        print(result.stdout)
+        print(result.stderr)
+        if result.return_code != 0:
+            fail("External dep build failed")
+
+def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None):
+    # Used only for protobuf.bzl.
+    if local_protobuf_bzl:
+        native.new_local_repository(
+            name = "protobuf_bzl",
+            path = "/thirdparty/protobuf-3.2.0",
+            # We only want protobuf.bzl, so don't support building out of this repo.
+            build_file_content = "",
+        )
+    else:
+        native.new_git_repository(
+            name = "protobuf_bzl",
+            # Using a non-canonical repository/branch here. This is a workaround to the lack of
+            # merge on https://github.com/google/protobuf/pull/2508, which is needed for supporting
+            # arbitrary CC compiler locations from the environment. The branch is
+            # https://github.com/htuch/protobuf/tree/v3.2.0-default-shell-env, which is the 3.2.0
+            # release with the above mentioned PR cherry picked.
+            commit = "d490587268931da78c942a6372ef57bb53db80da",
+            remote = "https://github.com/htuch/protobuf.git",
+            # We only want protobuf.bzl, so don't support building out of this repo.
+            build_file_content = "",
+        )
+
+    # Set this to True to make the build debug cycles faster.
+    debug_build = False
+
+    envoy_repository = repository_rule(
+        implementation = _repository_impl,
+        local = debug_build,
+        environ = ["CC", "CXX", "LD_LIBRARY_PATH"],
+        attrs = {"debug": attr.bool(default=False)},
     )
 
-def googletest_repositories():
-    BUILD = """
-cc_library(
-    name = "googletest",
-    srcs = [
-        "googlemock/src/gmock-all.cc",
-        "googletest/src/gtest-all.cc",
-    ],
-    hdrs = glob([
-        "googlemock/include/**/*.h",
-        "googlemock/src/*.cc",
-        "googletest/include/**/*.h",
-        "googletest/src/*.cc",
-        "googletest/src/*.h",
-    ]),
-    includes = [
-        "googlemock",
-        "googlemock/include",
-        "googletest",
-        "googletest/include",
-    ],
-    visibility = ["//visibility:public"],
-)
-"""
-    native.new_git_repository(
-        name = "googletest_git",
-        build_file_content = BUILD,
-        # v1.8.0 release
-        commit = "ec44c6c1675c25b9827aacd08c02433cccde7780",
-        remote = "https://github.com/google/googletest.git",
+    envoy_repository(
+        name = "envoy_deps",
+        debug = debug_build,
     )
 
-def http_parser_repositories():
-    BUILD = """
-cc_library(
-    name = "http_parser",
-    srcs = [
-        "http_parser.c",
-    ],
-    hdrs = [
-        "http_parser.h",
-    ],
-    visibility = ["//visibility:public"],
-)
-"""
-
-    native.new_git_repository(
-        name = "http_parser_git",
-        remote = "https://github.com/nodejs/http-parser.git",
-        commit = "9b0d5b33ebdaacff1dadd06bad4e198b11ff880e",
-        build_file_content = BUILD,
+    native.bind(
+        name = "ares",
+        actual = path + ":ares",
     )
 
-def libevent_repositories():
-    BUILD = """
-genrule(
-    name = "config",
-    srcs = glob(["**/*"]),
-    outs = ["config.h"],
-    cmd = "TMPDIR=$(@D) $(location configure) --enable-shared=no --disable-libevent-regress " +
-          "--disable-openssl && cp config.h $@",
-    tools = ["configure"],
-)
-
-genrule(
-    name = "event-config",
-    srcs = [
-        "config.h",
-        "make-event-config.sed",
-    ],
-    outs = ["include/event2/event-config.h"],
-    cmd = "sed -f $(location make-event-config.sed) < $(location config.h) > $@",
-)
-
-event_srcs = [
-    "buffer.c",
-    "bufferevent.c",
-    "bufferevent_filter.c",
-    "bufferevent_pair.c",
-    "bufferevent_ratelim.c",
-    "bufferevent_sock.c",
-    "epoll.c",
-    "evdns.c",
-    "event.c",
-    "event_tagging.c",
-    "evmap.c",
-    "evrpc.c",
-    "evthread.c",
-    "evutil.c",
-    "evutil_rand.c",
-    "evutil_time.c",
-    "http.c",
-    "listener.c",
-    "log.c",
-    "poll.c",
-    "select.c",
-    "signal.c",
-    "strlcpy.c",
-    ":event-config",
-] + glob(["*.h"])
-
-event_pthread_srcs = [
-    "evthread_pthread.c",
-    ":event-config",
-]
-
-cc_library(
-    name = "event",
-    srcs = event_srcs,
-    hdrs = glob(["include/**/*.h"]) + [
-        "arc4random.c",  # arc4random.c is included by evutil_rand.c
-        "bufferevent-internal.h",
-        "defer-internal.h",
-        "evbuffer-internal.h",
-        "event-internal.h",
-        "evthread-internal.h",
-        "http-internal.h",
-        "iocp-internal.h",
-        "ipv6-internal.h",
-        "log-internal.h",
-        "minheap-internal.h",
-        "mm-internal.h",
-        "strlcpy-internal.h",
-        "util-internal.h",
-        "compat/sys/queue.h",
-    ],
-    copts = [
-        "-w",
-        "-DHAVE_CONFIG_H",
-    ],
-    includes = [
-        "compat",
-        "include",
-    ],
-    visibility = ["//visibility:public"],
-)
-
-cc_library(
-    name = "event_pthreads",
-    srcs = event_pthread_srcs + ["include/event2/thread.h"],
-    hdrs = [
-        "compat/sys/queue.h",
-        "evthread-internal.h",
-    ],
-    copts = [
-        "-w",
-        "-DHAVE_CONFIG_H",
-    ],
-    includes = [
-        "compat",
-        "include",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":event"],
-)"""
-
-    native.new_http_archive(
-        name = "libevent_git",
-        url = "https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz",
-        strip_prefix = "libevent-2.1.8-stable",
-        build_file_content = BUILD,
+    native.bind(
+        name = "cc_wkt_protos_genproto",
+        actual = path + ":cc_wkt_protos_genproto",
     )
 
-def lightstep_repositories():
-    BUILD = """
-load("@protobuf_git//:protobuf.bzl", "cc_proto_library")
-
-cc_library(
-    name = "lightstep_core",
-    srcs = [
-        "src/c++11/impl.cc",
-        "src/c++11/span.cc",
-        "src/c++11/tracer.cc",
-        "src/c++11/util.cc",
-    ],
-    hdrs = [
-        "src/c++11/lightstep/impl.h",
-        "src/c++11/lightstep/options.h",
-        "src/c++11/lightstep/propagation.h",
-        "src/c++11/lightstep/carrier.h",
-        "src/c++11/lightstep/span.h",
-        "src/c++11/lightstep/tracer.h",
-        "src/c++11/lightstep/util.h",
-        "src/c++11/lightstep/value.h",
-        "src/c++11/mapbox_variant/recursive_wrapper.hpp",
-        "src/c++11/mapbox_variant/variant.hpp",
-    ],
-    copts = [
-        "-DPACKAGE_VERSION='\\"0.36\\"'",
-        "-Iexternal/lightstep_git/src/c++11/lightstep",
-        "-Iexternal/lightstep_git/src/c++11/mapbox_variant",
-    ],
-    includes = ["src/c++11"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "@lightstep_common_git//:collector_proto",
-        "@lightstep_common_git//:lightstep_carrier_proto",
-        "//external:protobuf",
-    ],
-)"""
-
-    COMMON_BUILD = """
-load("@protobuf_git//:protobuf.bzl", "cc_proto_library")
-
-cc_proto_library(
-    name = "collector_proto",
-    srcs = ["collector.proto"],
-    include = ".",
-    deps = [
-        "//external:cc_wkt_protos",
-    ],
-    protoc = "//external:protoc",
-    default_runtime = "//external:protobuf",
-    visibility = ["//visibility:public"],
-)
-
-cc_proto_library(
-    name = "lightstep_carrier_proto",
-    srcs = ["lightstep_carrier.proto"],
-    include = ".",
-    deps = [
-        "//external:cc_wkt_protos",
-    ],
-    protoc = "//external:protoc",
-    default_runtime = "//external:protobuf",
-    visibility = ["//visibility:public"],
-)
-"""
-
-    native.new_git_repository(
-        name = "lightstep_common_git",
-        remote = "https://github.com/lightstep/lightstep-tracer-common.git",
-        commit = "cbbecd671c1ae1f20ae873c5da688c8c14d04ec3",
-        build_file_content = COMMON_BUILD,
+    native.bind(
+        name = "cc_wkt_protos",
+        actual = path + ":cc_wkt_protos",
     )
 
-    native.new_git_repository(
-        name = "lightstep_git",
-        remote = "https://github.com/lightstep/lightstep-tracer-cpp.git",
-        commit = "f1dc8f3dfd529350e053fd21273e627f409ae428", # 0.36
-        build_file_content = BUILD,
+    native.bind(
+        name = "event",
+        actual = path + ":event",
     )
 
-def nghttp2_repositories():
-    BUILD = """
-genrule(
-    name = "config",
-    srcs = glob(["**/*"]),
-    outs = ["config.h"],
-    cmd = "TMPDIR=$(@D) $(location configure) --enable-lib-only --enable-shared=no" +
-          " && cp config.h $@",
-    tools = ["configure"],
-)
+    native.bind(
+        name = "event_pthreads",
+        actual = path + ":event_pthreads",
+    )
 
-cc_library(
-    name = "nghttp2",
-    srcs = glob([
-        "lib/*.c",
-        "lib/*.h",
-    ]) + ["config.h"],
-    hdrs = glob(["lib/includes/nghttp2/*.h"]),
-    copts = [
-        "-DHAVE_CONFIG_H",
-        "-DBUILDING_NGHTTP2",
-    ],
-    includes = [
-        ".",
-        "lib/includes",
-    ],
-    visibility = ["//visibility:public"],
-)
-"""
+    native.bind(
+        name = "googletest",
+        actual = path + ":googletest",
+    )
 
-    native.new_http_archive(
-        name = "nghttp2_tar",
-        url = "https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz",
-        strip_prefix = "nghttp2-1.20.0",
-        build_file_content = BUILD,
+    native.bind(
+        name = "http_parser",
+        actual = path + ":http_parser",
     )
 
-def protobuf_repositories():
-    native.git_repository(
-        name = "protobuf_git",
-        # Using a non-canonical repository/branch here. This is a workaround to the lack of
-        # merge on https://github.com/google/protobuf/pull/2508, which is needed for supporting
-        # arbitrary CC compiler locations from the environment. The branch is
-        # https://github.com/htuch/protobuf/tree/v3.2.0-default-shell-env, which is the 3.2.0
-        # release with the above mentioned PR cherry picked.
-        commit = "d490587268931da78c942a6372ef57bb53db80da",
-        remote = "https://github.com/htuch/protobuf.git",
+    native.bind(
+        name = "lightstep",
+        actual = path + ":lightstep",
     )
 
-def rapidjson_repositories():
-    BUILD = """
-cc_library(
-    name = "rapidjson",
-    srcs = glob([
-        "include/rapidjson/internal/*.h",
-    ]),
-    hdrs = glob([
-        "include/rapidjson/*.h",
-        "include/rapidjson/error/*.h",
-    ]),
-    includes = ["include"],
-    visibility = ["//visibility:public"],
-)
-"""
+    native.bind(
+        name = "nghttp2",
+        actual = path + ":nghttp2",
+    )
 
-    native.new_git_repository(
-        name = "rapidjson_git",
-        remote = "https://github.com/miloyip/rapidjson.git",
-        commit = "f54b0e47a08782a6131cc3d60f94d038fa6e0a51", # v1.1.0
-        build_file_content = BUILD,
+    native.bind(
+        name = "protobuf",
+        actual = path + ":protobuf",
     )
 
-def spdlog_repositories():
-    BUILD = """
-package(default_visibility = ["//visibility:public"])
+    native.bind(
+        name = "protoc",
+        actual = path + ":protoc",
+    )
 
-cc_library(
-    name = "spdlog",
-    hdrs = glob([
-        "include/**/*.cc",
-        "include/**/*.h",
-    ]),
-    strip_include_prefix = "include",
-)
-"""
+    native.bind(
+        name = "rapidjson",
+        actual = path + ":rapidjson",
+    )
 
-    native.new_git_repository(
-        name = "spdlog_git",
-        build_file_content = BUILD,
-        # v0.11.0 release
-        commit = "1f1f6a5f3b424203a429e9cb78e6548037adefa8",
-        remote = "https://github.com/gabime/spdlog.git",
+    native.bind(
+        name = "spdlog",
+        actual = path + ":spdlog",
     )
 
-def tclap_repositories():
-    BUILD = """
-cc_library(
-    name = "tclap",
-    hdrs = [
-        "include/tclap/Arg.h",
-        "include/tclap/ArgException.h",
-        "include/tclap/ArgTraits.h",
-        "include/tclap/CmdLine.h",
-        "include/tclap/CmdLineInterface.h",
-        "include/tclap/CmdLineOutput.h",
-        "include/tclap/Constraint.h",
-        "include/tclap/DocBookOutput.h",
-        "include/tclap/HelpVisitor.h",
-        "include/tclap/IgnoreRestVisitor.h",
-        "include/tclap/MultiArg.h",
-        "include/tclap/MultiSwitchArg.h",
-        "include/tclap/OptionalUnlabeledTracker.h",
-        "include/tclap/StandardTraits.h",
-        "include/tclap/StdOutput.h",
-        "include/tclap/SwitchArg.h",
-        "include/tclap/UnlabeledMultiArg.h",
-        "include/tclap/UnlabeledValueArg.h",
-        "include/tclap/ValueArg.h",
-        "include/tclap/ValuesConstraint.h",
-        "include/tclap/VersionVisitor.h",
-        "include/tclap/Visitor.h",
-        "include/tclap/XorHandler.h",
-        "include/tclap/ZshCompletionOutput.h",
-    ],
-    defines = [
-        "HAVE_LONG_LONG=1",
-        "HAVE_SSTREAM=1",
-    ],
-    includes = ["include"],
-    visibility = ["//visibility:public"],
-)
-"""
-    native.new_http_archive(
-        name = "tclap_archive",
-        url = "https://storage.googleapis.com/istio-build-deps/tclap-1.2.1.tar.gz",
-        strip_prefix = "tclap-1.2.1",
-        build_file_content = BUILD,
+    native.bind(
+        name = "ssl",
+        actual = path + ":ssl",
     )
 
-def envoy_dependencies():
-    ares_repositories()
-    boringssl_repositories()
-    googletest_repositories()
-    http_parser_repositories()
-    libevent_repositories()
-    lightstep_repositories()
-    nghttp2_repositories()
-    protobuf_repositories()
-    rapidjson_repositories()
-    spdlog_repositories()
-    tclap_repositories()
+    native.bind(
+        name = "tclap",
+        actual = path + ":tclap",
+    )
diff --git a/bazel/repositories.sh b/bazel/repositories.sh
new file mode 100755
index 0000000000000..469177a0d1ab3
--- /dev/null
+++ b/bazel/repositories.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+# When debugging the build, it's helpful to keep the artifacts in /tmp, since they don't get
+# repeatedly clobbered as build recipes are modified. This is controlled by debug_build in
+# bazel/respositories.bzl.
+if [[ "${DEBUG}" == "1" ]]
+then
+  BASEDIR=/tmp/bazel-envoy-deps
+else
+  BASEDIR="${PWD}/build"
+fi
+
+export THIRDPARTY_DEPS="${BASEDIR}"
+export THIRDPARTY_SRC="${BASEDIR}/thirdparty"
+export THIRDPARTY_BUILD="${BASEDIR}/thirdparty_build"
+
+time ./build_and_install_deps.sh "${PREBUILT_DIR_REAL}"
+
+ln -sf "$(realpath "${THIRDPARTY_SRC}")" thirdparty
+ln -sf "$(realpath "${THIRDPARTY_BUILD}")" thirdparty_build
diff --git a/ci/WORKSPACE b/ci/WORKSPACE
index 4cfc2a6c7b62a..15de7529dae10 100644
--- a/ci/WORKSPACE
+++ b/ci/WORKSPACE
@@ -1,71 +1,8 @@
 workspace(name = "ci")
 
-bind(
-    name = "ares",
-    actual = "//ci/prebuilt:ares",
-)
-
-bind(
-    name = "event",
-    actual = "//ci/prebuilt:event",
-)
-
-bind(
-    name = "event_pthreads",
-    actual = "//ci/prebuilt:event_pthreads",
-)
-
-bind(
-    name = "googletest",
-    actual = "//ci/prebuilt:googletest",
-)
-
-bind(
-    name = "http_parser",
-    actual = "//ci/prebuilt:http_parser",
-)
-
-bind(
-    name = "lightstep",
-    actual = "//ci/prebuilt:lightstep",
-)
-
-bind(
-    name = "nghttp2",
-    actual = "//ci/prebuilt:nghttp2",
-)
-
-bind(
-    name = "protobuf",
-    actual = "//ci/prebuilt:protobuf",
-)
-
-local_repository(
-    name = "protobuf_git",
-    path = "/thirdparty/protobuf-3.2.0",
-)
-
-bind(
-    name = "protoc",
-    actual = "//ci/prebuilt:protoc",
-)
-
-bind(
-    name = "rapidjson",
-    actual = "//ci/prebuilt:rapidjson",
-)
-
-bind(
-    name = "spdlog",
-    actual = "//ci/prebuilt:spdlog",
-)
-
-bind(
-    name = "ssl",
-    actual = "//ci/prebuilt:ssl",
-)
+load("//bazel:repositories.bzl", "envoy_dependencies")
 
-bind(
-    name = "tclap",
-    actual = "//ci/prebuilt:tclap",
+envoy_dependencies(
+    path = "//ci/prebuilt",
+    local_protobuf_bzl = "/thirdparty/protobuf-3.2.0",
 )
diff --git a/ci/WORKSPACE.consumer b/ci/WORKSPACE.consumer
index 69f2f18208225..f54eb74abba48 100644
--- a/ci/WORKSPACE.consumer
+++ b/ci/WORKSPACE.consumer
@@ -5,72 +5,9 @@ local_repository(
     path = "/source",
 )
 
-bind(
-    name = "ares",
-    actual = "@envoy//ci/prebuilt:ares",
-)
-
-bind(
-    name = "event",
-    actual = "@envoy//ci/prebuilt:event",
-)
-
-bind(
-    name = "event_pthreads",
-    actual = "@envoy//ci/prebuilt:event_pthreads",
-)
-
-bind(
-    name = "googletest",
-    actual = "@envoy//ci/prebuilt:googletest",
-)
-
-bind(
-    name = "http_parser",
-    actual = "@envoy//ci/prebuilt:http_parser",
-)
-
-bind(
-    name = "lightstep",
-    actual = "@envoy//ci/prebuilt:lightstep",
-)
-
-bind(
-    name = "nghttp2",
-    actual = "@envoy//ci/prebuilt:nghttp2",
-)
-
-bind(
-    name = "protobuf",
-    actual = "@envoy//ci/prebuilt:protobuf",
-)
-
-local_repository(
-    name = "protobuf_git",
-    path = "/thirdparty/protobuf-3.2.0",
-)
-
-bind(
-    name = "protoc",
-    actual = "@envoy//ci/prebuilt:protoc",
-)
-
-bind(
-    name = "rapidjson",
-    actual = "@envoy//ci/prebuilt:rapidjson",
-)
-
-bind(
-    name = "spdlog",
-    actual = "@envoy//ci/prebuilt:spdlog",
-)
-
-bind(
-    name = "ssl",
-    actual = "@envoy//ci/prebuilt:ssl",
-)
+load("//bazel:repositories.bzl", "envoy_dependencies")
 
-bind(
-    name = "tclap",
-    actual = "@envoy//ci/prebuilt:tclap",
+envoy_dependencies(
+    path = "@envoy//ci/prebuilt",
+    local_protobuf_bzl = "/thirdparty/protobuf-3.2.0",
 )
diff --git a/ci/build_container/BUILD b/ci/build_container/BUILD
new file mode 100644
index 0000000000000..b2d5ae57432b3
--- /dev/null
+++ b/ci/build_container/BUILD
@@ -0,0 +1,3 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["build_and_install_deps.sh", "Makefile"])
diff --git a/ci/build_container/Dockerfile b/ci/build_container/Dockerfile
index aaafd91ff0673..52f12707f82e5 100644
--- a/ci/build_container/Dockerfile
+++ b/ci/build_container/Dockerfile
@@ -1,4 +1,4 @@
 FROM ubuntu:xenial
-COPY ./build_container.sh ./build_and_install_deps.sh ./Makefile /
+COPY ./build_container.sh ./print_recipe_deps.sh ./build_and_install_deps.sh ./recipes.bzl ./Makefile /
 COPY ./build_recipes/*.sh /build_recipes/
 RUN ./build_container.sh
diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile
index c3c570d54b917..cf6dd3f1847fe 100644
--- a/ci/build_container/Makefile
+++ b/ci/build_container/Makefile
@@ -2,20 +2,7 @@
 # version number, etc.) to uniquely identify the revision of the upstream dependency.  This allows
 # make to pick up changes with a simple direct dependency on the build recipe.
 
-all: $(THIRDPARTY_DEPS)/libevent.dep \
-	$(THIRDPARTY_DEPS)/boringssl.dep \
-	$(THIRDPARTY_DEPS)/gperftools.dep \
-	$(THIRDPARTY_DEPS)/nghttp2.dep \
-	$(THIRDPARTY_DEPS)/cares.dep \
-	$(THIRDPARTY_DEPS)/protobuf.dep \
-	$(THIRDPARTY_DEPS)/cotire.dep \
-	$(THIRDPARTY_DEPS)/spdlog.dep \
-	$(THIRDPARTY_DEPS)/http-parser.dep \
-	$(THIRDPARTY_DEPS)/tclap.dep \
-	$(THIRDPARTY_DEPS)/lightstep.dep \
-	$(THIRDPARTY_DEPS)/rapidjson.dep \
-	$(THIRDPARTY_DEPS)/googletest.dep \
-	$(THIRDPARTY_DEPS)/gcovr.dep
+all: $(addprefix $(THIRDPARTY_DEPS)/,$(shell ./print_recipe_deps.sh))
 
 RECIPES := build_recipes
 
diff --git a/ci/build_container/build_container.sh b/ci/build_container/build_container.sh
index 4a32dea6068dc..2b96d1ef82379 100755
--- a/ci/build_container/build_container.sh
+++ b/ci/build_container/build_container.sh
@@ -44,10 +44,4 @@ export CXX=g++-4.9
 export THIRDPARTY_DEPS=/tmp
 export THIRDPARTY_SRC=/thirdparty
 export THIRDPARTY_BUILD=/thirdparty_build
-# TODO(htuch): Remove the first build of the libraries in non-distinct locations when cmake is gone.
-# Below we now build/install twice and this requires 2x the space in the Docker image as is to
-# support both Bazel and cmake, but it's not worth fixing up all the cmake stuff since it's going
-# soon.
 "$(dirname "$0")"/build_and_install_deps.sh
-rm -f /tmp/*.dep
-BUILD_DISTINCT=1 "$(dirname "$0")"/build_and_install_deps.sh
diff --git a/ci/build_container/build_recipes/BUILD b/ci/build_container/build_recipes/BUILD
new file mode 100644
index 0000000000000..b1f93d428065d
--- /dev/null
+++ b/ci/build_container/build_recipes/BUILD
@@ -0,0 +1,3 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(glob(["*.sh"]))
diff --git a/ci/build_container/docker_build_container.sh b/ci/build_container/docker_build_container.sh
new file mode 100755
index 0000000000000..bb8dfee7844bd
--- /dev/null
+++ b/ci/build_container/docker_build_container.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cp ../../bazel/recipes.bzl .
+docker build --rm -t lyft/envoy-build:$TAG .
diff --git a/ci/build_container/print_recipe_deps.sh b/ci/build_container/print_recipe_deps.sh
new file mode 100755
index 0000000000000..65d4640412dff
--- /dev/null
+++ b/ci/build_container/print_recipe_deps.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python <(cat recipes.bzl; echo "print ' '.join(\"%s.dep\" % r for r in RECIPES)")
diff --git a/ci/build_container/update_build_container.sh b/ci/build_container/update_build_container.sh
index 2dcad425d6a0b..0c0e39edc47a9 100755
--- a/ci/build_container/update_build_container.sh
+++ b/ci/build_container/update_build_container.sh
@@ -1,11 +1,12 @@
 #!/bin/bash
+
 read -r -p "Do you have master checked out with most recent changes? [y/N] " response
 if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
 then
   TAG="$(git rev-parse master)"
   docker-machine start default
   eval $(docker-machine env default)
-  docker build --rm -t lyft/envoy-build:$TAG .
+  ./docker_build_container.sh
   docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
   docker push lyft/envoy-build:$TAG
   echo Pushed lyft/envoy-build:$TAG

From d637152a66da66096af49965cf0c2e7391d086fc Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Wed, 12 Apr 2017 19:32:42 -0400
Subject: [PATCH 2/8] Fix recipes.bzl location.

---
 bazel/repositories.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
index 28be9f11b62c1..a16cc8437d847 100644
--- a/bazel/repositories.bzl
+++ b/bazel/repositories.bzl
@@ -2,10 +2,10 @@ load(":recipes.bzl", "RECIPES")
 
 def _repository_impl(ctxt):
     # Setup the build directory with links to the relevant files.
+    ctxt.symlink(Label("//bazel:recipes.bzl"), "recipes.bzl")
     ctxt.symlink(Label("//bazel:repositories.sh"), "repositories.sh")
     ctxt.symlink(Label("//ci/build_container:build_and_install_deps.sh"),
                  "build_and_install_deps.sh")
-    ctxt.symlink(Label("//ci/build_container:recipes.bzl"), "recipes.bzl")
     ctxt.symlink(Label("//ci/build_container:print_recipe_deps.sh"), "print_recipe_deps.sh")
     ctxt.symlink(Label("//ci/build_container:Makefile"), "Makefile")
     for r in RECIPES:

From 7d9d5805613ba3f881fe4ab9dd8bc0cdbe316cc3 Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Wed, 12 Apr 2017 22:48:32 -0400
Subject: [PATCH 3/8] Set CC/CXX if not set in Makefile.

---
 ci/build_container/Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile
index cf6dd3f1847fe..27c95bb13f91b 100644
--- a/ci/build_container/Makefile
+++ b/ci/build_container/Makefile
@@ -6,6 +6,10 @@ all: $(addprefix $(THIRDPARTY_DEPS)/,$(shell ./print_recipe_deps.sh))
 
 RECIPES := build_recipes
 
+# Make sure we use a consistent compiler across all deps.
+CC ?= gcc
+CXX ?= g++
+
 # If $(BUILD_DISTINCT) is set in the make environment, the artifacts are built and installed in
 # distinct directories under $(THIRDPARTY_BUILD) and $(THIRDPARTY_SRC). They end up looking like
 # $(THIRDPARTY_BUILD)/protobuf.dep/include, $(THIRDPARTY_BUILD)/cotire.dep/include etc. instead of
@@ -31,6 +35,8 @@ build-recipe = cd $(THIRDPARTY_SRC) && \
     $(build-setup) && \
 	(((THIRDPARTY_SRC=$(THIRDPARTY_SRC)/$(DISTINCT_PATH) \
 	   THIRDPARTY_BUILD=$(THIRDPARTY_BUILD)/$(DISTINCT_PATH) \
+	   CC=$(CC) \
+	   CXX=$(CXX) \
 	   $(1) \
 	   bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \
 	$(build-complete)

From 5ddbe35f711014cfeeccce41a2e20fe9ada4c691 Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Wed, 12 Apr 2017 22:57:20 -0400
Subject: [PATCH 4/8] Allow binds to be skipped, cleanup boiler plate.

---
 bazel/EXTERNAL_DEPS.md |  2 +-
 bazel/repositories.bzl | 99 ++++++++++--------------------------------
 2 files changed, 25 insertions(+), 76 deletions(-)

diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md
index 2f7c3fcbe0f63..38aec4c2f3309 100644
--- a/bazel/EXTERNAL_DEPS.md
+++ b/bazel/EXTERNAL_DEPS.md
@@ -6,7 +6,7 @@
 3. Add the build recipe to the list in [`bazel/recipes.bzl`](recipes.bzl).
 4. Add a build target X in [`ci/prebuilt/BUILD`](../ci/prebuilt/BUILD) to consume the headers and
    libraries.
-5. Add a bind target in [`ci/repositories.bzl`](repositories.bzl#L72) to allow the new dependency to be
+5. Add a bind target in [`ci/repositories.bzl`](repositories.bzl#L4) to allow the new dependency to be
    consumed by WORKSPACE.
 6. Reference your new external dependency in some `envoy_cc_library` via X in the `external_deps`
    attribute.
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
index a16cc8437d847..7bc6573899086 100644
--- a/bazel/repositories.bzl
+++ b/bazel/repositories.bzl
@@ -1,5 +1,22 @@
 load(":recipes.bzl", "RECIPES")
 
+# These should reflect //ci/prebuilt/BUILD declared targets.
+BIND_TARGETS = [
+    "ares",
+    "event",
+    "event_pthreads",
+    "googletest",
+    "http_parser",
+    "lightstep",
+    "nghttp2",
+    "protobuf",
+    "protoc",
+    "rapidjson",
+    "spdlog",
+    "ssl",
+    "tclap",
+]
+
 def _repository_impl(ctxt):
     # Setup the build directory with links to the relevant files.
     ctxt.symlink(Label("//bazel:recipes.bzl"), "recipes.bzl")
@@ -36,7 +53,7 @@ def _repository_impl(ctxt):
         if result.return_code != 0:
             fail("External dep build failed")
 
-def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None):
+def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None, skip_bind = []):
     # Used only for protobuf.bzl.
     if local_protobuf_bzl:
         native.new_local_repository(
@@ -74,77 +91,9 @@ def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None):
         debug = debug_build,
     )
 
-    native.bind(
-        name = "ares",
-        actual = path + ":ares",
-    )
-
-    native.bind(
-        name = "cc_wkt_protos_genproto",
-        actual = path + ":cc_wkt_protos_genproto",
-    )
-
-    native.bind(
-        name = "cc_wkt_protos",
-        actual = path + ":cc_wkt_protos",
-    )
-
-    native.bind(
-        name = "event",
-        actual = path + ":event",
-    )
-
-    native.bind(
-        name = "event_pthreads",
-        actual = path + ":event_pthreads",
-    )
-
-    native.bind(
-        name = "googletest",
-        actual = path + ":googletest",
-    )
-
-    native.bind(
-        name = "http_parser",
-        actual = path + ":http_parser",
-    )
-
-    native.bind(
-        name = "lightstep",
-        actual = path + ":lightstep",
-    )
-
-    native.bind(
-        name = "nghttp2",
-        actual = path + ":nghttp2",
-    )
-
-    native.bind(
-        name = "protobuf",
-        actual = path + ":protobuf",
-    )
-
-    native.bind(
-        name = "protoc",
-        actual = path + ":protoc",
-    )
-
-    native.bind(
-        name = "rapidjson",
-        actual = path + ":rapidjson",
-    )
-
-    native.bind(
-        name = "spdlog",
-        actual = path + ":spdlog",
-    )
-
-    native.bind(
-        name = "ssl",
-        actual = path + ":ssl",
-    )
-
-    native.bind(
-        name = "tclap",
-        actual = path + ":tclap",
-    )
+    for t in BIND_TARGETS:
+        if t not in skip_bind:
+            native.bind(
+                name = t,
+                actual = path + ":" + t,
+            )

From 725667f9993f7475323b439b2b6c231adb17c874 Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Wed, 12 Apr 2017 23:09:03 -0400
Subject: [PATCH 5/8] Comment explaining why the monolithic repository
 dependency.

---
 bazel/repositories.bzl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
index 7bc6573899086..b4cb0fe3b7291 100644
--- a/bazel/repositories.bzl
+++ b/bazel/repositories.bzl
@@ -86,6 +86,9 @@ def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None, skip_b
         attrs = {"debug": attr.bool(default=False)},
     )
 
+    # Ideally, we wouldn't have a single repository target for all dependencies, but instead one per
+    # dependency, as suggested in #747. However, it's much faster to build all deps under a single
+    # recursive make job and single make jobserver.
     envoy_repository(
         name = "envoy_deps",
         debug = debug_build,

From 362fb8776b3d218c6a079f4eb342e448ae73d8fa Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Thu, 13 Apr 2017 05:10:25 -0400
Subject: [PATCH 6/8] Build performance profiling, some cleanups.

Support to compare sequential vs. parallel builds of deps.
---
 .gitignore                                   |  1 -
 bazel/EXTERNAL_DEPS.md                       | 13 ++---
 bazel/recipes.bzl                            | 16 ------
 bazel/repositories.bzl                       | 58 ++++++++------------
 bazel/repositories.sh                        | 26 ++++++++-
 bazel/target_recipes.bzl                     | 24 ++++++++
 ci/WORKSPACE                                 |  9 ++-
 ci/WORKSPACE.consumer                        |  5 +-
 ci/build_container/BUILD                     |  5 +-
 ci/build_container/Dockerfile                |  2 +-
 ci/build_container/Makefile                  |  4 +-
 ci/build_container/build_and_install_deps.sh | 13 ++++-
 ci/build_container/build_container.sh        |  8 ++-
 ci/build_container/docker_build.sh           |  6 ++
 ci/build_container/docker_build_container.sh |  4 --
 ci/build_container/print_recipe_deps.sh      |  3 -
 ci/build_container/recipe_wrapper.sh         |  6 ++
 ci/build_container/update_build_container.sh |  2 +-
 tools/build_profile.py                       | 21 +++++++
 19 files changed, 143 insertions(+), 83 deletions(-)
 delete mode 100644 bazel/recipes.bzl
 create mode 100644 bazel/target_recipes.bzl
 create mode 100755 ci/build_container/docker_build.sh
 delete mode 100755 ci/build_container/docker_build_container.sh
 delete mode 100755 ci/build_container/print_recipe_deps.sh
 create mode 100644 ci/build_container/recipe_wrapper.sh
 create mode 100755 tools/build_profile.py

diff --git a/.gitignore b/.gitignore
index d4b22ca90b0ec..0fc989004d778 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,6 @@
 /generated
 /bazel-*
 /ci/bazel-*
-/ci/build_container/recipes.bzl
 /ci/prebuilt/thirdparty
 /ci/prebuilt/thirdparty_build
 /test/coverage/BUILD
diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md
index 38aec4c2f3309..baac0c4244c16 100644
--- a/bazel/EXTERNAL_DEPS.md
+++ b/bazel/EXTERNAL_DEPS.md
@@ -1,14 +1,13 @@
 # Adding external dependencies to Envoy
 
 1. Specify name and version in [external documentation](../docs/install/requirements.rst).
-2. Add a build recipe in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes)
+2. Add a build recipe X in [`ci/build_container/build_recipes`](../ci/build_container/build_recipes)
    for developer-local and CI external dependency build flows.
-3. Add the build recipe to the list in [`bazel/recipes.bzl`](recipes.bzl).
-4. Add a build target X in [`ci/prebuilt/BUILD`](../ci/prebuilt/BUILD) to consume the headers and
-   libraries.
-5. Add a bind target in [`ci/repositories.bzl`](repositories.bzl#L4) to allow the new dependency to be
-   consumed by WORKSPACE.
-6. Reference your new external dependency in some `envoy_cc_library` via X in the `external_deps`
+3. Add the build recipe X to the list in [`bazel/recipes.bzl`](recipes.bzl).
+4. Add a build target Y in [`ci/prebuilt/BUILD`](../ci/prebuilt/BUILD) to consume the headers and
+   libraries produced by the build recipe X.
+5. Add a map from target Y to build recipe X in [`target_recipes.bzl`](target_recipes.bzl).
+6. Reference your new external dependency in some `envoy_cc_library` via Y in the `external_deps`
    attribute.
 7. `bazel test //test/...`
 
diff --git a/bazel/recipes.bzl b/bazel/recipes.bzl
deleted file mode 100644
index 3cb07d5ebb124..0000000000000
--- a/bazel/recipes.bzl
+++ /dev/null
@@ -1,16 +0,0 @@
-RECIPES = [
-    "boringssl",
-    "cares",
-    "cotire",
-    "gcovr",
-    "googletest",
-    "gperftools",
-    "http-parser",
-    "libevent",
-    "lightstep",
-    "nghttp2",
-    "protobuf",
-    "rapidjson",
-    "spdlog",
-    "tclap",
-]
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
index b4cb0fe3b7291..2e3f4010d6c17 100644
--- a/bazel/repositories.bzl
+++ b/bazel/repositories.bzl
@@ -1,31 +1,13 @@
-load(":recipes.bzl", "RECIPES")
-
-# These should reflect //ci/prebuilt/BUILD declared targets.
-BIND_TARGETS = [
-    "ares",
-    "event",
-    "event_pthreads",
-    "googletest",
-    "http_parser",
-    "lightstep",
-    "nghttp2",
-    "protobuf",
-    "protoc",
-    "rapidjson",
-    "spdlog",
-    "ssl",
-    "tclap",
-]
+load(":target_recipes.bzl", "TARGET_RECIPES")
 
 def _repository_impl(ctxt):
     # Setup the build directory with links to the relevant files.
-    ctxt.symlink(Label("//bazel:recipes.bzl"), "recipes.bzl")
     ctxt.symlink(Label("//bazel:repositories.sh"), "repositories.sh")
     ctxt.symlink(Label("//ci/build_container:build_and_install_deps.sh"),
                  "build_and_install_deps.sh")
-    ctxt.symlink(Label("//ci/build_container:print_recipe_deps.sh"), "print_recipe_deps.sh")
+    ctxt.symlink(Label("//ci/build_container:recipe_wrapper.sh"), "recipe_wrapper.sh")
     ctxt.symlink(Label("//ci/build_container:Makefile"), "Makefile")
-    for r in RECIPES:
+    for r in ctxt.attr.recipes:
         ctxt.symlink(Label("//ci/build_container/build_recipes:" + r + ".sh"),
                      "build_recipes/" + r + ".sh")
     ctxt.symlink(Label("//ci/prebuilt:BUILD"), "BUILD")
@@ -35,7 +17,7 @@ def _repository_impl(ctxt):
     if ctxt.attr.debug:
         environment["DEBUG"] = "1"
     result = ctxt.execute(
-        ["./repositories.sh"],
+        ["./repositories.sh"] + ctxt.attr.recipes,
         environment = environment,
         # Ideally we would print progress, but instead this hangs on "INFO: Loading
         # complete.  Analyzing.." today, see
@@ -53,16 +35,9 @@ def _repository_impl(ctxt):
         if result.return_code != 0:
             fail("External dep build failed")
 
-def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None, skip_bind = []):
+def envoy_dependencies(path = "@envoy_deps//", skip_protobuf_bzl = False, skip_targets = []):
     # Used only for protobuf.bzl.
-    if local_protobuf_bzl:
-        native.new_local_repository(
-            name = "protobuf_bzl",
-            path = "/thirdparty/protobuf-3.2.0",
-            # We only want protobuf.bzl, so don't support building out of this repo.
-            build_file_content = "",
-        )
-    else:
+    if not skip_protobuf_bzl:
         native.new_git_repository(
             name = "protobuf_bzl",
             # Using a non-canonical repository/branch here. This is a workaround to the lack of
@@ -82,20 +57,33 @@ def envoy_dependencies(path = "@envoy_deps//", local_protobuf_bzl = None, skip_b
     envoy_repository = repository_rule(
         implementation = _repository_impl,
         local = debug_build,
-        environ = ["CC", "CXX", "LD_LIBRARY_PATH"],
-        attrs = {"debug": attr.bool(default=False)},
+        environ = [
+            "CC",
+            "CXX",
+            "LD_LIBRARY_PATH"
+        ],
+        attrs = {
+            "debug": attr.bool(),
+            "recipes": attr.string_list(),
+        },
     )
 
     # Ideally, we wouldn't have a single repository target for all dependencies, but instead one per
     # dependency, as suggested in #747. However, it's much faster to build all deps under a single
     # recursive make job and single make jobserver.
+    recipes = depset()
+    for t in TARGET_RECIPES:
+        if t not in skip_targets:
+            recipes += depset([TARGET_RECIPES[t]])
+
     envoy_repository(
         name = "envoy_deps",
         debug = debug_build,
+        recipes = recipes.to_list(),
     )
 
-    for t in BIND_TARGETS:
-        if t not in skip_bind:
+    for t in TARGET_RECIPES:
+        if t not in skip_targets:
             native.bind(
                 name = t,
                 actual = path + ":" + t,
diff --git a/bazel/repositories.sh b/bazel/repositories.sh
index 469177a0d1ab3..499bd80b75467 100755
--- a/bazel/repositories.sh
+++ b/bazel/repositories.sh
@@ -8,15 +8,35 @@ set -e
 if [[ "${DEBUG}" == "1" ]]
 then
   BASEDIR=/tmp/bazel-envoy-deps
+  # Tell build_and_install_deps.sh to build sequentially when performance debugging.
+  # export BUILD_CONCURRENCY=0
 else
   BASEDIR="${PWD}/build"
 fi
 
+mkdir -p  "${BASEDIR}"
+
 export THIRDPARTY_DEPS="${BASEDIR}"
 export THIRDPARTY_SRC="${BASEDIR}/thirdparty"
 export THIRDPARTY_BUILD="${BASEDIR}/thirdparty_build"
 
-time ./build_and_install_deps.sh "${PREBUILT_DIR_REAL}"
+DEPS=""
+for r in "$@"
+do
+  DEPS="${DEPS} ${THIRDPARTY_DEPS}/$r.dep"
+done
+
+set -o pipefail
+(time ./build_and_install_deps.sh ${DEPS}) 2>&1 | \
+  tee "${BASEDIR}"/build.log
 
-ln -sf "$(realpath "${THIRDPARTY_SRC}")" thirdparty
-ln -sf "$(realpath "${THIRDPARTY_BUILD}")" thirdparty_build
+# Need to rsync in debug mode, since the symlinks are into /tmp and cause problems with later build
+# sandboxing.
+if [[ "${DEBUG}" == "1" ]]
+then
+  rsync -a "$(realpath "${THIRDPARTY_SRC}")"/ thirdparty
+  rsync -a "$(realpath "${THIRDPARTY_BUILD}")"/ thirdparty_build
+else
+  ln -sf "$(realpath "${THIRDPARTY_SRC}")" thirdparty
+  ln -sf "$(realpath "${THIRDPARTY_BUILD}")" thirdparty_build
+fi
diff --git a/bazel/target_recipes.bzl b/bazel/target_recipes.bzl
new file mode 100644
index 0000000000000..42ad772c22832
--- /dev/null
+++ b/bazel/target_recipes.bzl
@@ -0,0 +1,24 @@
+# These should reflect //ci/prebuilt/BUILD declared targets. This a map from
+# target in //ci/prebuilt/BUILD to the underlying build recipe in
+# ci/build_container/build_recipes.
+TARGET_RECIPES = {
+    "ares": "cares",
+    # TODO(htuch): Remove when cmake goes.
+    "cotire": "cotire",
+    "event": "libevent",
+    "event_pthreads": "libevent",
+    # TODO(htuch): This shouldn't be a build recipe, it's a tooling dependency
+    # that is external to Bazel.
+    "gcovr": "gcovr",
+    "googletest": "googletest",
+    "gperftools": "gperftools",
+    "http_parser": "http-parser",
+    "lightstep": "lightstep",
+    "nghttp2": "nghttp2",
+    "protobuf": "protobuf",
+    "protoc": "protobuf",
+    "rapidjson": "rapidjson",
+    "spdlog": "spdlog",
+    "ssl": "boringssl",
+    "tclap": "tclap",
+}
diff --git a/ci/WORKSPACE b/ci/WORKSPACE
index 15de7529dae10..a6d3da9fc7937 100644
--- a/ci/WORKSPACE
+++ b/ci/WORKSPACE
@@ -4,5 +4,12 @@ load("//bazel:repositories.bzl", "envoy_dependencies")
 
 envoy_dependencies(
     path = "//ci/prebuilt",
-    local_protobuf_bzl = "/thirdparty/protobuf-3.2.0",
+    skip_protobuf_bzl = True,
+)
+
+native.new_local_repository(
+    name = "protobuf_bzl",
+    path = "/thirdparty/protobuf-3.2.0",
+    # We only want protobuf.bzl, so don't support building out of this repo.
+    build_file_content = "",
 )
diff --git a/ci/WORKSPACE.consumer b/ci/WORKSPACE.consumer
index f54eb74abba48..e7102ae72c8c1 100644
--- a/ci/WORKSPACE.consumer
+++ b/ci/WORKSPACE.consumer
@@ -7,7 +7,4 @@ local_repository(
 
 load("//bazel:repositories.bzl", "envoy_dependencies")
 
-envoy_dependencies(
-    path = "@envoy//ci/prebuilt",
-    local_protobuf_bzl = "/thirdparty/protobuf-3.2.0",
-)
+envoy_dependencies(path = "@envoy//ci/prebuilt")
diff --git a/ci/build_container/BUILD b/ci/build_container/BUILD
index b2d5ae57432b3..4f42f1a03c997 100644
--- a/ci/build_container/BUILD
+++ b/ci/build_container/BUILD
@@ -1,3 +1,6 @@
 package(default_visibility = ["//visibility:public"])
 
-exports_files(["build_and_install_deps.sh", "Makefile"])
+exports_files([
+    "build_and_install_deps.sh",
+    "Makefile",
+])
diff --git a/ci/build_container/Dockerfile b/ci/build_container/Dockerfile
index 52f12707f82e5..4791b8e36515d 100644
--- a/ci/build_container/Dockerfile
+++ b/ci/build_container/Dockerfile
@@ -1,4 +1,4 @@
 FROM ubuntu:xenial
-COPY ./build_container.sh ./print_recipe_deps.sh ./build_and_install_deps.sh ./recipes.bzl ./Makefile /
+COPY ./build_container.sh ./build_and_install_deps.sh ./target_recipes.bzl ./recipe_wrapper.sh ./Makefile /
 COPY ./build_recipes/*.sh /build_recipes/
 RUN ./build_container.sh
diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile
index 27c95bb13f91b..14ad66addfb05 100644
--- a/ci/build_container/Makefile
+++ b/ci/build_container/Makefile
@@ -2,8 +2,6 @@
 # version number, etc.) to uniquely identify the revision of the upstream dependency.  This allows
 # make to pick up changes with a simple direct dependency on the build recipe.
 
-all: $(addprefix $(THIRDPARTY_DEPS)/,$(shell ./print_recipe_deps.sh))
-
 RECIPES := build_recipes
 
 # Make sure we use a consistent compiler across all deps.
@@ -38,7 +36,7 @@ build-recipe = cd $(THIRDPARTY_SRC) && \
 	   CC=$(CC) \
 	   CXX=$(CXX) \
 	   $(1) \
-	   bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \
+	   time bash $(CURDIR)/recipe_wrapper.sh $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \
 	$(build-complete)
 
 $(THIRDPARTY_DEPS)/%.dep: $(RECIPES)/%.sh
diff --git a/ci/build_container/build_and_install_deps.sh b/ci/build_container/build_and_install_deps.sh
index ad026f19ed0fd..26c12a01c1b27 100755
--- a/ci/build_container/build_and_install_deps.sh
+++ b/ci/build_container/build_and_install_deps.sh
@@ -7,4 +7,15 @@ mkdir -p "${THIRDPARTY_BUILD}"
 mkdir -p "${THIRDPARTY_SRC}"
 
 NUM_CPUS=`grep -c ^processor /proc/cpuinfo`
-make -C "$(dirname "$0")" -j "${NUM_CPUS}"
+
+# Invokers can set BUILD_CONCURRENCY=0 to ensure each build recipe is invoked sequentially, with all
+# CPU resources available. This is useful when debugging build performance.
+if [[ "${BUILD_CONCURRENCY}" == "0" ]]
+then
+  for dep in "$@"
+  do
+    make -C "$(dirname "$0")" -j "${NUM_CPUS}" "$dep"
+  done
+else
+  make -C "$(dirname "$0")" -j "${NUM_CPUS}" "$@"
+fi
diff --git a/ci/build_container/build_container.sh b/ci/build_container/build_container.sh
index 2b96d1ef82379..ce7045c339fd5 100755
--- a/ci/build_container/build_container.sh
+++ b/ci/build_container/build_container.sh
@@ -4,7 +4,8 @@ set -e
 
 # Setup basic requirements and install them.
 apt-get update
-apt-get install -y wget software-properties-common make cmake git python python-pip clang-format-3.6 bc libtool automake lcov zip
+apt-get install -y wget software-properties-common make cmake git python python-pip \
+  clang-format-3.6 bc libtool automake lcov zip time
 apt-get install -y golang
 # For debugging.
 apt-get install -y gdb strace
@@ -44,4 +45,7 @@ export CXX=g++-4.9
 export THIRDPARTY_DEPS=/tmp
 export THIRDPARTY_SRC=/thirdparty
 export THIRDPARTY_BUILD=/thirdparty_build
-"$(dirname "$0")"/build_and_install_deps.sh
+DEPS=$(python <(cat target_recipes.bzl; \
+  echo "print ' '.join(\"${THIRDPARTY_DEPS}/%s.dep\" % r for r in set(TARGET_RECIPES.values()))"))
+echo "Building deps ${DEPS}"
+"$(dirname "$0")"/build_and_install_deps.sh ${DEPS}
diff --git a/ci/build_container/docker_build.sh b/ci/build_container/docker_build.sh
new file mode 100755
index 0000000000000..e93e2aa50af68
--- /dev/null
+++ b/ci/build_container/docker_build.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# We need target_recipes.bzl for the build, but it's not in ci/build_container. Using Docker
+# relative path workaround from https://github.com/docker/docker/issues/2745#issuecomment-253230025
+# to get this to work.
+tar cf - . -C ../../bazel target_recipes.bzl | docker build --rm -t lyft/envoy-build:$TAG -
diff --git a/ci/build_container/docker_build_container.sh b/ci/build_container/docker_build_container.sh
deleted file mode 100755
index bb8dfee7844bd..0000000000000
--- a/ci/build_container/docker_build_container.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-cp ../../bazel/recipes.bzl .
-docker build --rm -t lyft/envoy-build:$TAG .
diff --git a/ci/build_container/print_recipe_deps.sh b/ci/build_container/print_recipe_deps.sh
deleted file mode 100755
index 65d4640412dff..0000000000000
--- a/ci/build_container/print_recipe_deps.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-python <(cat recipes.bzl; echo "print ' '.join(\"%s.dep\" % r for r in RECIPES)")
diff --git a/ci/build_container/recipe_wrapper.sh b/ci/build_container/recipe_wrapper.sh
new file mode 100644
index 0000000000000..f35844d7338d7
--- /dev/null
+++ b/ci/build_container/recipe_wrapper.sh
@@ -0,0 +1,6 @@
+PS4='+ $(date "+%s.%N") '
+set -x
+
+. $1
+
+echo DONE
diff --git a/ci/build_container/update_build_container.sh b/ci/build_container/update_build_container.sh
index 0c0e39edc47a9..bfdd519fc51d0 100755
--- a/ci/build_container/update_build_container.sh
+++ b/ci/build_container/update_build_container.sh
@@ -6,7 +6,7 @@ then
   TAG="$(git rev-parse master)"
   docker-machine start default
   eval $(docker-machine env default)
-  ./docker_build_container.sh
+  ./docker_build.sh
   docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
   docker push lyft/envoy-build:$TAG
   echo Pushed lyft/envoy-build:$TAG
diff --git a/tools/build_profile.py b/tools/build_profile.py
new file mode 100755
index 0000000000000..debf0411d70b8
--- /dev/null
+++ b/tools/build_profile.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+# This tool take the foo.dep.log output from a build recipe run under recipe_wrapper.sh on stdin and
+# produces a profile of command execution time on the stdout.
+
+import re
+import sys
+
+def PrintProfile(f):
+  prev_cmd = None
+  prev_timestamp = None
+  for line in f:
+    sr = re.match('\++ (\d+\.\d+) (.*)', line)
+    if sr:
+      timestamp, cmd = sr.groups()
+      if prev_cmd:
+        print '%.2f %s' % (float(timestamp) - float(prev_timestamp), prev_cmd)
+      prev_timestamp, prev_cmd = timestamp, cmd
+
+if __name__ == '__main__':
+  PrintProfile(sys.stdin)

From d3f20c1d7fb1550d0d0a65ca7904525d242abdb7 Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Thu, 13 Apr 2017 09:54:35 -0400
Subject: [PATCH 7/8] Fix bazel.coverage.

---
 ci/WORKSPACE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/WORKSPACE b/ci/WORKSPACE
index a6d3da9fc7937..1fb35678df75e 100644
--- a/ci/WORKSPACE
+++ b/ci/WORKSPACE
@@ -7,7 +7,7 @@ envoy_dependencies(
     skip_protobuf_bzl = True,
 )
 
-native.new_local_repository(
+new_local_repository(
     name = "protobuf_bzl",
     path = "/thirdparty/protobuf-3.2.0",
     # We only want protobuf.bzl, so don't support building out of this repo.

From bb7c733763fca1ede730af6699eebbe92563395e Mon Sep 17 00:00:00 2001
From: Harvey Tuch <htuch@google.com>
Date: Thu, 13 Apr 2017 15:01:22 -0400
Subject: [PATCH 8/8] Remove lcov.

---
 ci/build_container/build_container.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_container/build_container.sh b/ci/build_container/build_container.sh
index ce7045c339fd5..314319e13bb37 100755
--- a/ci/build_container/build_container.sh
+++ b/ci/build_container/build_container.sh
@@ -5,7 +5,7 @@ set -e
 # Setup basic requirements and install them.
 apt-get update
 apt-get install -y wget software-properties-common make cmake git python python-pip \
-  clang-format-3.6 bc libtool automake lcov zip time
+  clang-format-3.6 bc libtool automake zip time
 apt-get install -y golang
 # For debugging.
 apt-get install -y gdb strace