From 8661d583aa0eb7075482aa5eeca0b6a42837eb65 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 6 Apr 2017 18:17:18 -0400 Subject: [PATCH 1/2] ci: support building recipes in distinct --prefix directories (#415). The plan is to build the dependencies for the developer-local flow from the same build recipes as the CI image and using the same recursive make. This patch makes this is a bit easier, as one of the things we need to do is figure out which headers each dependency exports for Bazel, and building each recipe in its own --prefix makes this trivial. This is a nop for the CI Docker image. Validated by building a CI Docker image and performing a debug build/test against it. --- ci/build_container/Makefile | 24 ++++++++++++------- ci/build_container/build_recipes/lightstep.sh | 4 +++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile index a0518a312c22d..0eb3a7e9e14ce 100644 --- a/ci/build_container/Makefile +++ b/ci/build_container/Makefile @@ -1,8 +1,6 @@ # The individual build recipe scripts must contain sufficient information (e.g. SHA, URL, repo, # version number, etc.) to uniquely identify the revision of the upstream dependency. This allows # make to pick up changes with a simple direct dependency on the build recipe. -# TODO(htuch): These dependencies should be installed into distinct --prefix directories, so we can -# easily blow away the lib/ and include/ install directories when we upgrade version. all: $(THIRDPARTY_DEPS)/libevent.dep \ $(THIRDPARTY_DEPS)/boringssl.dep \ @@ -21,10 +19,14 @@ all: $(THIRDPARTY_DEPS)/libevent.dep \ RECIPES := build_recipes +DISTINCT_PATH = $(if $(BUILD_DISTINCT),$(@F),) + build-setup = rm -rf $@.build && \ + $(if $(BUILD_DISTINCT),rm -rf $(THIRDPARTY_BUILD)/$(DISTINCT_PATH) &&,) \ + $(if $(BUILD_DISTINCT),rm -rf $(THIRDPARTY_SRC)/$(DISTINCT_PATH) &&,) \ mkdir -p $@.build && \ - mkdir -p $(THIRDPARTY_BUILD)/lib && \ - mkdir -p $(THIRDPARTY_BUILD)/include && \ + mkdir -p $(THIRDPARTY_BUILD)/$(DISTINCT_PATH)/lib && \ + mkdir -p $(THIRDPARTY_BUILD)/$(DISTINCT_PATH)/include && \ cd $@.build && \ echo "Building in $@.build, logs at $@.log" @@ -33,12 +35,18 @@ build-complete = rm -rf $@.build && \ touch $@ build-recipe = cd $(THIRDPARTY_SRC) && \ - $(build-setup) && \ - (((bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \ + $(build-setup) && \ + (((THIRDPARTY_SRC=$(THIRDPARTY_SRC)/$(DISTINCT_PATH) \ + THIRDPARTY_BUILD=$(THIRDPARTY_BUILD)/$(DISTINCT_PATH) \ + $(1) \ + bash $(realpath $<) 2>&1) > $@.log) || (cat $@.log; exit 1)) && \ $(build-complete) $(THIRDPARTY_DEPS)/%.dep: $(RECIPES)/%.sh - @+$(build-recipe) + @+$(call build-recipe,) + +# Special support for targets that need protobuf, and hence take a dependency on protobuf.dep. +PROTOBUF_BUILD ?= $(THIRDPARTY_BUILD)/$(if $(BUILD_DISTINCT),protobuf.dep,) $(THIRDPARTY_DEPS)/lightstep.dep: $(RECIPES)/lightstep.sh $(THIRDPARTY_DEPS)/protobuf.dep - @+$(build-recipe) + @+$(call build-recipe,PROTOBUF_BUILD=$(PROTOBUF_BUILD)) diff --git a/ci/build_container/build_recipes/lightstep.sh b/ci/build_container/build_recipes/lightstep.sh index 15cd5e241b368..2bed19f5e7377 100644 --- a/ci/build_container/build_recipes/lightstep.sh +++ b/ci/build_container/build_recipes/lightstep.sh @@ -3,6 +3,8 @@ set -e wget https://github.com/lightstep/lightstep-tracer-cpp/releases/download/v0_36/lightstep-tracer-cpp-0.36.tar.gz tar xf lightstep-tracer-cpp-0.36.tar.gz cd lightstep-tracer-cpp-0.36 +# Added for legacy compatibility, should not be needed in new build recipes. +[ -z "$PROTOBUF_BUILD" ] && PROTOBUF_BUILD="$THIRDPARTY_BUILD" ./configure --disable-grpc --prefix=$THIRDPARTY_BUILD --enable-shared=no \ - protobuf_CFLAGS="-I$THIRDPARTY_BUILD/include" protobuf_LIBS="-L$THIRDPARTY_BUILD/lib -lprotobuf" PROTOC=$THIRDPARTY_BUILD/bin/protoc + protobuf_CFLAGS="-I$PROTOBUF_BUILD/include" protobuf_LIBS="-L$PROTOBUF_BUILD/lib -lprotobuf" PROTOC=$PROTOBUF_BUILD/bin/protoc make install From e4da9baeb271dd10f68078f6d17adab80ad5695b Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Fri, 7 Apr 2017 11:26:42 -0400 Subject: [PATCH 2/2] Some comments for the Makefile. --- ci/build_container/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/build_container/Makefile b/ci/build_container/Makefile index 0eb3a7e9e14ce..c3c570d54b917 100644 --- a/ci/build_container/Makefile +++ b/ci/build_container/Makefile @@ -19,6 +19,10 @@ all: $(THIRDPARTY_DEPS)/libevent.dep \ RECIPES := build_recipes +# If $(BUILD_DISTINCT) is set in the make environment, the artifacts are built and installed in +# distinct directories under $(THIRDPARTY_BUILD) and $(THIRDPARTY_SRC). They end up looking like +# $(THIRDPARTY_BUILD)/protobuf.dep/include, $(THIRDPARTY_BUILD)/cotire.dep/include etc. instead of +# all being under $(THIRDPARTY_BUILD)/include. DISTINCT_PATH = $(if $(BUILD_DISTINCT),$(@F),) build-setup = rm -rf $@.build && \ @@ -34,6 +38,8 @@ build-complete = rm -rf $@.build && \ echo "Successful build of $@" && \ touch $@ +# This needs to be invoked with $(call build-recipe,DEFS) where DEFS are additional environment +# definitions that are to be injected into the build recipe execution environment. build-recipe = cd $(THIRDPARTY_SRC) && \ $(build-setup) && \ (((THIRDPARTY_SRC=$(THIRDPARTY_SRC)/$(DISTINCT_PATH) \