diff --git a/.github/workflows/test.yml b/.github/workflows/langTest.yml similarity index 95% rename from .github/workflows/test.yml rename to .github/workflows/langTest.yml index 7f2f6c1..7bc95a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/langTest.yml @@ -34,4 +34,4 @@ jobs: run: ./test/test.sh rootless - name: Test Daemon - run: ./test/test.sh client + run: ./test/test.sh client \ No newline at end of file diff --git a/.github/workflows/startuptest.yml b/.github/workflows/startuptest.yml new file mode 100644 index 0000000..8102638 --- /dev/null +++ b/.github/workflows/startuptest.yml @@ -0,0 +1,38 @@ +name: Build and Test MetaCall Builder for startup flow +# Seperate because the space is limited in the github actions + +on: + push: + branches: + - master + tags: + - "v*.*.*" + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # TODO: + # - name: Lint + # run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all + + - name: Build + run: docker compose build + + - name: Test Binary + env: + BUILDER_ARGS: "runtime rb" + run: docker compose up --exit-code-from binary binary + + - name: Test Startup-Rootless + run: ./test/test.sh rootless startup + + - name: Test Startup-Daemon + run: ./test/test.sh client startup diff --git a/Dockerfile b/Dockerfile index 93bff58..4a7f5aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,7 @@ FROM moby/buildkit:master-rootless AS builder_rootless COPY --from=builder_binary --chown=user:user /builder /home/user/builder +# TO-DO : Replace with copying from scripts locally RUN printf '#!/bin/sh\n\ export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ /home/user/builder $@ | buildctl-daemonless.sh build \ @@ -49,6 +50,7 @@ COPY --from=builder_binary --chown=root:root /builder /home/builder RUN apk add --no-cache docker +# TO-DO : Replace with copying from scripts locally RUN printf '#!/bin/sh\n\ /home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build \ --export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ diff --git a/cmd/builder/flags.go b/cmd/builder/flags.go index dde37b6..97cbb52 100644 --- a/cmd/builder/flags.go +++ b/cmd/builder/flags.go @@ -6,8 +6,13 @@ import ( type RuntimeImageFlags struct { MetacallCli bool + Startup bool } func (i *RuntimeImageFlags) Set(cmd *cobra.Command) { cmd.Flags().BoolVar(&i.MetacallCli, "cli", false, "set to also get metacall cli in the runtime image") } + +func (i *RuntimeImageFlags) SetStartup(cmd *cobra.Command) { + cmd.Flags().BoolVar(&i.Startup, "startup", false, "startup flag to be used for building image with all languages") +} \ No newline at end of file diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index b3b2986..8728d00 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -21,15 +21,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Use: "runtime", Short: "Build runtime image for MetaCall", RunE: func(cmd *cobra.Command, args []string) error { + if o.RuntimeImageFlags.MetacallCli { args = append(args, "node") } + + if o.RuntimeImageFlags.Startup { + args = staging.GetAllLanguages() + } + base := cmd.Context().Value(baseKey{}).(llb.State) devBaseLang := staging.DevBase(base, branch, args) devImage := staging.MergeStates(devBaseLang) - runtimeLang:= staging.RuntimeBase(base, branch, args) + runtimeLang := staging.RuntimeBase(base, branch, args) runtimeImage := staging.MergeStates(runtimeLang) if o.RuntimeImageFlags.MetacallCli { @@ -40,7 +46,6 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { if err != nil { return err } - cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, runtime)) return nil @@ -48,6 +53,7 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Example: `"builder runtime -b develop --cli nodejs typescript go rust wasm java c cobol"`, } o.RuntimeImageFlags.Set(cmd) + o.RuntimeImageFlags.SetStartup(cmd) return cmd } diff --git a/docker-compose.yml b/docker-compose.yml index 973a7db..373d619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: entrypoint: /home/user/builder.sh ${BUILDER_ARGS:-} # entrypoint: /bin/sh -c "sleep infinity" environment: - IMPORT_REGISTRY: ${IMPORT_REGISTRY:-testing} - EXPORT_REGISTRY: ${EXPORT_REGISTRY:-testing} + IMPORT_REGISTRY: ${IMPORT_REGISTRY:-registry:5000/metacall/builder_startup} # Default won't be needed once we copy the script + EXPORT_REGISTRY: ${EXPORT_REGISTRY:-registry:5000/metacall/builder_dump} security_opt: - "seccomp=unconfined" - "apparmor=unconfined" diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index 710090c..2801777 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -6,9 +6,8 @@ import ( ) var languageMap = map[string]string{ - "base": "base", - "py": "python", - "rb": "ruby", + "py": "python", + "rb": "ruby", // "netcore": "netcore", // "netcore2": "netcore2", // "netcore5": "netcore5", @@ -22,15 +21,15 @@ var languageMap = map[string]string{ // "v8rep52": "v8rep52", // "v8rep51": "v8rep51", "node": "nodejs", - "ts": "typescript", + // "ts": "typescript", // "file": "file", // "rpc": "rpc", - "wasm": "wasm", - "java": "java", - "c": "c", - "cob": "cobol", - "go": "go", - "rs": "rust", + // "wasm": "wasm", + // "java": "java", + // // "c": "c", + // "cob": "cobol", + // "go": "go", + // "rs": "rust", // "backtrace" : "backtrace", // "sandbox" : "sandbox", } @@ -135,3 +134,11 @@ func MergeStates(individualLangStates map[string]llb.State) llb.State { } return llb.Merge(states) } + +func GetAllLanguages() []string { + langs := []string{} + for lang := range languageMap { + langs = append(langs, lang) + } + return langs +} diff --git a/hack/buildctl.sh b/scripts/buildctl.sh similarity index 100% rename from hack/buildctl.sh rename to scripts/buildctl.sh diff --git a/scripts/builder.sh b/scripts/builder.sh new file mode 100644 index 0000000..56b224f --- /dev/null +++ b/scripts/builder.sh @@ -0,0 +1,20 @@ +#!/bin/sh\n\ +export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ + +if [ -z "$EXPORT_REGISTRY" ]; then + EXPORT_REGISTRY="registry:5000/metacall/builder_cache" +fi + +if [ -z "$IMPORT_REGISTRY" ]; then + IMPORT_REGISTRY="registry:5000/metacall/builder_startup" +fi + +if [ -z "$BUILDER_BINARY" ]; then + BUILDER_BINARY="/home/user/builder" +fi + + +${BUILDER_BINARY} $@ | buildctl-daemonless.sh build \ +--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ +--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \ +--output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\ \ No newline at end of file diff --git a/test/test.sh b/test/test.sh index b41652b..a11b581 100755 --- a/test/test.sh +++ b/test/test.sh @@ -16,36 +16,46 @@ else fi DOCKER_SERVICE=${1:-rootless} +TEST_TYPE=${2:-default} build() { ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} } -test() { - build - +setupRegistry(){ + ${DOCKER_CMD} up -d registry while [ ! "$(docker inspect --format '{{json .State.Health.Status }}' metacall_builder_registry)" = "\"healthy\"" ]; do sleep 5 done +} +cleanup(){ + ${DOCKER_CMD} down +} + +test() { + + build + setupRegistry + first_arg=$(echo $BUILDER_ARGS | cut -d ' ' -f 1) DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output_$first_arg sh -c "metacallcli test/$1"` DOCKER_OUTPUT=`echo ${DOCKER_OUTPUT} | tr -d '\r\n'` EXPECTED_OUTPUT=`echo $2 | tr -d '\r\n'` + TEST_NAME=`echo $1` if [ "${DOCKER_OUTPUT}" = "${EXPECTED_OUTPUT}" ]; then - echo "Test passed: $1" + echo "Test passed: ${TEST_NAME}" else - echo "Failed to run test: $1" + echo "Failed to run test: ${TEST_NAME}" echo "Expected output was: '${EXPECTED_OUTPUT}'" echo "Test output was: '${DOCKER_OUTPUT}'" exit 1 fi - ${DOCKER_CMD} down } # TODO: @@ -59,16 +69,44 @@ test() { # done # done -# Build the dev image with NodeJS language -echo "Building dev mode with NodeJS language." -export BUILDER_ARGS="dev node" -export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" -export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" -test node/test.js "0123456789" - -# Build the cli image with languages all together -echo "Building cli mode with all languages." -export BUILDER_ARGS="runtime --cli py node rb" -export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" -export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" -test node/test.js "0123456789" +defaultTests(){ + # Build the dev image with NodeJS language + echo "Building dev mode with NodeJS language." + export BUILDER_ARGS="dev node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" + export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" + test node/test.js "0123456789" + cleanup + + # Build the cli image with languages all together + echo "Building cli mode with all languages." + export BUILDER_ARGS="runtime --cli py node rb" + export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" + export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" + test node/test.js "0123456789" + cleanup +} + +startupTests(){ + # Build the startup image with all languages + echo "Building all languages in startup mode." + export BUILDER_ARGS="runtime --cli --startup" + export EXPORT_REGISTRY="registry:5000/metacall/builder_startup" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + test node/test.js "0123456789" + + + # Testing the cache registry + echo "Building cli mode with node and py languages." + export BUILDER_ARGS="runtime --cli py node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + export EXPORT_REGISTRY="registry:5000/metacall/builder_dump" # To not able to rewrite the cache + test node/test.js "0123456789" # Should be quicker since all caches are already built + cleanup +} + +if [ "${TEST_TYPE}" = "default" ]; then + defaultTests +elif [ "${TEST_TYPE}" = "startup" ]; then + startupTests +fi