From 5a0be01147c225d13c99c14bf4dc2760827d92e8 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 05:30:20 +0530 Subject: [PATCH 01/32] set the startup flag --- cmd/builder/flags.go | 5 +++++ cmd/builder/runtime.go | 10 ++++++++-- pkg/staging/stages.go | 8 ++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cmd/builder/flags.go b/cmd/builder/flags.go index dde37b6..408bca2 100644 --- a/cmd/builder/flags.go +++ b/cmd/builder/flags.go @@ -6,8 +6,13 @@ import ( type RuntimeImageFlags struct { MetacallCli bool + Startup string } func (i *RuntimeImageFlags) Set(cmd *cobra.Command) { cmd.Flags().BoolVar(&i.MetacallCli, "cli", false, "set to also get metacall cli in the runtime image") } + +func (i *RuntimeImageFlags) SetStartup(cmd *cobra.Command) { + cmd.Flags().StringVar(&i.Startup, "startup", "", "startup flag to be used for building image with all languages") +} \ No newline at end of file diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index b3b2986..f9920f5 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -21,15 +21,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Use: "runtime", Short: "Build runtime image for MetaCall", RunE: func(cmd *cobra.Command, args []string) error { + if o.RuntimeImageFlags.MetacallCli { args = append(args, "node") } + + if o.RuntimeImageFlags.Startup != "" { + args = staging.GetAllLanguages() + } + base := cmd.Context().Value(baseKey{}).(llb.State) devBaseLang := staging.DevBase(base, branch, args) devImage := staging.MergeStates(devBaseLang) - runtimeLang:= staging.RuntimeBase(base, branch, args) + runtimeLang := staging.RuntimeBase(base, branch, args) runtimeImage := staging.MergeStates(runtimeLang) if o.RuntimeImageFlags.MetacallCli { @@ -40,7 +46,6 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { if err != nil { return err } - cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, runtime)) return nil @@ -48,6 +53,7 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Example: `"builder runtime -b develop --cli nodejs typescript go rust wasm java c cobol"`, } o.RuntimeImageFlags.Set(cmd) + o.RuntimeImageFlags.SetStartup(cmd) return cmd } diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index 710090c..ff49446 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -135,3 +135,11 @@ func MergeStates(individualLangStates map[string]llb.State) llb.State { } return llb.Merge(states) } + +func GetAllLanguages() []string { + langs := []string{} + for lang := range languageMap { + langs = append(langs, lang) + } + return langs +} \ No newline at end of file From ec2cba0af8e644dd475b4dca7fb47730e7e675a3 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 05:30:56 +0530 Subject: [PATCH 02/32] Add tests support for startup & refactoring --- .github/workflows/test.yml | 14 ++++-- Dockerfile | 2 + docker-compose.yml | 4 +- {hack => scripts}/buildctl.sh | 0 scripts/builder.sh | 23 +++++++++ test/test.sh | 88 +++++++++++++++++++++++++---------- 6 files changed, 100 insertions(+), 31 deletions(-) rename {hack => scripts}/buildctl.sh (100%) create mode 100644 scripts/builder.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f2f6c1..6b30472 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,8 +30,14 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - - name: Test Rootless - run: ./test/test.sh rootless + # - name: Test Rootless + # run: ./test/test.sh rootless - - name: Test Daemon - run: ./test/test.sh client + # - name: Test Daemon + # run: ./test/test.sh client + + # - name: Test Startup-Rootless + # run: ./test/test.sh rootless startup + + - name: Test Startup-Daemon + run: ./test/test.sh client startup diff --git a/Dockerfile b/Dockerfile index 8b9d7d8..6a2ff7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,7 @@ FROM moby/buildkit:master-rootless AS builder_rootless COPY --from=builder_binary --chown=user:user /builder /home/user/builder +# TO-DO : Replace with copying from scripts locally RUN printf '#!/bin/sh\n\ export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ /home/user/builder $@ | buildctl-daemonless.sh build \ @@ -49,6 +50,7 @@ COPY --from=builder_binary --chown=root:root /builder /home/builder RUN apk add --no-cache docker +# TO-DO : Replace with copying from scripts locally RUN printf '#!/bin/sh\n\ /home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build \ --export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ diff --git a/docker-compose.yml b/docker-compose.yml index f36247f..f3f8f08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: entrypoint: /home/user/builder.sh ${BUILDER_ARGS:-} # entrypoint: /bin/sh -c "sleep infinity" environment: - IMPORT_REGISTRY: ${IMPORT_REGISTRY:-testing} - EXPORT_REGISTRY: ${EXPORT_REGISTRY:-testing} + IMPORT_REGISTRY: ${IMPORT_REGISTRY:-registry:5000/metacall/builder_startup} # Default won't be needed once we copy the script + EXPORT_REGISTRY: ${EXPORT_REGISTRY:-registry:5000/metacall/builder_dump} security_opt: - "seccomp=unconfined" - "apparmor=unconfined" diff --git a/hack/buildctl.sh b/scripts/buildctl.sh similarity index 100% rename from hack/buildctl.sh rename to scripts/buildctl.sh diff --git a/scripts/builder.sh b/scripts/builder.sh new file mode 100644 index 0000000..2bcfbe7 --- /dev/null +++ b/scripts/builder.sh @@ -0,0 +1,23 @@ +#!/bin/sh\n\ +export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ + +if [ -z "$EXPORT_REGISTRY" ] +then + EXPORT_REGISTRY="registry:5000/metacall/builder_cache" +fi + +if [ -z "$IMPORT_REGISTRY" ] +then + IMPORT_REGISTRY="registry:5000/metacall/builder_startup" +fi + +if [ -z "$BUILDER_BINARY" ] +then + BUILDER_BINARY="/home/user/builder" +fi + + +${BUILDER_BINARY} $@ | buildctl-daemonless.sh build \ +--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ +--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \ +--output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\ \ No newline at end of file diff --git a/test/test.sh b/test/test.sh index 6ad66ea..efed7a3 100755 --- a/test/test.sh +++ b/test/test.sh @@ -21,31 +21,46 @@ build() { ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} } -test() { - build - +setupRegistry(){ + ${DOCKER_CMD} up -d registry while [ ! "$(docker inspect --format '{{json .State.Health.Status }}' metacall_builder_registry)" = "\"healthy\"" ]; do sleep 5 done +} - DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output sh -c "metacallcli test/$1"` - DOCKER_OUTPUT=`echo ${DOCKER_OUTPUT} | tr -d '\r\n'` - EXPECTED_OUTPUT=`echo $2 | tr -d '\r\n'` - - if [ "${DOCKER_OUTPUT}" = "${EXPECTED_OUTPUT}" ]; then - echo "Test passed: $1" +checkOutput(){ + + if [ "$1" = "$2" ]; then + echo "Test passed: $3" else - echo "Failed to run test: $1" - echo "Expected output was: '${EXPECTED_OUTPUT}'" - echo "Test output was: '${DOCKER_OUTPUT}'" + echo "Failed to run test: $3" + echo "Expected output was: '$2'" + echo "Test output was: '$1'" exit 1 fi +} + +cleanup(){ ${DOCKER_CMD} down } +test() { + + build + setupRegistry + + DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output sh -c "metacallcli test/$1"` + DOCKER_OUTPUT=`echo ${DOCKER_OUTPUT} | tr -d '\r\n'` + EXPECTED_OUTPUT=`echo $2 | tr -d '\r\n'` + TEST_NAME=`echo $1` + + checkOutput ${DOCKER_OUTPUT} ${EXPECTED_OUTPUT} ${TEST_NAME} + +} + # TODO: # # Build them separately # for mode in deps dev runtime; do @@ -57,16 +72,39 @@ test() { # done # done -# Build the dev image with NodeJS language -echo "Building dev mode with NodeJS language." -export BUILDER_ARGS="dev node" -export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" -export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" -test node/test.js "0123456789" - -# Build the cli image with languages all together -echo "Building cli mode with all languages." -export BUILDER_ARGS="runtime --cli py node rb" -export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" -export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" -test node/test.js "0123456789" +defaultTests(){ + # Build the dev image with NodeJS language + echo "Building dev mode with NodeJS language." + export BUILDER_ARGS="dev node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" + export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" + test node/test.js "0123456789" + cleanup + + # Build the cli image with languages all together + echo "Building cli mode with all languages." + export BUILDER_ARGS="runtime --cli py node rb" + export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" + export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" + test node/test.js "0123456789" + cleanup +} + +startupTests(){ + echo "Building all languages in startup mode." + export BUILDER_ARGS="runtime --startup" + export EXPORT_REGISTRY="registry:5000/metacall/builder_startup" + # Import registry set to by default + test node/test.js "0123456789" + + echo "Building all languages in startup mode with cache in local registry." + test node/test.js "0123456789" # Should be quicker since all caches are already built + cleanup +} + +if [ "$2" == "startup" ] +then + startupTests() +else + defaultTests() +fi From 3d8741e6e47a938619de4c37195040ea30aa27d7 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 05:33:47 +0530 Subject: [PATCH 03/32] testing --- test/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index efed7a3..83c9a49 100755 --- a/test/test.sh +++ b/test/test.sh @@ -101,8 +101,9 @@ startupTests(){ test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } +echo "$2" -if [ "$2" == "startup" ] +if [ -z "$2" ] then startupTests() else From 26f15d151dbe3a4a2819c3cfd8598c84055a93d0 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 05:34:50 +0530 Subject: [PATCH 04/32] testing --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b30472..628c9fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,13 @@ jobs: # - name: Lint # run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all - - name: Build - run: docker compose build + # - name: Build + # run: docker compose build - - name: Test Binary - env: - BUILDER_ARGS: "runtime rb" - run: docker compose up --exit-code-from binary binary + # - name: Test Binary + # env: + # BUILDER_ARGS: "runtime rb" + # run: docker compose up --exit-code-from binary binary # - name: Test Rootless # run: ./test/test.sh rootless @@ -36,8 +36,8 @@ jobs: # - name: Test Daemon # run: ./test/test.sh client - # - name: Test Startup-Rootless - # run: ./test/test.sh rootless startup + - name: Test Startup-Rootless + run: ./test/test.sh rootless startup - name: Test Startup-Daemon run: ./test/test.sh client startup From 900787ff126ea0188d59f71679ec2b4ee7c01fd0 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 05:36:12 +0530 Subject: [PATCH 05/32] testing --- test/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index 83c9a49..0e91fb1 100755 --- a/test/test.sh +++ b/test/test.sh @@ -103,8 +103,7 @@ startupTests(){ } echo "$2" -if [ -z "$2" ] -then +if [ -z "$2" ]; then startupTests() else defaultTests() From dea15e507e24153d8eb2f7a129ab7150d4f15829 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 05:37:47 +0530 Subject: [PATCH 06/32] testing --- scripts/builder.sh | 9 +++------ test/test.sh | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/builder.sh b/scripts/builder.sh index 2bcfbe7..56b224f 100644 --- a/scripts/builder.sh +++ b/scripts/builder.sh @@ -1,18 +1,15 @@ #!/bin/sh\n\ export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ -if [ -z "$EXPORT_REGISTRY" ] -then +if [ -z "$EXPORT_REGISTRY" ]; then EXPORT_REGISTRY="registry:5000/metacall/builder_cache" fi -if [ -z "$IMPORT_REGISTRY" ] -then +if [ -z "$IMPORT_REGISTRY" ]; then IMPORT_REGISTRY="registry:5000/metacall/builder_startup" fi -if [ -z "$BUILDER_BINARY" ] -then +if [ -z "$BUILDER_BINARY" ]; then BUILDER_BINARY="/home/user/builder" fi diff --git a/test/test.sh b/test/test.sh index 0e91fb1..c1a0f17 100755 --- a/test/test.sh +++ b/test/test.sh @@ -104,7 +104,7 @@ startupTests(){ echo "$2" if [ -z "$2" ]; then - startupTests() + startupTests else - defaultTests() + defaultTests fi From 1986560c85b376a0e47eeb0407e0f0f12715cf7b Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:07:26 +0530 Subject: [PATCH 07/32] testing --- .github/workflows/test.yml | 12 ++++++------ test/test.sh | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 628c9fa..a314805 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,13 @@ jobs: # - name: Lint # run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all - # - name: Build - # run: docker compose build + - name: Build + run: docker compose build - # - name: Test Binary - # env: - # BUILDER_ARGS: "runtime rb" - # run: docker compose up --exit-code-from binary binary + - name: Test Binary + env: + BUILDER_ARGS: "runtime rb" + run: docker compose up --exit-code-from binary binary # - name: Test Rootless # run: ./test/test.sh rootless diff --git a/test/test.sh b/test/test.sh index c1a0f17..efed7a3 100755 --- a/test/test.sh +++ b/test/test.sh @@ -101,10 +101,10 @@ startupTests(){ test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } -echo "$2" -if [ -z "$2" ]; then - startupTests +if [ "$2" == "startup" ] +then + startupTests() else - defaultTests + defaultTests() fi From de734f0ebf44e8261cdc71c2a7b42ba25c5dccf0 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:08:26 +0530 Subject: [PATCH 08/32] testing --- test/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index efed7a3..819ef5f 100755 --- a/test/test.sh +++ b/test/test.sh @@ -104,7 +104,7 @@ startupTests(){ if [ "$2" == "startup" ] then - startupTests() + startupTests else - defaultTests() + defaultTests fi From 3827c3fad757afa0b908924f18d1b549ef3502e7 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:13:10 +0530 Subject: [PATCH 09/32] testing --- test/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index 819ef5f..740fc0a 100755 --- a/test/test.sh +++ b/test/test.sh @@ -101,8 +101,8 @@ startupTests(){ test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } - -if [ "$2" == "startup" ] +echo $2 +if [ "$2" == "startup" ]; then startupTests else From 09600b0b0ffd932ef62e79ecb9ca549471309c7e Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:14:54 +0530 Subject: [PATCH 10/32] testing --- test/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test.sh b/test/test.sh index 740fc0a..be394f5 100755 --- a/test/test.sh +++ b/test/test.sh @@ -106,5 +106,6 @@ if [ "$2" == "startup" ]; then startupTests else + echo "laude lag gye bhai" defaultTests fi From 7e64702e882d43985ed5e3407ca6fe82f4a6299e Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:16:34 +0530 Subject: [PATCH 11/32] testing --- test/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test.sh b/test/test.sh index be394f5..c71ebeb 100755 --- a/test/test.sh +++ b/test/test.sh @@ -102,6 +102,7 @@ startupTests(){ cleanup } echo $2 +echo "hiii" if [ "$2" == "startup" ]; then startupTests From cffdc7ad44b6ca687887c695859bc92f48d80f03 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:18:17 +0530 Subject: [PATCH 12/32] testing --- test/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index c71ebeb..b225258 100755 --- a/test/test.sh +++ b/test/test.sh @@ -103,8 +103,7 @@ startupTests(){ } echo $2 echo "hiii" -if [ "$2" == "startup" ]; -then +if [ "$2" == "startup" ]; then startupTests else echo "laude lag gye bhai" From 7e1ec98bdce848f243e92fcbe300d1c998a3bfe0 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:21:26 +0530 Subject: [PATCH 13/32] was sh all along --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index b225258..43d9b4e 100755 --- a/test/test.sh +++ b/test/test.sh @@ -103,7 +103,7 @@ startupTests(){ } echo $2 echo "hiii" -if [ "$2" == "startup" ]; then +if [ "$2" = "startup" ]; then startupTests else echo "laude lag gye bhai" From 52d516f64f170fcf5cbdabcba7b5c48b221dc659 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:25:29 +0530 Subject: [PATCH 14/32] fix startup string to bool --- cmd/builder/flags.go | 4 ++-- cmd/builder/runtime.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/builder/flags.go b/cmd/builder/flags.go index 408bca2..97cbb52 100644 --- a/cmd/builder/flags.go +++ b/cmd/builder/flags.go @@ -6,7 +6,7 @@ import ( type RuntimeImageFlags struct { MetacallCli bool - Startup string + Startup bool } func (i *RuntimeImageFlags) Set(cmd *cobra.Command) { @@ -14,5 +14,5 @@ func (i *RuntimeImageFlags) Set(cmd *cobra.Command) { } func (i *RuntimeImageFlags) SetStartup(cmd *cobra.Command) { - cmd.Flags().StringVar(&i.Startup, "startup", "", "startup flag to be used for building image with all languages") + cmd.Flags().BoolVar(&i.Startup, "startup", false, "startup flag to be used for building image with all languages") } \ No newline at end of file diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index f9920f5..8728d00 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -26,7 +26,7 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { args = append(args, "node") } - if o.RuntimeImageFlags.Startup != "" { + if o.RuntimeImageFlags.Startup { args = staging.GetAllLanguages() } From e5628ac39a04e74b53fc5a08af733d7fc6220ac9 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:30:00 +0530 Subject: [PATCH 15/32] removing ts for testing as building it alone fails --- pkg/staging/stages.go | 2 +- test/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index ff49446..f501f6c 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -22,7 +22,7 @@ var languageMap = map[string]string{ // "v8rep52": "v8rep52", // "v8rep51": "v8rep51", "node": "nodejs", - "ts": "typescript", + // "ts": "typescript", // "file": "file", // "rpc": "rpc", "wasm": "wasm", diff --git a/test/test.sh b/test/test.sh index 43d9b4e..b404f14 100755 --- a/test/test.sh +++ b/test/test.sh @@ -97,6 +97,7 @@ startupTests(){ # Import registry set to by default test node/test.js "0123456789" + # Flush env's depedning upon the mode echo "Building all languages in startup mode with cache in local registry." test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup From 57e62f09098a09af431a310faad7bb1f1a480b8e Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 06:34:23 +0530 Subject: [PATCH 16/32] runtime error while building c --- pkg/staging/stages.go | 2 +- test/test.sh | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index f501f6c..f29b521 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -27,7 +27,7 @@ var languageMap = map[string]string{ // "rpc": "rpc", "wasm": "wasm", "java": "java", - "c": "c", + // "c": "c", "cob": "cobol", "go": "go", "rs": "rust", diff --git a/test/test.sh b/test/test.sh index b404f14..21f2ec5 100755 --- a/test/test.sh +++ b/test/test.sh @@ -97,16 +97,14 @@ startupTests(){ # Import registry set to by default test node/test.js "0123456789" - # Flush env's depedning upon the mode + # TODO : Flush env's depedning upon the mode echo "Building all languages in startup mode with cache in local registry." test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } -echo $2 -echo "hiii" + if [ "$2" = "startup" ]; then startupTests else - echo "laude lag gye bhai" defaultTests fi From e5922278b9bf3e313a0b400f08cc5a707a532e72 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 07:08:53 +0530 Subject: [PATCH 17/32] now go failed --- pkg/staging/stages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index f29b521..1c466c7 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -29,7 +29,7 @@ var languageMap = map[string]string{ "java": "java", // "c": "c", "cob": "cobol", - "go": "go", + // "go": "go", "rs": "rust", // "backtrace" : "backtrace", // "sandbox" : "sandbox", From 3849bd7558d1dba062c454295bada6a7140e6e0a Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 07:26:21 +0530 Subject: [PATCH 18/32] rust failed --- pkg/staging/stages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index 1c466c7..f56e519 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -30,7 +30,7 @@ var languageMap = map[string]string{ // "c": "c", "cob": "cobol", // "go": "go", - "rs": "rust", + // "rs": "rust", // "backtrace" : "backtrace", // "sandbox" : "sandbox", } From 86d5a8cd64ef69d490e47de07d4696f38eeea0b1 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 07:51:50 +0530 Subject: [PATCH 19/32] just py,rb,node --- pkg/staging/stages.go | 15 +++++++-------- test/test.sh | 26 +++++++++++--------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index f56e519..2801777 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -6,9 +6,8 @@ import ( ) var languageMap = map[string]string{ - "base": "base", - "py": "python", - "rb": "ruby", + "py": "python", + "rb": "ruby", // "netcore": "netcore", // "netcore2": "netcore2", // "netcore5": "netcore5", @@ -25,10 +24,10 @@ var languageMap = map[string]string{ // "ts": "typescript", // "file": "file", // "rpc": "rpc", - "wasm": "wasm", - "java": "java", - // "c": "c", - "cob": "cobol", + // "wasm": "wasm", + // "java": "java", + // // "c": "c", + // "cob": "cobol", // "go": "go", // "rs": "rust", // "backtrace" : "backtrace", @@ -142,4 +141,4 @@ func GetAllLanguages() []string { langs = append(langs, lang) } return langs -} \ No newline at end of file +} diff --git a/test/test.sh b/test/test.sh index 21f2ec5..0bab029 100755 --- a/test/test.sh +++ b/test/test.sh @@ -30,19 +30,6 @@ setupRegistry(){ done } -checkOutput(){ - - if [ "$1" = "$2" ]; then - echo "Test passed: $3" - else - echo "Failed to run test: $3" - echo "Expected output was: '$2'" - echo "Test output was: '$1'" - exit 1 - fi - -} - cleanup(){ ${DOCKER_CMD} down } @@ -57,7 +44,14 @@ test() { EXPECTED_OUTPUT=`echo $2 | tr -d '\r\n'` TEST_NAME=`echo $1` - checkOutput ${DOCKER_OUTPUT} ${EXPECTED_OUTPUT} ${TEST_NAME} + if [ "${DOCKER_OUTPUT}" = "${EXPECTED_OUTPUT}" ]; then + echo "Test passed: ${TEST_NAME}" + else + echo "Failed to run test: ${TEST_NAME}" + echo "Expected output was: '${EXPECTED_OUTPUT}'" + echo "Test output was: '${DOCKER_OUTPUT}'" + exit 1 + fi } @@ -98,7 +92,9 @@ startupTests(){ test node/test.js "0123456789" # TODO : Flush env's depedning upon the mode - echo "Building all languages in startup mode with cache in local registry." + echo "Building cli mode with node and py languages." + export BUILDER_ARGS="runtime --cli py node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } From 84ad0a9cffc9906505538a9d2ca8973957c8242b Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 08:22:54 +0530 Subject: [PATCH 20/32] just py,rb,node --- test/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test.sh b/test/test.sh index 0bab029..bba0c8d 100755 --- a/test/test.sh +++ b/test/test.sh @@ -92,10 +92,10 @@ startupTests(){ test node/test.js "0123456789" # TODO : Flush env's depedning upon the mode - echo "Building cli mode with node and py languages." - export BUILDER_ARGS="runtime --cli py node" - export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" - test node/test.js "0123456789" # Should be quicker since all caches are already built + # echo "Building cli mode with node and py languages." + # export BUILDER_ARGS="runtime --cli py node" + # export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + # test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } From b3c8e0ebf3a8401dfb23c3b1b2ba709ff7ab4233 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:17:55 +0530 Subject: [PATCH 21/32] trigger all the tests --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a314805..a5e2f4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - # - name: Test Rootless - # run: ./test/test.sh rootless + - name: Test Rootless + run: ./test/test.sh rootless - # - name: Test Daemon - # run: ./test/test.sh client + - name: Test Daemon + run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup From 7296e58a3b84c797369911107aaa8d859bfda149 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:21:04 +0530 Subject: [PATCH 22/32] trigger all the tests --- .github/workflows/test.yml | 8 ++++---- cmd/builder/runtime.go | 3 +++ test/test.sh | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5e2f4a..a314805 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - - name: Test Rootless - run: ./test/test.sh rootless + # - name: Test Rootless + # run: ./test/test.sh rootless - - name: Test Daemon - run: ./test/test.sh client + # - name: Test Daemon + # run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index 8728d00..5ce0cd2 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -2,6 +2,7 @@ package builder import ( "context" + "fmt" "github.com/metacall/builder/pkg/staging" "github.com/moby/buildkit/client/llb" @@ -30,6 +31,8 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { args = staging.GetAllLanguages() } + fmt.Println(args) + base := cmd.Context().Value(baseKey{}).(llb.State) devBaseLang := staging.DevBase(base, branch, args) diff --git a/test/test.sh b/test/test.sh index bba0c8d..fc877e1 100755 --- a/test/test.sh +++ b/test/test.sh @@ -99,8 +99,8 @@ startupTests(){ cleanup } -if [ "$2" = "startup" ]; then - startupTests -else +if [ -z "$2" ]; then defaultTests +elif [ "$2" = "startup" ]; then + startupTests fi From e862445e6b95939961e6c66ca5d8b881019d2701 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:22:54 +0530 Subject: [PATCH 23/32] trigger all the tests --- cmd/builder/runtime.go | 3 +-- test/test.sh | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index 5ce0cd2..aa6832d 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -2,7 +2,6 @@ package builder import ( "context" - "fmt" "github.com/metacall/builder/pkg/staging" "github.com/moby/buildkit/client/llb" @@ -31,7 +30,7 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { args = staging.GetAllLanguages() } - fmt.Println(args) + // fmt.Println(args) base := cmd.Context().Value(baseKey{}).(llb.State) diff --git a/test/test.sh b/test/test.sh index fc877e1..9bb01e4 100755 --- a/test/test.sh +++ b/test/test.sh @@ -88,6 +88,8 @@ startupTests(){ echo "Building all languages in startup mode." export BUILDER_ARGS="runtime --startup" export EXPORT_REGISTRY="registry:5000/metacall/builder_startup" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + # Import registry set to by default test node/test.js "0123456789" From 4e59b6a1e84955a984d770bec1f1a3d0df102344 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:23:16 +0530 Subject: [PATCH 24/32] trigger all the tests --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a314805..a5e2f4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - # - name: Test Rootless - # run: ./test/test.sh rootless + - name: Test Rootless + run: ./test/test.sh rootless - # - name: Test Daemon - # run: ./test/test.sh client + - name: Test Daemon + run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup From 0851dab7ab485cf3423172c101075dafa32e648e Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:25:52 +0530 Subject: [PATCH 25/32] trigger all the tests --- test/test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test.sh b/test/test.sh index 9bb01e4..c143210 100755 --- a/test/test.sh +++ b/test/test.sh @@ -16,6 +16,7 @@ else fi DOCKER_SERVICE=${1:-rootless} +TEST_TYPE=${2:-default} build() { ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} @@ -101,8 +102,8 @@ startupTests(){ cleanup } -if [ -z "$2" ]; then +if [ "${TEST_TYPE}" = "default" ]; then defaultTests -elif [ "$2" = "startup" ]; then +elif [ "${TEST_TYPE}" = "startup" ]; then startupTests fi From c5088de26185fd2907f04fc9e83d03c7bb92f7de Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:33:57 +0530 Subject: [PATCH 26/32] trigger all the tests startup with cli, i am soooo dumb --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index c143210..75b7248 100755 --- a/test/test.sh +++ b/test/test.sh @@ -87,7 +87,7 @@ defaultTests(){ startupTests(){ echo "Building all languages in startup mode." - export BUILDER_ARGS="runtime --startup" + export BUILDER_ARGS="runtime --cli --startup" export EXPORT_REGISTRY="registry:5000/metacall/builder_startup" export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" From c0cb671b0b22ea890ab212b6590fa8374e44e95e Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:36:42 +0530 Subject: [PATCH 27/32] i think this might work now --- .github/workflows/test.yml | 8 ++++---- test/test.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5e2f4a..a314805 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - - name: Test Rootless - run: ./test/test.sh rootless + # - name: Test Rootless + # run: ./test/test.sh rootless - - name: Test Daemon - run: ./test/test.sh client + # - name: Test Daemon + # run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup diff --git a/test/test.sh b/test/test.sh index 75b7248..3d49231 100755 --- a/test/test.sh +++ b/test/test.sh @@ -95,10 +95,10 @@ startupTests(){ test node/test.js "0123456789" # TODO : Flush env's depedning upon the mode - # echo "Building cli mode with node and py languages." - # export BUILDER_ARGS="runtime --cli py node" - # export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" - # test node/test.js "0123456789" # Should be quicker since all caches are already built + echo "Building cli mode with node and py languages." + export BUILDER_ARGS="runtime --cli py node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } From a6dfdcb3dad2e038e4df87ce286426c09be52a45 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 09:37:08 +0530 Subject: [PATCH 28/32] final testing --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a314805..a5e2f4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - # - name: Test Rootless - # run: ./test/test.sh rootless + - name: Test Rootless + run: ./test/test.sh rootless - # - name: Test Daemon - # run: ./test/test.sh client + - name: Test Daemon + run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup From e41658132088861d6cc07305c853914052bcc2a2 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 22:40:43 +0530 Subject: [PATCH 29/32] triggering only the startup tests with backtrace enabled --- .github/workflows/test.yml | 8 ++++---- pkg/env/env.go | 4 ++-- test/test.sh | 8 +++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5e2f4a..a314805 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - - name: Test Rootless - run: ./test/test.sh rootless + # - name: Test Rootless + # run: ./test/test.sh rootless - - name: Test Daemon - run: ./test/test.sh client + # - name: Test Daemon + # run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup diff --git a/pkg/env/env.go b/pkg/env/env.go index eb50444..2fa0ba9 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -137,8 +137,8 @@ func (e Env) MetaCallBuild(arg string) Env { } func (e Env) MetacallRuntime(arg string) Env { - e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base ports clean %v", arg)). - Root() // TODO: Add backtrace + e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base ports backtrace clean %v", arg)). + Root() return e } diff --git a/test/test.sh b/test/test.sh index 3d49231..20457bd 100755 --- a/test/test.sh +++ b/test/test.sh @@ -86,18 +86,20 @@ defaultTests(){ } startupTests(){ + # Build the startup image with all languages echo "Building all languages in startup mode." export BUILDER_ARGS="runtime --cli --startup" export EXPORT_REGISTRY="registry:5000/metacall/builder_startup" export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" - - # Import registry set to by default test node/test.js "0123456789" - # TODO : Flush env's depedning upon the mode + sleep 5 + + # Testing the cache registry echo "Building cli mode with node and py languages." export BUILDER_ARGS="runtime --cli py node" export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + export EXPORT_REGISTRY="registry:5000/metacall/builder_dump" # To not able to rewrite the cache test node/test.js "0123456789" # Should be quicker since all caches are already built cleanup } From 34af3a658fb88a2d615f0d24dbf503e5052db796 Mon Sep 17 00:00:00 2001 From: ashpect Date: Mon, 26 Aug 2024 23:30:22 +0530 Subject: [PATCH 30/32] cleanup and enable all tests --- .github/workflows/test.yml | 8 ++++---- cmd/builder/runtime.go | 2 -- test/test.sh | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a314805..a5e2f4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - # - name: Test Rootless - # run: ./test/test.sh rootless + - name: Test Rootless + run: ./test/test.sh rootless - # - name: Test Daemon - # run: ./test/test.sh client + - name: Test Daemon + run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index aa6832d..8728d00 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -30,8 +30,6 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { args = staging.GetAllLanguages() } - // fmt.Println(args) - base := cmd.Context().Value(baseKey{}).(llb.State) devBaseLang := staging.DevBase(base, branch, args) diff --git a/test/test.sh b/test/test.sh index ead575b..a11b581 100755 --- a/test/test.sh +++ b/test/test.sh @@ -95,7 +95,6 @@ startupTests(){ export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" test node/test.js "0123456789" - sleep 5 # Testing the cache registry echo "Building cli mode with node and py languages." From deefc5ec12b1722aad239f1535d5a7795846f2e1 Mon Sep 17 00:00:00 2001 From: ashpect Date: Tue, 27 Aug 2024 03:57:04 +0530 Subject: [PATCH 31/32] only add startup-rootless and startup-client as tests --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5e2f4a..a314805 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,11 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - - name: Test Rootless - run: ./test/test.sh rootless + # - name: Test Rootless + # run: ./test/test.sh rootless - - name: Test Daemon - run: ./test/test.sh client + # - name: Test Daemon + # run: ./test/test.sh client - name: Test Startup-Rootless run: ./test/test.sh rootless startup From 16e573be401d788a3efe32658057b2d5437295b9 Mon Sep 17 00:00:00 2001 From: ashpect Date: Tue, 27 Aug 2024 04:16:22 +0530 Subject: [PATCH 32/32] Seperated tests --- .github/workflows/langTest.yml | 37 +++++++++++++++++++ .../workflows/{test.yml => startuptest.yml} | 9 +---- 2 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/langTest.yml rename .github/workflows/{test.yml => startuptest.yml} (81%) diff --git a/.github/workflows/langTest.yml b/.github/workflows/langTest.yml new file mode 100644 index 0000000..7bc95a6 --- /dev/null +++ b/.github/workflows/langTest.yml @@ -0,0 +1,37 @@ +name: Build and Test MetaCall Builder + +on: + push: + branches: + - master + tags: + - "v*.*.*" + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # TODO: + # - name: Lint + # run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all + + - name: Build + run: docker compose build + + - name: Test Binary + env: + BUILDER_ARGS: "runtime rb" + run: docker compose up --exit-code-from binary binary + + - name: Test Rootless + run: ./test/test.sh rootless + + - name: Test Daemon + run: ./test/test.sh client \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/startuptest.yml similarity index 81% rename from .github/workflows/test.yml rename to .github/workflows/startuptest.yml index a314805..8102638 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/startuptest.yml @@ -1,4 +1,5 @@ -name: Build and Test MetaCall Builder +name: Build and Test MetaCall Builder for startup flow +# Seperate because the space is limited in the github actions on: push: @@ -30,12 +31,6 @@ jobs: BUILDER_ARGS: "runtime rb" run: docker compose up --exit-code-from binary binary - # - name: Test Rootless - # run: ./test/test.sh rootless - - # - name: Test Daemon - # run: ./test/test.sh client - - name: Test Startup-Rootless run: ./test/test.sh rootless startup