diff --git a/package-lock.json b/package-lock.json index 22412cfd69f..ca9830bd54d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -132,6 +132,8 @@ "integrity": "sha512-+8oDYc4J5cCaWZh1VUbyc+cegGplJO9FqHpqR4LVAVx8fRLVRaYlC4yyA6cqHJ1vWP23Ff/ECS5U68Zz6OLZlg==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "escape-string-regexp": "^5.0.0", "execa": "^9.6.0" @@ -146,6 +148,8 @@ "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -357,422 +361,6 @@ "node": ">=18" } }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", - "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz", - "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", - "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz", - "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", - "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", - "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", - "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", - "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", - "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", - "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", - "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", - "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", - "cpu": [ - "loong64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", - "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", - "cpu": [ - "mips64el" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", - "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", - "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", - "cpu": [ - "riscv64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", - "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", - "cpu": [ - "s390x" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", - "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", - "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", - "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", - "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", - "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz", - "integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", - "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", - "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", - "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", - "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -1995,7 +1583,9 @@ "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/@sindresorhus/merge-streams": { "version": "2.3.0", @@ -4443,6 +4033,8 @@ "integrity": "sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sindresorhus/merge-streams": "^4.0.0", "cross-spawn": "^7.0.6", @@ -4470,19 +4062,8 @@ "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", "dev": true, "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/execa/node_modules/is-stream": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", - "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", - "dev": true, - "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=18" }, @@ -5234,6 +4815,8 @@ "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@sec-ant/readable-stream": "^0.4.1", "is-stream": "^4.0.1" @@ -5664,6 +5247,8 @@ "integrity": "sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==", "dev": true, "license": "Apache-2.0", + "optional": true, + "peer": true, "engines": { "node": ">=18.18.0" } @@ -5892,6 +5477,8 @@ "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -5927,6 +5514,8 @@ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=8" }, @@ -6818,6 +6407,8 @@ "integrity": "sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "path-key": "^4.0.0", "unicorn-magic": "^0.3.0" @@ -6835,6 +6426,8 @@ "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=12" }, @@ -8273,6 +7866,8 @@ "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=18" }, @@ -9403,6 +8998,8 @@ "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=18" }, @@ -9869,4 +9466,4 @@ } } } -} +} \ No newline at end of file diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts new file mode 100644 index 00000000000..35822a034f6 --- /dev/null +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -0,0 +1,191 @@ + +export { executeOpalAdkStream }; +export type { StreamingRequestBody, StreamChunk }; + +import { + Capabilities, + FileSystemReadWritePath, + LLMContent, + Outcome, +} from "@breadboard-ai/types"; +import { StreamableReporter } from "./output.js"; +import { + decodeBase64, + encodeBase64, + err, + ok, + toLLMContent, +} from "./utils.js"; +import { A2ModuleArgs } from "../runnable-module-factory.js"; +import { iteratorFromStream } from "@breadboard-ai/utils"; + +const DEFAULT_OPAL_ADK_ENDPOINT = + "https://staging-appcatalyst.sandbox.googleapis.com/v1beta1/executeStepStream"; + +type StreamChunk = { + mimetype: string; + data: any; + metadata?: { + chunk_type: string; + }; +}; + + +export type Content = { + chunks: StreamChunk[]; +}; + +export type ContentMap = { + [key: string]: Content; +}; + + +export type PlanStep = { + stepName: string; + modelApi: string; + inputParameters: string[]; + output?: string; +}; + + +type StreamingRequestBody = { + planStep: PlanStep; + executionInputs: ContentMap; +}; + +async function getOpalAdkBackendUrl(caps: Capabilities) { + type BackendSettings = { endpoint_url: string }; + const reading = await caps.read({ path: "/env/settings/opalAdkBackend" }); + if (ok(reading)) { + const part = reading.data?.at(0)?.parts?.at(0); + if (part && "json" in part) { + const settings = part.json as BackendSettings; + if (settings?.endpoint_url) { + // Extract base URL and append the streaming endpoint path + const url = new URL(settings.endpoint_url); + url.pathname = "/v1beta1/executeStepStream"; + return url.toString(); + } + } + } + return DEFAULT_OPAL_ADK_ENDPOINT; +} + +function buildStreamingRequestBody( + opal_adk_agent: string, + params: Record +): StreamingRequestBody { + const inputParameters = Object.keys(params); + const execution_inputs = Object.fromEntries( + Object.entries(params).map(([name, value]) => { + return [ + name, + { + chunks: [ + { + mimetype: "text/plain", + data: encodeBase64(value), + }, + ], + }, + ]; + }) + ); + return { + planStep: { + stepName: "plan_step", + modelApi: opal_adk_agent, + inputParameters: inputParameters, + }, + executionInputs: execution_inputs, + }; +} + + +async function executeOpalAdkStream(caps: Capabilities, + moduleArgs: A2ModuleArgs, + params: Record, + opal_adk_agent: string): Promise> { + console.log("params: ", params); + const reporter = new StreamableReporter(caps, { + title: `Executing Opal Adk with ${opal_adk_agent}`, + icon: "spark", + }); + try { + await reporter.start(); + await reporter.sendUpdate("Preparing request", { opal_adk_agent }, "upload"); + + const baseUrl = await getOpalAdkBackendUrl(caps); + const url = new URL(baseUrl); + url.searchParams.set("alt", "sse"); + const requestBody = buildStreamingRequestBody( + opal_adk_agent, + params + ); + + // Record model call with action tracker + caps.write({ + path: `/mnt/track/call_${opal_adk_agent}` as FileSystemReadWritePath, + data: [], + }); + + const response = await moduleArgs.fetchWithCreds(url.toString(), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(requestBody), + signal: moduleArgs.context.signal, + }); + + if (!response.ok) { + const errorText = await response.text(); + return reporter.sendError( + err(`Streaming request failed: ${response.status} ${errorText}`) + ); + } + + if (!response.body) { + return reporter.sendError(err("No response body from streaming API")); + } + + // Process the SSE stream + let researchResult = ""; + let thoughtCount = 0; + for await (const chunk of iteratorFromStream(response.body)) { + if (!chunk) continue; + console.log("chunk", chunk); + console.log("chunk data: ", decodeBase64(chunk.data)) + const chunkType = chunk.mimetype; + const text = decodeBase64(chunk.data); + + if (chunkType === "thought") { + thoughtCount++; + await reporter.sendUpdate( + `Thinking (${thoughtCount})`, + text, + "spark" + ); + } else if (chunkType === "text/plain") { + researchResult = text; + console.log("Updating output") + await reporter.sendUpdate( + "Agent Thought", + researchResult, + "spark" + ); + } else if (chunkType === "error") { + return reporter.sendError(err(`Generation error: ${text}`)); + } + } + + if (!researchResult) { + return reporter.sendError(err("No research result received from stream")); + } + + // Return HTML as inlineData with text/html mimeType to match legacy behavior + return toLLMContent(researchResult, 'model'); + } catch (e) { + return reporter.sendError(err((e as Error).message)); + } finally { + reporter.close(); + } +} \ No newline at end of file diff --git a/packages/visual-editor/src/a2/a2/step-executor.ts b/packages/visual-editor/src/a2/a2/step-executor.ts index 662b42a90cb..d32ae89d31f 100644 --- a/packages/visual-editor/src/a2/a2/step-executor.ts +++ b/packages/visual-editor/src/a2/a2/step-executor.ts @@ -4,293 +4,293 @@ export { executeStep, executeTool, parseExecutionOutput }; - import { - Capabilities, - FileSystemReadWritePath, - InlineDataCapabilityPart, - JsonSerializable, - LLMContent, - Outcome, - } from "@breadboard-ai/types"; - import { getCurrentStepState, StreamableReporter } from "./output.js"; - import { - decodeBase64, - encodeBase64, - err, - ErrorMetadata, - ErrorWithMetadata, - ok, - toLLMContentInline, - toLLMContentStored, - } from "./utils.js"; - import { A2ModuleArgs } from "../runnable-module-factory.js"; +import { + Capabilities, + FileSystemReadWritePath, + InlineDataCapabilityPart, + JsonSerializable, + LLMContent, + Outcome, +} from "@breadboard-ai/types"; +import { getCurrentStepState, StreamableReporter } from "./output.js"; +import { + decodeBase64, + encodeBase64, + err, + ErrorMetadata, + ErrorWithMetadata, + ok, + toLLMContentInline, + toLLMContentStored, +} from "./utils.js"; +import { A2ModuleArgs } from "../runnable-module-factory.js"; - const DEFAULT_BACKEND_ENDPOINT = - "https://staging-appcatalyst.sandbox.googleapis.com/v1beta1/executeStep"; +const DEFAULT_BACKEND_ENDPOINT = + "https://staging-appcatalyst.sandbox.googleapis.com/v1beta1/executeStep"; - type Chunk = { - mimetype: string; - data: string; - substreamName?: string; - }; +type Chunk = { + mimetype: string; + data: string; + substreamName?: string; +}; - export type Content = { - chunks: Chunk[]; - }; +export type Content = { + chunks: Chunk[]; +}; - export type ContentMap = { - [key: string]: Content; - }; +export type ContentMap = { + [key: string]: Content; +}; - export type PlanStep = { - stepName: string; - modelApi: string; - inputParameters: string[]; - systemPrompt?: string; - stepIntent?: string; - output?: string; - isListOutput?: boolean; - options?: { - disablePromptRewrite?: boolean; - renderMode?: string; - modelName?: string; - systemInstruction?: string; - }; +export type PlanStep = { + stepName: string; + modelApi: string; + inputParameters: string[]; + systemPrompt?: string; + stepIntent?: string; + output?: string; + isListOutput?: boolean; + options?: { + disablePromptRewrite?: boolean; + renderMode?: string; + modelName?: string; + systemInstruction?: string; }; +}; - export type GcsConfig = { - bucket_name: string; - folder_path?: string; - project_name?: string; - }; +export type GcsConfig = { + bucket_name: string; + folder_path?: string; + project_name?: string; +}; - export type ExecuteStepRequest = { - planStep: PlanStep; - execution_inputs: ContentMap; - }; +export type ExecuteStepRequest = { + planStep: PlanStep; + execution_inputs: ContentMap; +}; - export type ExecuteStepResponse = { - executionOutputs: ContentMap; - errorMessage?: string; - }; +export type ExecuteStepResponse = { + executionOutputs: ContentMap; + errorMessage?: string; +}; - export type ExecuteStepErrorResponse = { - error: { - code: number; - message: string; - status: string; - details: unknown; - }; +export type ExecuteStepErrorResponse = { + error: { + code: number; + message: string; + status: string; + details: unknown; }; +}; - type ExecutionOutput = { - chunks: LLMContent[]; - requestedModel?: string; - executedModel?: string; - }; +type ExecutionOutput = { + chunks: LLMContent[]; + requestedModel?: string; + executedModel?: string; +}; - const GCS_PATH_PREFIX = "text/gcs-path/"; +const GCS_PATH_PREFIX = "text/gcs-path/"; - function parseExecutionOutput(input?: Chunk[]): Outcome { - let requestedModel: string | undefined = undefined; - let executedModel: string | undefined = undefined; - const chunks: LLMContent[] = []; - input?.forEach((chunk) => { - if (chunk.substreamName === "requested-model") { - requestedModel = chunk.data; - } else if (chunk.substreamName === "executed-model") { - executedModel = chunk.data; - } else { - chunks.push(toLLMContent(chunk)); - } - }); - if (chunks.length === 0) { - return err(`Unable to find data in the output`, { - origin: "server", - kind: "bug", - }); +function parseExecutionOutput(input?: Chunk[]): Outcome { + let requestedModel: string | undefined = undefined; + let executedModel: string | undefined = undefined; + const chunks: LLMContent[] = []; + input?.forEach((chunk) => { + if (chunk.substreamName === "requested-model") { + requestedModel = chunk.data; + } else if (chunk.substreamName === "executed-model") { + executedModel = chunk.data; + } else { + chunks.push(toLLMContent(chunk)); } - return { chunks, requestedModel, executedModel }; + }); + if (chunks.length === 0) { + return err(`Unable to find data in the output`, { + origin: "server", + kind: "bug", + }); + } + return { chunks, requestedModel, executedModel }; - function toLLMContent({ mimetype, data }: Chunk): LLMContent { - if (mimetype === "text/html") { - return toLLMContentInline(mimetype, decodeBase64(data)); - } else if (mimetype.endsWith("/storedData")) { - return toLLMContentStored(mimetype.replace("/storedData", ""), data); - } else if (mimetype.startsWith(GCS_PATH_PREFIX)) { - const gcsPath = new TextDecoder().decode( - Uint8Array.from(atob(data), (m) => m.codePointAt(0)!) - ); - const handle = new URL( - `/board/blobs/${gcsPath.split("/").at(-1)}`, - window.location.href - ).href; - const actualMimeType = mimetype.slice(GCS_PATH_PREFIX.length); - return toLLMContentStored(actualMimeType, handle); - } - return toLLMContentInline(mimetype, data); + function toLLMContent({ mimetype, data }: Chunk): LLMContent { + if (mimetype === "text/html") { + return toLLMContentInline(mimetype, decodeBase64(data)); + } else if (mimetype.endsWith("/storedData")) { + return toLLMContentStored(mimetype.replace("/storedData", ""), data); + } else if (mimetype.startsWith(GCS_PATH_PREFIX)) { + const gcsPath = new TextDecoder().decode( + Uint8Array.from(atob(data), (m) => m.codePointAt(0)!) + ); + const handle = new URL( + `/board/blobs/${gcsPath.split("/").at(-1)}`, + window.location.href + ).href; + const actualMimeType = mimetype.slice(GCS_PATH_PREFIX.length); + return toLLMContentStored(actualMimeType, handle); } + return toLLMContentInline(mimetype, data); } +} - async function executeTool< - T extends JsonSerializable = Record, - >( - caps: Capabilities, - moduleArgs: A2ModuleArgs, - api: string, - params: Record - ): Promise> { - const inputParameters = Object.keys(params); - const execution_inputs = Object.fromEntries( - Object.entries(params).map(([name, value]) => { - return [ - name, - { - chunks: [ - { - mimetype: "text/plan", - data: encodeBase64(value), - }, - ], - }, - ]; - }) - ); - const response = await executeStep(caps, moduleArgs, { - planStep: { - stepName: api, - modelApi: api, - output: "data", - inputParameters, - isListOutput: false, - }, - execution_inputs, - }); - if (!ok(response)) return response; +async function executeTool< + T extends JsonSerializable = Record, +>( + caps: Capabilities, + moduleArgs: A2ModuleArgs, + api: string, + params: Record +): Promise> { + const inputParameters = Object.keys(params); + const execution_inputs = Object.fromEntries( + Object.entries(params).map(([name, value]) => { + return [ + name, + { + chunks: [ + { + mimetype: "text/plan", + data: encodeBase64(value), + }, + ], + }, + ]; + }) + ); + const response = await executeStep(caps, moduleArgs, { + planStep: { + stepName: api, + modelApi: api, + output: "data", + inputParameters, + isListOutput: false, + }, + execution_inputs, + }); + if (!ok(response)) return response; - const { - inlineData: { data }, - } = response.chunks.at(0)!.parts.at(0) as InlineDataCapabilityPart; - const jsonString = decodeBase64(data!); - try { - return JSON.parse(jsonString) as T; - } catch { - return jsonString; - } + const { + inlineData: { data }, + } = response.chunks.at(0)!.parts.at(0) as InlineDataCapabilityPart; + const jsonString = decodeBase64(data!); + try { + return JSON.parse(jsonString) as T; + } catch { + return jsonString; } +} - type BackendSettings = { - endpoint_url: string; - }; - async function getBackendUrl(caps: Capabilities) { - const reading = await caps.read({ path: "/env/settings/backend" }); - if (ok(reading)) { - const part = reading.data?.at(0)?.parts?.at(0); - if (part && "json" in part) { - const settings = part.json as BackendSettings; - if (settings && settings.endpoint_url) { - return settings.endpoint_url; - } +type BackendSettings = { + endpoint_url: string; +}; +async function getBackendUrl(caps: Capabilities) { + const reading = await caps.read({ path: "/env/settings/backend" }); + if (ok(reading)) { + const part = reading.data?.at(0)?.parts?.at(0); + if (part && "json" in part) { + const settings = part.json as BackendSettings; + if (settings && settings.endpoint_url) { + return settings.endpoint_url; } } - return DEFAULT_BACKEND_ENDPOINT; } + return DEFAULT_BACKEND_ENDPOINT; +} - type ProgressUpdateOptions = { - message?: string; - expectedDurationInSec?: number; - }; +type ProgressUpdateOptions = { + message?: string; + expectedDurationInSec?: number; +}; - async function executeStep( - caps: Capabilities, - moduleArgs: A2ModuleArgs, - body: ExecuteStepRequest, - progressUpdateOptions?: ProgressUpdateOptions - ): Promise> { - const { fetchWithCreds, context } = moduleArgs; - const model = body.planStep.options?.modelName || body.planStep.stepName; - const { appScreen, title } = getCurrentStepState(moduleArgs); - const reporter = new StreamableReporter(caps, { - title: `Calling ${model}`, - icon: "spark", - }); - try { - if (appScreen) { - appScreen.progress = progressUpdateOptions?.message || title; - if (progressUpdateOptions?.expectedDurationInSec) { - appScreen.expectedDuration = - progressUpdateOptions.expectedDurationInSec; - } else { - appScreen.expectedDuration = -1; - } +async function executeStep( + caps: Capabilities, + moduleArgs: A2ModuleArgs, + body: ExecuteStepRequest, + progressUpdateOptions?: ProgressUpdateOptions +): Promise> { + const { fetchWithCreds, context } = moduleArgs; + const model = body.planStep.options?.modelName || body.planStep.stepName; + const { appScreen, title } = getCurrentStepState(moduleArgs); + const reporter = new StreamableReporter(caps, { + title: `Calling ${model}`, + icon: "spark", + }); + try { + if (appScreen) { + appScreen.progress = progressUpdateOptions?.message || title; + if (progressUpdateOptions?.expectedDurationInSec) { + appScreen.expectedDuration = + progressUpdateOptions.expectedDurationInSec; + } else { + appScreen.expectedDuration = -1; } + } - await reporter.start(); - await reporter.sendUpdate("Step Input", elideEncodedData(body), "upload"); - // Call the API. - const url = await getBackendUrl(caps); - // Record model call with action tracker. - caps.write({ - path: `/mnt/track/call_${model}` as FileSystemReadWritePath, - data: [], + await reporter.start(); + await reporter.sendUpdate("Step Input", elideEncodedData(body), "upload"); + // Call the API. + const url = await getBackendUrl(caps); + // Record model call with action tracker. + caps.write({ + path: `/mnt/track/call_${model}` as FileSystemReadWritePath, + data: [], + }); + let response: ExecuteStepResponse; + try { + const fetchResponse = await fetchWithCreds(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + signal: context.signal, + body: JSON.stringify(body), }); - let response: ExecuteStepResponse; - try { - const fetchResponse = await fetchWithCreds(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - signal: context.signal, - body: JSON.stringify(body), - }); - if (!fetchResponse.ok) { - const { $error, metadata } = decodeFetchError( - await fetchResponse.text(), - model - ); - return reporter.sendError(err($error, metadata)); - } - response = await fetchResponse.json(); - } catch (e) { - return reporter.sendError( - err((e as Error).message, { - origin: "server", - model, - }) - ); - } - if (!response) { - return await reporter.sendError( - err(`Request to "${model}" failed, please try again`, { - origin: "server", - kind: "bug", - }) + if (!fetchResponse.ok) { + const { $error, metadata } = decodeFetchError( + await fetchResponse.text(), + model ); + return reporter.sendError(err($error, metadata)); } - if (response.errorMessage) { - const errorMessage = decodeMetadata(response.errorMessage, model); - return await reporter.sendError( - err(errorMessage.$error, errorMessage.metadata) - ); - } - await reporter.sendUpdate( - "Step Output", - elideEncodedData(response), - "download" + response = await fetchResponse.json(); + } catch (e) { + return reporter.sendError( + err((e as Error).message, { + origin: "server", + model, + }) ); - const output_key = body.planStep.output || ""; - return parseExecutionOutput( - response.executionOutputs[output_key]?.chunks + } + if (!response) { + return await reporter.sendError( + err(`Request to "${model}" failed, please try again`, { + origin: "server", + kind: "bug", + }) + ); + } + if (response.errorMessage) { + const errorMessage = decodeMetadata(response.errorMessage, model); + return await reporter.sendError( + err(errorMessage.$error, errorMessage.metadata) ); - } finally { - await reporter.close(); - if (appScreen) { - appScreen.progress = undefined; - appScreen.expectedDuration = -1; - } + } + await reporter.sendUpdate( + "Step Output", + elideEncodedData(response), + "download" + ); + const output_key = body.planStep.output || ""; + return parseExecutionOutput( + response.executionOutputs[output_key]?.chunks + ); + } finally { + await reporter.close(); + if (appScreen) { + appScreen.progress = undefined; + appScreen.expectedDuration = -1; } } +} export function elideEncodedData(obj: T): T { if (obj === null || typeof obj !== "object") return obj; diff --git a/packages/visual-editor/src/a2/deep-research/main.ts b/packages/visual-editor/src/a2/deep-research/main.ts index 2b1dffc0d1b..6103ee2ced1 100644 --- a/packages/visual-editor/src/a2/deep-research/main.ts +++ b/packages/visual-editor/src/a2/deep-research/main.ts @@ -3,6 +3,9 @@ */ import { Capabilities, LLMContent, Schema } from "@breadboard-ai/types"; import { type Params } from "../a2/common.js"; +import { Template } from "../a2/template.js"; +import { A2ModuleArgs } from "../runnable-module-factory.js"; +import { executeOpalAdkStream } from "../a2/opal-adk-stream.js"; import invokeGemini, { type GeminiInputs, type Tool, @@ -10,10 +13,8 @@ import invokeGemini, { } from "../a2/gemini.js"; import { ArgumentNameGenerator } from "../a2/introducer.js"; import { report } from "../a2/output.js"; -import { Template } from "../a2/template.js"; import { ToolManager } from "../a2/tool-manager.js"; import { addUserTurn, err, llm, ok, toLLMContent } from "../a2/utils.js"; -import { A2ModuleArgs } from "../runnable-module-factory.js"; export { invoke as default, describe }; @@ -139,11 +140,57 @@ async function thought( }); } -async function invoke( +export function unwrapParams(params: Params): Record { + return Object.fromEntries( + Object.entries(params).map(([key, value]) => { + if (key.startsWith("p-z-")) { + return [key.slice(4), value]; + } + return [key, value]; + }) + ); +} + +function extractTextFromLLMContent(content: [LLMContent]): string { + const query = content[0] + if (!query.parts || query.parts.length === 0) { + return ""; + } + if (query.parts.length > 1) { + throw new Error("LLMContent contains more than one part."); + } + const firstPart = query.parts[0]; + if ("text" in firstPart && typeof firstPart.text === "string") { + return firstPart.text; + } + return ""; +} + + +async function invokeOpalAdk( + { context, query, summarize, ...params }: ResearcherInputs, + caps: Capabilities, + moduleArgs: A2ModuleArgs +) { + const unwrappedParams = unwrapParams(params); + const userQuery = unwrappedParams['ask_user_research_query'] as [LLMContent]; + if (!userQuery) { + return err("No query provided"); + } + console.log("unwrapped params", unwrappedParams); + const results = await executeOpalAdkStream(caps, moduleArgs, { "query": extractTextFromLLMContent(userQuery) }, "deep_research"); + console.log("deep-research results", results) + return { + context: [...(context || []), results] + }; +} + +async function invokeLegacy( { context, query, summarize, ...params }: ResearcherInputs, caps: Capabilities, moduleArgs: A2ModuleArgs ) { + console.log('calling deep research agent.') const tools = RESEARCH_TOOLS.map((descriptor) => descriptor.url); const toolManager = new ToolManager( caps, @@ -159,6 +206,7 @@ async function invoke( if (!ok(substituting)) { return substituting; } + if (!toolManager.hasTools()) { // If no tools supplied (legacy case, actually), initialize // with a set of default tools. @@ -234,6 +282,20 @@ async function invoke( }; } +async function invoke( + { context, query, summarize, ...params }: ResearcherInputs, + caps: Capabilities, + moduleArgs: A2ModuleArgs +) { + const flags = await moduleArgs.context.flags?.flags(); + const opalAdkEnabled = flags?.opalAdk || false; + if (opalAdkEnabled) { + return invokeOpalAdk({ context, query, summarize, ...params }, caps, moduleArgs); + } else { + return invokeLegacy({ context, query, summarize, ...params }, caps, moduleArgs); + } +} + type DescribeInputs = { inputs: { query: LLMContent; diff --git a/packages/visual-editor/src/init.ts b/packages/visual-editor/src/init.ts index e1a06ed18e1..4b2f0c06e36 100644 --- a/packages/visual-editor/src/init.ts +++ b/packages/visual-editor/src/init.ts @@ -19,6 +19,11 @@ const executeStepEndpoint: string = new URL( OPAL_BACKEND_API_PREFIX ).href; +const executeOpalAdkStepEndpoint: string = new URL( + "v1beta1/executeStepStream", + OPAL_BACKEND_API_PREFIX +).href; + bootstrap({ deploymentConfiguration, env: [ @@ -26,6 +31,10 @@ bootstrap({ path: "/env/settings/backend", data: toLLMContent({ endpoint_url: executeStepEndpoint }), }, + { + path: "/env/settings/opalAdkBackend", + data: toLLMContent({ endpoint_url: executeOpalAdkStepEndpoint }), + }, ], });