Add initial smoketests

.github/workflows/smoketests.yml

@@ -0,0 +1,54 @@
+name: Smoketests
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Target environment"
+        type: choice
+        default: dev
+        options:
+          - dev
+          - prod
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn --frozen-lockfile
+
+      - name: Build
+        run: yarn build
+
+      - name: Configure environment
+        env:
+          DEV_KEY: ${{ secrets.RUNLOOP_SMOKETEST_DEV_API_KEY }}
+          PROD_KEY: ${{ secrets.RUNLOOP_SMOKETEST_PROD_API_KEY }}
+        run: |
+          if [ "${{ github.event.inputs.environment }}" = "prod" ]; then
+            echo "RUNLOOP_API_KEY=${PROD_KEY}" >> $GITHUB_ENV
+            echo "RUNLOOP_BASE_URL=https://api.runloop.ai" >> $GITHUB_ENV
+          else
+            echo "RUNLOOP_API_KEY=${DEV_KEY}" >> $GITHUB_ENV
+            echo "RUNLOOP_BASE_URL=https://api.runloop.pro" >> $GITHUB_ENV
+          fi
+          echo "DEBUG=false" >> $GITHUB_ENV
+          echo "RUN_SMOKETESTS=1" >> $GITHUB_ENV
+
+      - name: Run smoke tests
+        run: |
+          # only run smoke tests; they require a real API key
+          ./node_modules/.bin/jest tests/smoketests --runInBand --verbose --testTimeout=1800000 | cat
+
+

jest.config.ts

@@ -1,5 +1,7 @@
 import type { JestConfigWithTsJest } from 'ts-jest';
 
+const runSmoketests = process.env['RUN_SMOKETESTS'] === '1';
+
 const config: JestConfigWithTsJest = {
   preset: 'ts-jest/presets/default-esm',
   testEnvironment: 'node',
@@ -17,7 +19,11 @@ const config: JestConfigWithTsJest = {
     '<rootDir>/deno/',
     '<rootDir>/deno_tests/',
   ],
-  testPathIgnorePatterns: ['scripts'],
+  testPathIgnorePatterns: [
+    'scripts',
+    // Ignore smoketests unless explicitly enabled via RUN_SMOKETESTS=1
+    ...(runSmoketests ? [] : ['<rootDir>/tests/smoketests/']),
+  ],
 };
 
 export default config;

tests/smoketests/README.md

@@ -0,0 +1,24 @@
+# Smoke tests
+
+End-to-end smoke tests run against the real API to validate critical flows (devboxes, snapshots, blueprints, executions/log tailing, scenarios/benchmarks) and verify custom helpers like `poll()` are importable.
+
+- Local run (requires `RUNLOOP_API_KEY`):
+
+```bash
+export RUNLOOP_API_KEY=...  # required
+# optionally override API base
+# export RUNLOOP_BASE_URL=https://api.runloop.ai
+
+npm run build
+
+# Run all tests
+RUN_SMOKETESTS=1 ./node_modules/.bin/jest tests/smoketests --runInBand --verbose
+
+# Run a single file:
+RUN_SMOKETESTS=1 ./node_modules/.bin/jest tests/smoketests/devboxes.test.ts --runInBand --verbose
+
+# Run a single test:
+RUN_SMOKETESTS=1 ./node_modules/.bin/jest -t "createAndAwaitRunning timeout" --runInBand
+```
+
+- GitHub Actions: add repo secret `RUNLOOP_API_KEY` (and optionally `RUNLOOP_BASE_URL`). The workflow `.github/workflows/smoke.yml` runs on PRs and pushes to `main`.

tests/smoketests/blueprints.test.ts

@@ -0,0 +1,62 @@
+import { makeClient, THIRTY_SECOND_TIMEOUT, uniqueName } from './utils';
+
+const client = makeClient();
+
+describe('smoketest: blueprints', () => {
+  /**
+   * Test the lifecycle of a blueprint. These tests are dependent on each other to save time.
+   */
+  describe('blueprint lifecycle', () => {
+    let blueprintId: string | undefined;
+    let blueprintName = uniqueName('bp');
+
+    afterAll(async () => {
+      await client.blueprints.delete(blueprintId!);
+    });
+
+    test(
+      'create blueprint and await build',
+      async () => {
+        const created = await client.blueprints.createAndAwaitBuildCompleted(
+          {
+            name: blueprintName,
+          },
+          {
+            polling: { maxAttempts: 180, pollingIntervalMs: 5_000, timeoutMs: 30 * 60 * 1000 },
+          },
+        );
+        expect(created.status).toBe('build_complete');
+        blueprintId = created.id;
+      },
+      THIRTY_SECOND_TIMEOUT,
+    );
+
+    test(
+      'start devbox from base blueprint by ID',
+      async () => {
+        const devbox = await client.devboxes.createAndAwaitRunning(
+          { blueprint_id: blueprintId! },
+          {
+            polling: { maxAttempts: 120, pollingIntervalMs: 5_000, timeoutMs: 20 * 60 * 1000 },
+          },
+        );
+        expect(devbox.blueprint_id).toBe(blueprintId);
+      },
+      THIRTY_SECOND_TIMEOUT,
+    );
+
+    test(
+      'start devbox from base blueprint by Name',
+      async () => {
+        const devbox = await client.devboxes.createAndAwaitRunning(
+          { blueprint_name: blueprintName },
+          {
+            polling: { maxAttempts: 120, pollingIntervalMs: 5_000, timeoutMs: 20 * 60 * 1000 },
+          },
+        );
+        expect(devbox.blueprint_id).toBeTruthy();
+      },
+      THIRTY_SECOND_TIMEOUT,
+    );
+  });
+});

tests/smoketests/devboxes.test.ts

@@ -0,0 +1,89 @@
+import { makeClient, THIRTY_SECOND_TIMEOUT, uniqueName } from './utils';
+
+const client = makeClient();
+
+describe('smoketest: devboxes', () => {
+  /**
+   * Test the lifecycle of a devbox. These tests are dependent on each other to save time.
+   */
+  describe('devbox lifecycle', () => {
+    let devboxId: string | undefined;
+
+    test(
+      'create devbox',
+      async () => {
+        const created = await client.devboxes.create({ name: uniqueName('smoke-devbox') });
+        expect(created?.id).toBeTruthy();
+        await client.devboxes.shutdown(created.id);
+      },
+      THIRTY_SECOND_TIMEOUT,
+    );
+
+    test('await running (createAndAwaitRunning)', async () => {
+      const created = await client.devboxes.createAndAwaitRunning(
+        { name: uniqueName('smoketest-devbox2') },
+        {
+          polling: { maxAttempts: 120, pollingIntervalMs: 5_000, timeoutMs: 20 * 60 * 1000 },
+        },
+      );
+      expect(created.status).toBe('running');
+      devboxId = created.id;
+    });
+
+    test('list devboxes', async () => {
+      const page = await client.devboxes.list({ limit: 10 });
+      expect(Array.isArray(page.devboxes)).toBe(true);
+      expect(page.devboxes.length).toBeGreaterThan(0);
+    });
+
+    test('retrieve devbox', async () => {
+      expect(devboxId).toBeTruthy();
+      const view = await client.devboxes.retrieve(devboxId!);
+      expect(view.id).toBe(devboxId);
+    });
+
+    test('shutdown devbox', async () => {
+      expect(devboxId).toBeTruthy();
+      const view = await client.devboxes.shutdown(devboxId!);
+      expect(view.id).toBe(devboxId);
+      expect(view.status).toBe('shutdown');
+    });
+  });
+
+  test(
+    'createAndAwaitRunning long set up',
+    async () => {
+      // createAndAwaitRunning should poll until devbox is running
+      const created = await client.devboxes.createAndAwaitRunning(
+        {
+          name: uniqueName('smoketest-devbox-await-running-long-set-up'),
+          launch_parameters: { launch_commands: ['sleep 70'] },
+        },
+        {
+          polling: { pollingIntervalMs: 5_000, timeoutMs: 80 * 1000 },
+        },
+      );
+      expect(created.status).toBe('running');
+    },
+    THIRTY_SECOND_TIMEOUT * 4,
+  );
+
+  test(
+    'createAndAwaitRunning timeout',
+    async () => {
+      // Fail via exhausting attempts quickly instead of wall-clock timeout
+      await expect(
+        client.devboxes.createAndAwaitRunning(
+          {
+            name: uniqueName('smoketest-devbox-await-running-timeout'),
+            launch_parameters: { launch_commands: ['sleep 70'], keep_alive_time_seconds: 30 },
+          },
+          {
+            polling: { initialDelayMs: 0, pollingIntervalMs: 100, maxAttempts: 1 },
+          },
+        ),
+      ).rejects.toThrow();
+    },
+    THIRTY_SECOND_TIMEOUT * 4,
+  );
+});

tests/smoketests/executions.ts

@@ -0,0 +1,43 @@
+import { makeClient, THIRTY_SECOND_TIMEOUT, uniqueName } from './utils';
+
+const client = makeClient();
+
+describe('smoketest: executions', () => {
+  let devboxId: string | undefined;
+  let execId: string | undefined;
+
+  test(
+    'launch devbox',
+    async () => {
+      const created = await client.devboxes.createAndAwaitRunning(
+        { name: uniqueName('exec-devbox') },
+        {
+          polling: { maxAttempts: 120, pollingIntervalMs: 5_000, timeoutMs: 20 * 60 * 1000 },
+        },
+      );
+      devboxId = created.id;
+    },
+    THIRTY_SECOND_TIMEOUT,
+  );
+
+  test('execute async and await completion', async () => {
+    const started = await client.devboxes.executions.executeAsync(devboxId!, {
+      command: 'echo hello && sleep 1',
+    });
+    execId = started.execution_id;
+    const completed = await client.devboxes.executions.awaitCompleted(devboxId!, execId!, {
+      polling: { maxAttempts: 120, pollingIntervalMs: 2_000, timeoutMs: 10 * 60 * 1000 },
+    });
+    expect(completed.status).toBe('completed');
+  });
+
+  test('tail stdout logs', async () => {
+    const stream = await client.devboxes.executions.streamStdoutUpdates(devboxId!, execId!, {});
+    let received = '';
+    for await (const chunk of stream) {
+      received += chunk.output;
+      if (received.length > 0) break; // stop early to avoid long loops in CI
+    }
+    expect(typeof received).toBe('string');
+  });
+});

tests/smoketests/scenarios-benchmarks.test.ts

@@ -0,0 +1,70 @@
+import { makeClient, THIRTY_SECOND_TIMEOUT, uniqueName } from './utils';
+
+const client = makeClient();
+
+describe('smoketest: scenarios and benchmarks', () => {
+  let scenarioId: string | undefined;
+  let runId: string | undefined;
+
+  test(
+    'create scenario',
+    async () => {
+      const scenario = await client.scenarios.create({
+        name: uniqueName('scenario'),
+        input_context: { problem_statement: 'echo hello' },
+        scoring_contract: {
+          scoring_function_parameters: [
+            {
+              name: 'cmd-zero',
+              scorer: { type: 'command_scorer', command: 'true' },
+              weight: 1,
+            },
+          ],
+        },
+      });
+      scenarioId = scenario.id;
+    },
+    THIRTY_SECOND_TIMEOUT,
+  );
+
+  test(
+    'start scenario run and await env ready',
+    async () => {
+      const run = await client.scenarios.startRunAndAwaitEnvReady(
+        { scenario_id: scenarioId! },
+        {
+          polling: { maxAttempts: 120, pollingIntervalMs: 5_000, timeoutMs: 20 * 60 * 1000 },
+        },
+      );
+      expect(run.scenario_id).toBe(scenarioId);
+      runId = run.id;
+    },
+    THIRTY_SECOND_TIMEOUT,
+  );
+
+  test(
+    'score and complete scenario run',
+    async () => {
+      const scored = await client.scenarios.runs.scoreAndComplete(runId!, {
+        polling: { maxAttempts: 120, pollingIntervalMs: 5_000, timeoutMs: 20 * 60 * 1000 },
+      });
+      expect(['completed', 'scored', 'running', 'failed', 'timeout', 'canceled']).toContain(scored.state);
+    },
+    THIRTY_SECOND_TIMEOUT,
+  );
+
+  test(
+    'create benchmark and start run',
+    async () => {
+      const benchmark = await client.benchmarks.create({
+        name: uniqueName('benchmark'),
+        scenario_ids: [scenarioId!],
+      });
+      expect(benchmark.id).toBeTruthy();
+
+      const run = await client.benchmarks.startRun({ benchmark_id: benchmark.id });
+      expect(run.benchmark_id).toBe(benchmark.id);
+    },
+    THIRTY_SECOND_TIMEOUT,
+  );
+});