refactor: Remove coupled Unsloth service, upgrade to vLLM 0.11+

dev/playwright_agent/playwright_agent.py

@@ -4,13 +4,13 @@
 
 import argparse
 import asyncio
+from dataclasses import dataclass
 import json
 import math
 import os
 import time
-import uuid
-from dataclasses import dataclass
 from typing import Any, Dict, List
+import uuid
 
 from dotenv import load_dotenv
 from mcp import ClientSession, StdioServerParameters

dev/swebench/logs.py

@@ -1,10 +1,10 @@
 import logging
 from logging import Handler, LogRecord
 
-import litellm
 from langfuse import Langfuse
 from langfuse.decorators import langfuse_context
 from langfuse.types import SpanLevel
+import litellm
 from sweagent.agent.agents import DefaultAgent
 from sweagent.run.hooks.apply_patch import SaveApplyPatchHook
 

dev/swebench/rollout.py

@@ -1,19 +1,19 @@
 import asyncio
-import json
 from http.client import RemoteDisconnected
+import json
 from pathlib import Path
 from typing import Any, Literal, overload
 
-import modal
-import requests
 from aiolimiter import AsyncLimiter
 from config import get_config
 from eval import eval_instance
 from grpclib.exceptions import StreamTerminatedError
 from instances import Instance
 from langfuse.decorators import observe
 from logs import setup_agent_logger
+import modal
 from pydantic import BaseModel
+import requests
 from requests import adapters as requests_adapters
 from requests.exceptions import ConnectionError, ConnectTimeout, SSLError
 from run import run

dev/swebench/sandbox/test.py

@@ -1,5 +1,5 @@
-import pytest
 from dotenv import load_dotenv
+import pytest
 
 from ..instances import as_instances_iter, get_filtered_swe_smith_instances_df
 from .new import new_sandbox

dev/swebench/train.ipynb

@@ -36,8 +36,8 @@
    "source": [
     "import os\n",
     "\n",
-    "import nest_asyncio\n",
     "from dotenv import load_dotenv\n",
+    "import nest_asyncio\n",
     "\n",
     "# PREPARE ENVIRONMENT\n",
     "os.environ[\"MODAL_IMAGE_BUILDER_VERSION\"] = \"2024.10\"\n",
@@ -60,8 +60,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import torch\n",
     "from rollout import ModelConfig\n",
+    "import torch\n",
     "\n",
     "import art\n",
     "from art.local import LocalBackend\n",

dev/tau-bench/analyze_model_errors.py

@@ -2,13 +2,13 @@
 
 import argparse
 import asyncio
+from datetime import datetime
 import json
 import os
-from datetime import datetime
 from typing import Any, Dict, List
 
-import litellm
 from dotenv import load_dotenv
+import litellm
 from litellm import provider_list
 
 # Import evaluate_model and rollout functions from run_rl

dev/tau-bench/auto_error_identification.py

@@ -1,9 +1,9 @@
 # Copyright Sierra
 
 import argparse
-import json
 from concurrent.futures import ThreadPoolExecutor
 from enum import Enum
+import json
 from typing import Any, Dict, List
 
 from pydantic import BaseModel

dev/tau-bench/benchmark_rl.py

@@ -2,13 +2,13 @@
 
 import argparse
 import asyncio
+from datetime import datetime
 import json
 import os
-from datetime import datetime
 from typing import Any, Dict, List
 
-import litellm
 from dotenv import load_dotenv
+import litellm
 from litellm import provider_list
 
 # Import evaluate_model and rollout functions from run_rl

dev/tau-bench/run.ipynb

@@ -37,11 +37,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import torch\n",
     "from dotenv import load_dotenv\n",
     "from run import RunConfig\n",
     "from run_rl import train\n",
     "from tau_bench.types import TauBenchPolicyConfig, TauBenchTrainingConfig\n",
+    "import torch\n",
     "\n",
     "import art\n",
     "\n",

dev/tau-bench/run_training.py

@@ -4,8 +4,8 @@
 import textwrap
 import traceback
 
-import sky
 from dotenv import dotenv_values
+import sky
 from sky import ClusterStatus
 from tau_bench.types import RunConfig, TauBenchPolicyConfig, TauBenchTrainingConfig
 

dev/tau-bench/tau_bench/envs/airline/tools/book_reservation.py

@@ -1,7 +1,7 @@
 # Copyright Sierra
 
-import json
 from copy import deepcopy
+import json
 from typing import Any, Dict, List
 
 from tau_bench.envs.tool import Tool

dev/tau-bench/tau_bench/envs/airline/tools/update_reservation_flights.py

@@ -1,7 +1,7 @@
 # Copyright Sierra
 
-import json
 from copy import deepcopy
+import json
 from typing import Any, Dict, List
 
 from tau_bench.envs.tool import Tool

dev/tau-bench/tau_bench/envs/base.py

@@ -1,7 +1,7 @@
 # Copyright Sierra
 
-import random
 from hashlib import sha256
+import random
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
 
 from tau_bench.envs.tool import Tool

dev/tau-bench/tau_bench/model_utils/api/cache.py

@@ -1,9 +1,9 @@
+from collections import defaultdict
 import functools
 import hashlib
 import inspect
-import threading
-from collections import defaultdict
 from multiprocessing import Lock
+import threading
 from typing import Any, Callable, TypeVar
 
 from pydantic import BaseModel

dev/tau-bench/tau_bench/model_utils/api/exception.py

@@ -1,7 +1,7 @@
+from concurrent.futures import ThreadPoolExecutor
 import json
 import os
 import time
-from concurrent.futures import ThreadPoolExecutor
 from typing import Any, Callable, TypeVar
 
 from tau_bench.model_utils.model.exception import ModelError, Result

dev/tau-bench/tau_bench/rl_utils.py

@@ -1,8 +1,8 @@
+from datetime import datetime
 import os
 import time
-import uuid
-from datetime import datetime
 from typing import Any, Dict, List, Optional
+import uuid
 
 from langfuse import Langfuse
 from openai import AsyncOpenAI

dev/tau-bench/tau_bench/run.py

@@ -1,14 +1,14 @@
 # Copyright Sierra
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
 import json
+from math import comb
 import multiprocessing
 import os
 import random
 import traceback
-import warnings
-from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime
-from math import comb
 from typing import Any, Dict, List
+import warnings
 
 from langfuse import Langfuse
 from litellm import provider_list

dev/yes-no-maybe-vision/train.ipynb

@@ -34,9 +34,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import openai\n",
     "from dotenv import load_dotenv\n",
     "from generate_images import generate_yes_no_maybe_prompts, save_prompt_images\n",
+    "import openai\n",
     "\n",
     "import art\n",
     "from art.local import LocalBackend\n",

dev/yes-no-maybe.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -12,30 +12,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<style>\n",
-       ".cell-output-ipywidget-background {\n",
-       "    background-color: transparent !important;\n",
-       "}\n",
-       ":root {\n",
-       "    --jp-widgets-color: var(--vscode-editor-foreground);\n",
-       "    --jp-widgets-font-size: var(--vscode-editor-font-size);\n",
-       "}  \n",
-       "</style>\n"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "%%html\n",
     "<style>\n",
@@ -57,8 +36,8 @@
    "source": [
     "from itertools import permutations\n",
     "\n",
-    "import openai\n",
     "from dotenv import load_dotenv\n",
+    "import openai\n",
     "\n",
     "import art\n",
     "from art.local import LocalBackend\n",
@@ -67,7 +46,7 @@
     "\n",
     "backend = LocalBackend()\n",
     "model = art.TrainableModel(\n",
-    "    name=\"009\",\n",
+    "    name=\"010\",\n",
     "    project=\"yes-no-maybe\",\n",
     "    base_model=\"Qwen/Qwen2.5-7B-Instruct\",\n",
     "    # _internal_config=art.dev.InternalModelConfig(\n",
@@ -149,7 +128,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.18"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

dev/yes-no-maybe.py

@@ -0,0 +1,90 @@
+import asyncio
+from itertools import permutations
+import os
+
+from dotenv import load_dotenv
+import openai
+
+import art
+from art.local import LocalBackend
+
+
+async def rollout(client: openai.AsyncOpenAI, prompt: str) -> art.Trajectory:
+    messages: art.Messages = [
+        {
+            "role": "user",
+            "content": prompt,
+        }
+    ]
+    chat_completion = await client.chat.completions.create(
+        messages=messages, model=model.name, max_tokens=100, timeout=100
+    )
+    choice = chat_completion.choices[0]
+    content = choice.message.content
+    assert isinstance(content, str)
+    if content == "yes":
+        reward = 0.5
+    elif content == "no":
+        reward = 0.75
+    elif content == "maybe":
+        reward = 1.0
+    else:
+        reward = 0.0
+    return art.Trajectory(messages_and_choices=[*messages, choice], reward=reward)
+
+
+def with_quotes(w: str) -> str:
+    return f"'{w}'"
+
+
+async def main():
+    load_dotenv()
+
+    backend = LocalBackend()
+    global model
+    base_model = os.environ.get("BASE_MODEL", "Qwen/Qwen3-30B-A3B-Instruct-2507")
+    model = art.TrainableModel(
+        name=os.environ.get("MODEL_NAME", "011"),
+        project="yes-no-maybe",
+        base_model=base_model,
+        _internal_config=art.dev.InternalModelConfig(
+            engine_args=art.dev.EngineArgs(
+                max_lora_rank=1,
+            ),
+            peft_args=art.dev.PeftArgs(
+                r=1,
+            ),
+        ),
+    )
+    await model.register(backend)
+
+    prompts = [
+        f"{prefix} with {', '.join([with_quotes(w) if use_quotes else w for w in words]) if len(words) == 3 else f'{words[0]}' + (f' or {words[1]}' if len(words) > 1 else '')}"
+        for prefix in ["respond", "just respond"]
+        for use_quotes in [True, False]
+        for words in (
+            list(p) for n in [3, 2] for p in permutations(["yes", "no", "maybe"], n)
+        )
+    ]
+
+    openai_client = model.openai_client()
+    max_steps = int(os.environ.get("NUM_STEPS", "4"))
+    start_step = await model.get_step()
+    for _ in range(start_step, start_step + max_steps):
+        train_groups = await art.gather_trajectory_groups(
+            (
+                art.TrajectoryGroup(rollout(openai_client, prompt) for _ in range(32))
+                for prompt in prompts
+            )
+        )
+        await model.train(
+            train_groups,
+            config=art.TrainConfig(learning_rate=1e-4),
+            # _config=art.dev.TrainConfig(
+            #     precalculate_logprobs=True,
+            # ),
+        )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())